From bb9cd0f13546fb42ba20bb13978917d67efcae8b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 11:36:36 -0700 Subject: [PATCH 01/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- Dockerfiles/arkime.Dockerfile | 31 ++++- arkime/etc/config.ini | 106 +++++++++------- arkime/scripts/docker_entrypoint.sh | 43 +++++++ arkime/supervisord.conf | 27 +++- config/arkime-live.env.example | 7 ++ config/arkime-offline.env.example | 12 ++ config/arkime.env.example | 2 - docs/malcolm-config.md | 2 +- .../interface/sensor_ctl/arkime/config.ini | 65 +++++----- shared/bin/pcap_processor.py | 119 +++++++++++------- 10 files changed, 283 insertions(+), 131 deletions(-) create mode 100644 config/arkime-live.env.example create mode 100644 config/arkime-offline.env.example diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 96701db93..d50dfa200 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -92,7 +92,13 @@ ENV DEFAULT_UID $DEFAULT_UID ENV DEFAULT_GID $DEFAULT_GID ENV PUSER "arkime" ENV PGROUP "arkime" -ENV PUSER_PRIV_DROP true +# not dropping privileges globally: supervisord will take care of it +# for all processes, but first we need root to sure capabilities for +# traffic capturing tools are in-place before they are started. +# despite doing setcap here in the Dockerfile, the chown in +# docker-uid-gid-setup.sh will cause them to be lost, so we need +# a final check in docker_entrypoint.sh before startup +ENV PUSER_PRIV_DROP false ENV PUSER_RLIMIT_UNLOCK true ENV DEBIAN_FRONTEND noninteractive @@ -106,12 +112,19 @@ ARG MALCOLM_USERNAME=admin ARG ARKIME_ECS_PROVIDER=arkime ARG ARKIME_ECS_DATASET=session ARG ARKIME_INTERFACE=eth0 -ARG ARKIME_ANALYZE_PCAP_THREADS=1 +ARG ARKIME_AUTO_ANALYZE_PCAP_FILES=false +ARG ARKIME_AUTO_ANALYZE_PCAP_THREADS=1 ARG OPENSEARCH_MAX_SHARDS_PER_NODE=2500 ARG WISE=on ARG VIEWER=on #Whether or not Arkime is in charge of deleting old PCAP files to reclaim space ARG MANAGE_PCAP_FILES=false +ARG ARKIME_PCAP_PROCESSOR=true +ARG ARKIME_LIVE_CAPTURE=false +ARG ARKIME_ROTATED_PCAP=true +ARG ARKIME_COMPRESSION_TYPE=none +ARG ARKIME_COMPRESSION_LEVEL=0 + #Whether or not to auto-tag logs based on filename ARG AUTO_TAG=true ARG PCAP_PIPELINE_VERBOSITY="" @@ -130,7 +143,13 @@ ENV ARKIME_PASSWORD "ignored" ENV ARKIME_ECS_PROVIDER $ARKIME_ECS_PROVIDER ENV ARKIME_ECS_DATASET $ARKIME_ECS_DATASET ENV ARKIME_DIR "/opt/arkime" -ENV ARKIME_ANALYZE_PCAP_THREADS $ARKIME_ANALYZE_PCAP_THREADS +ENV ARKIME_AUTO_ANALYZE_PCAP_FILES $ARKIME_AUTO_ANALYZE_PCAP_FILES +ENV ARKIME_AUTO_ANALYZE_PCAP_THREADS $ARKIME_AUTO_ANALYZE_PCAP_THREADS +ENV ARKIME_PCAP_PROCESSOR $ARKIME_PCAP_PROCESSOR +ENV ARKIME_LIVE_CAPTURE $ARKIME_LIVE_CAPTURE +ENV ARKIME_COMPRESSION_TYPE $ARKIME_COMPRESSION_TYPE +ENV ARKIME_COMPRESSION_LEVEL $ARKIME_COMPRESSION_LEVEL +ENV ARKIME_ROTATED_PCAP $ARKIME_ROTATED_PCAP ENV OPENSEARCH_MAX_SHARDS_PER_NODE $OPENSEARCH_MAX_SHARDS_PER_NODE ENV WISE $WISE ENV VIEWER $VIEWER @@ -147,7 +166,9 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ apt-get install -q -y --no-install-recommends \ + bc \ curl \ + ethtool \ file \ geoip-bin \ gettext \ @@ -191,6 +212,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --chmod=755 shared/bin/self_signed_key_gen.sh /usr/local/bin/ +COPY --chmod=755 shared/bin/nic-capture-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/opensearch_status.sh /opt COPY --chmod=755 shared/bin/pcap_processor.py /opt/ COPY --chmod=644 shared/bin/pcap_utils.py /opt/ @@ -226,6 +248,9 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ cp -f /opt/arkime_update_geo.sh $ARKIME_DIR/bin/arkime_update_geo.sh && \ mv $ARKIME_DIR/etc/config.ini $ARKIME_DIR/etc/config.orig.ini && \ chmod u+s $ARKIME_DIR/bin/capture && \ + chown root:${PGROUP} /sbin/ethtool $ARKIME_DIR/bin/capture && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ mkdir -p /var/run/arkime && \ chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime #Update Path diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 3caf5a270..83fa01f82 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -4,69 +4,81 @@ # [default] -elasticsearch=http://opensearch:9200 +antiSynDrop=false +certFile=/opt/arkime/etc/viewer.crt +compressES=false cronQueries=true -rotateIndex=daily -passwordSecret=Malcolm +dropGroup=arkime +dropUser=arkime +elasticsearch=http://opensearch:9200 +freeSpaceG=10% +geoLite2ASN=/opt/arkime/etc/GeoLite2-ASN.mmdb +geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb httpRealm=Arkime -userAuthIps=::,0.0.0.0/0 +icmpTimeout=10 interface=eth0 -wiseHost=127.0.0.1 -wisePort=8081 +keyFile=/opt/arkime/etc/viewer.key +logESRequests=false +logEveryXPackets=500000 +logFileCreation=true +logHTTPConnections=false +logUnknownProtocols=false +maxESConns=30 +maxESRequests=500 +maxFileSizeG=4 +maxFileTimeM=180 +maxPackets=10000 +maxReqBody=64 +maxStreams=1000000 +ouiFile=/opt/arkime/etc/oui.txt +packetsPerPoll=50000 +parseQSValue=false +parsersDir=/opt/arkime/parsers +parseSMB=true +parseSMTP=true +passwordSecret=Malcolm pcapDir=/data/pcap/processed +plugins=wise.so +pluginsDir=/opt/arkime/plugins readTruncatedPackets=true -maxFileSizeG=4 -tcpTimeout=600 +reqBodyOnlyUtf8=true +rirFile=/opt/arkime/etc/ipv4-address-space.csv +rotateIndex=daily +smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: +spiDataMaxIndices=7 +supportSha256=false tcpSaveTimeout=720 +tcpTimeout=600 udpTimeout=30 -icmpTimeout=10 -maxStreams=1000000 -maxPackets=10000 -freeSpaceG=10% +uploadCommand= +userAuthIps=::,0.0.0.0/0 +viewerPlugins=wise.js viewPort=8005 -certFile=/opt/arkime/etc/viewer.crt -keyFile=/opt/arkime/etc/viewer.key -geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb -geoLite2ASN=/opt/arkime/etc/GeoLite2-ASN.mmdb -rirFile=/opt/arkime/etc/ipv4-address-space.csv -ouiFile=/opt/arkime/etc/oui.txt -dropUser=arkime -dropGroup=arkime +wiseHost=127.0.0.1 +wisePort=8081 # implicit auto-creation of users for Arkime (see https://github.com/arkime/arkime/pull/1120) # The userAutoCreateTmpl should more or less match what's in /etc/user_settings.json # which is what's used when creating the default admin user. userNameHeader=http_auth_http_user userAutoCreateTmpl={"userId": "${this.http_auth_http_user}", "userName": "${this.http_auth_http_user}", "enabled": true, "createEnabled": false, "webEnabled": true, "headerAuthEnabled": true, "emailSearch": true, "removeEnabled": false, "packetSearch": true, "hideStats": false, "hideFiles": false, "hidePcap": false, "disablePcapDownload": false, "settings": { "timezone": "local", "detailFormat": "last", "showTimestamps": "last", "sortColumn": "firstPacket", "sortDirection": "desc", "spiGraph": "protocol", "connSrcField": "source.ip", "connDstField": "destination.ip", "numPackets": "last", "theme" : "custom1: #222222,#E2E2E2,#FFFFFF,#00789E,#004A79,#017D73,#092B40,#42b7c5,#2A7580,#ecb30a,#333333,#89ADCC,#6D6D6D,#FFE7E7,#ECFEFF", "manualQuery": false }, "tableStates": { "sessionsNew": { "order": [ [ "firstPacket", "desc" ] ], "visibleHeaders": [ "protocol", "event.provider", "event.dataset", "firstPacket", "lastPacket", "src", "source.port", "dst", "destination.port", "network.packets", "dbby", "tags", "info" ] } } } -parseSMTP=true -parseSMB=true -parseQSValue=false -supportSha256=false -maxReqBody=64 -reqBodyOnlyUtf8=true -smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: -parsersDir=/opt/arkime/parsers -pluginsDir=/opt/arkime/plugins -plugins=wise.so -viewerPlugins=wise.js -spiDataMaxIndices=7 -uploadCommand= -packetThreads=2 -pcapWriteMethod=simple -pcapWriteSize=262143 -simpleCompression=zstd -simpleZstdLevel=3 -compressES=false -maxESConns=30 -maxESRequests=500 -packetsPerPoll=50000 -antiSynDrop=true -logEveryXPackets=100000 -logUnknownProtocols=false -logESRequests=true -logFileCreation=true # temporarily disabling viewer autocomplete to see if it helps slugishness valueAutoComplete=false +### High Performance settings +# https://github.com/arkime/arkime/wiki/Settings#High_Performance_Settings +magicMode=basic +pcapReadMethod=tpacketv3 +tpacketv3NumThreads=2 +tpacketv3BlockSize=8388608 +pcapWriteMethod=simple +pcapWriteSize=2560000 +simpleCompression=none +simpleZstdLevel=3 +simpleGzipLevel=3 +packetThreads=2 +maxPacketsInQueue=300000 +dbBulkSize=4000000 + [custom-fields] # see https://docs.zeek.org/en/stable/script-reference/log-files.html for Zeek logfile documentation diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index a7b2fe542..bc344b0b0 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -11,6 +11,8 @@ function urlencodeall() { ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} +CAPTURE_INTERFACE=${PCAP_IFACE:-} +LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-false} MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} OPENSEARCH_URL_FINAL=${OPENSEARCH_URL:-"http://opensearch:9200"} @@ -47,9 +49,49 @@ fi if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_DIR}"/etc/config.ini + sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini + + # capture interface(s) + if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then + + # in config.ini multiple interfaces are separated by ; + ARKIME_CAPTURE_INTERFACE="$(echo "$CAPTURE_INTERFACE" | sed "s/,/;/g")" + + # place capture interfaces in the config file + sed -r -i "s|(interface)\s*=\s*.*|\1=$ARKIME_CAPTURE_INTERFACE|" "${ARKIME_DIR}"/etc/config.ini + sed -i "s/^\(readTruncatedPackets=\).*/\1"false"/" "${ARKIME_DIR}"/etc/config.ini + + # convert pcap rotation size units (MB to GB) and stick in config file + if [[ -n $PCAP_ROTATE_MEGABYTES ]]; then + PCAP_ROTATE_GIGABYTES=$(echo "($PCAP_ROTATE_MEGABYTES + 1024 - 1)/1024" | bc) + sed -r -i "s/(maxFileSizeG)\s*=\s*.*/\1=$PCAP_ROTATE_GIGABYTES/" "${ARKIME_DIR}"/etc/config.ini + fi + + # convert pcap rotation time units (sec to min) and stick in config file + if [[ -n $PCAP_ROTATE_SECONDS ]]; then + PCAP_ROTATE_MINUTES=$(echo "($PCAP_ROTATE_SECONDS + 60 - 1)/60" | bc) + sed -r -i "s/(maxFileTimeM)\s*=\s*.*/\1=$PCAP_ROTATE_MINUTES/" "${ARKIME_DIR}"/etc/config.ini + fi + + # pcap compression + COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" + COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" + sed -r -i "s/(simpleCompression)\s*=\s*.*/\1=$COMPRESSION_TYPE/" "$ARKIME_CONFIG_FILE" + if [[ "$COMPRESSION_TYPE" == "zstd" ]]; then + sed -r -i "s/(simpleZstdLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "$ARKIME_CONFIG_FILE" + elif [[ "$COMPRESSION_TYPE" == "gzip" ]]; then + sed -r -i "s/(simpleGzipLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "$ARKIME_CONFIG_FILE" + fi + + # ensure capabilities for capture + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capture || true + fi + + # comment-out features that are only unused in hedgehog run profile mode if [[ "$MALCOLM_PROFILE" == "hedgehog" ]]; then sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini @@ -60,6 +102,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then sed -i "s/^\(viewerPlugins=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini sed -i '/^\[custom-fields\]/,$d' "${ARKIME_DIR}"/etc/config.ini fi + chmod 600 "${ARKIME_DIR}"/etc/config.ini fi diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 44d0d10ed..013d521a5 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -6,6 +6,7 @@ chmod=0700 [supervisord] nodaemon=true +user=root logfile=/dev/null logfile_maxbytes=0 pidfile=/tmp/supervisord.pid @@ -28,6 +29,7 @@ directory=%(ENV_ARKIME_DIR)s stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:wise] command=/opt/wise_service.sh @@ -40,6 +42,7 @@ killasgroup=true directory=%(ENV_ARKIME_DIR)s/wiseService stdout_logfile=%(ENV_ARKIME_DIR)s/logs/wise.log redirect_stderr=true +user=%(ENV_PUSER)s [program:viewer] command=/opt/viewer_service.sh @@ -53,25 +56,43 @@ directory=%(ENV_ARKIME_DIR)s stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:pcap-arkime] command=python3 /opt/pcap_arkime_processor.py %(ENV_PCAP_PIPELINE_VERBOSITY)s --start-sleep 10 - --threads %(ENV_ARKIME_ANALYZE_PCAP_THREADS)s + --threads %(ENV_ARKIME_AUTO_ANALYZE_PCAP_THREADS)s --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" --host "%(ENV_PCAP_NODE_HOST)s" --arkime /opt/arkime/bin/capture + --autoarkime "%(ENV_ARKIME_AUTO_ANALYZE_PCAP_FILES)s" + --forcearkime "%(ENV_ARKIME_ROTATED_PCAP)s" --autotag "%(ENV_AUTO_TAG)s" --managed "%(ENV_MANAGE_PCAP_FILES)s" startsecs=15 startretries=1 -autostart=true -autorestart=true +autostart=%(ENV_ARKIME_PCAP_PROCESSOR)s +autorestart=%(ENV_ARKIME_PCAP_PROCESSOR)s stopasgroup=true killasgroup=true directory=%(ENV_ARKIME_DIR)s stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s + +[program:live-arkime] +command=/opt/zeek/bin/zeekdeploy.sh +autostart=%(ENV_ARKIME_LIVE_CAPTURE)s +autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s +startsecs=180 +stopwaitsecs=15 +startretries=3 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true +user=%(ENV_PUSER)s \ No newline at end of file diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example new file mode 100644 index 000000000..74cca0b3e --- /dev/null +++ b/config/arkime-live.env.example @@ -0,0 +1,7 @@ +# Whether or Arkime should monitor live traffic on a local +# interface (PCAP_IFACE in pcap-capture.env specifies interface) +ARKIME_LIVE_CAPTURE=false +ARKIME_COMPRESSION_TYPE=none +ARKIME_COMPRESSION_LEVEL=0 + +ARKIME_PCAP_PROCESSOR=false \ No newline at end of file diff --git a/config/arkime-offline.env.example b/config/arkime-offline.env.example new file mode 100644 index 000000000..73e441806 --- /dev/null +++ b/config/arkime-offline.env.example @@ -0,0 +1,12 @@ +# Whether or not Arkime should analyze uploaded PCAP files +ARKIME_AUTO_ANALYZE_PCAP_FILES=true +# The number of Arkime processes for analyzing uploaded PCAP files allowed +# to run concurrently +ARKIME_AUTO_ANALYZE_PCAP_THREADS=1 +# Whether or not Arkime should analyze captured PCAP files captured +# by netsniff-ng/tcpdump (see PCAP_ENABLE_NETSNIFF and PCAP_ENABLE_TCPDUMP +# below). If ARKIME_LIVE_CAPTURE is true, this should be false: otherwise +# Arkime will see duplicate traffic. +ARKIME_ROTATED_PCAP=true + +ARKIME_PCAP_PROCESSOR=true \ No newline at end of file diff --git a/config/arkime.env.example b/config/arkime.env.example index 8248a636d..2d935df51 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -2,7 +2,5 @@ # https://arkime.com/faq#pcap-deletion) MANAGE_PCAP_FILES=false ARKIME_FREESPACEG=10% -# The number of Arkime capture processes allowed to run concurrently -ARKIME_ANALYZE_PCAP_THREADS=1 OPENSEARCH_MAX_SHARDS_PER_NODE=2500 \ No newline at end of file diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 7ec63d604..149c3f2e2 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -9,7 +9,7 @@ Run `./scripts/configure` and answer the questions to configure Malcolm. For an Although the configuration script automates many of the following configuration and tuning parameters, some environment variables of particular interest are listed here for reference. * **`arkime.env`** and **`arkime-secret.env`** - settings for [Arkime](https://arkime.com/) - - `ARKIME_ANALYZE_PCAP_THREADS` – the number of threads available to Arkime for analyzing PCAP files (default `1`) + - `ARKIME_AUTO_ANALYZE_PCAP_THREADS` – the number of threads available to Arkime for analyzing PCAP files (default `1`) - `ARKIME_PASSWORD_SECRET` - the password hash secret for the Arkime viewer cluster (see `passwordSecret` in [Arkime INI Settings](https://arkime.com/settings)) used to secure the connection used when Arkime viewer retrieves a PCAP payload for display in its user interface - `MANAGE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will be marked as available for deletion by Arkime if available storage space becomes too low (default `false`) - `MAXMIND_GEOIP_DB_LICENSE_KEY` - Malcolm uses MaxMind's free GeoLite2 databases for GeoIP lookups. As of December 30, 2019, these databases are [no longer available](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/) for download via a public URL. Instead, they must be downloaded using a MaxMind license key (available without charge [from MaxMind](https://www.maxmind.com/en/geolite2/signup)). The license key can be specified here for GeoIP database downloads during build- and run-time. diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/sensor-iso/interface/sensor_ctl/arkime/config.ini index 9b67ff43e..7d3f6a5b8 100644 --- a/sensor-iso/interface/sensor_ctl/arkime/config.ini +++ b/sensor-iso/interface/sensor_ctl/arkime/config.ini @@ -2,49 +2,49 @@ # so you can (for the most part) ignore settings here that seem like dummy settings [default] +antiSynDrop=false +compressES=false +dropGroup=netdev +dropUser=sensor elasticsearch=http://192.168.0.1:9200 -rotateIndex=daily -passwordSecret=Malcolm +freeSpaceG=5% +geoLite2ASN=/dummy/GeoLite2-ASN.mmdb +geoLite2Country=/dummy/GeoLite2-Country.mmdb httpRealm=Arkime +icmpTimeout=10 interface=enp0s1 -pcapDir=/tmp +logESRequests=false +logEveryXPackets=500000 +logFileCreation=true +logHTTPConnections=false +logUnknownProtocols=false +maxESConns=30 +maxESRequests=500 maxFileSizeG=4 maxFileTimeM=180 -tcpTimeout=600 -tcpSaveTimeout=720 -udpTimeout=30 -icmpTimeout=10 -maxStreams=1000000 maxPackets=10000 -freeSpaceG=5% -viewPort=8005 -geoLite2Country=/dummy/GeoLite2-Country.mmdb -geoLite2ASN=/dummy/GeoLite2-ASN.mmdb -rirFile=/dummy/ipv4-address-space.csv +maxReqBody=64 +maxStreams=1000000 ouiFile=/dummy/oui.txt -dropUser=sensor -dropGroup=netdev -parseSMTP=true -parseSMB=true +packetsPerPoll=50000 parseQSValue=false -supportSha256=false -maxReqBody=64 -reqBodyOnlyUtf8=true -smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: parsersDir=/dummy/parsers +parseSMB=true +parseSMTP=true +passwordSecret=Malcolm +pcapDir=/tmp pluginsDir=/dummy/plugins -spiDataMaxIndices=2 +reqBodyOnlyUtf8=true +rirFile=/dummy/ipv4-address-space.csv +rotateIndex=daily +smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: +spiDataMaxIndices=7 +supportSha256=false +tcpSaveTimeout=720 +tcpTimeout=600 +udpTimeout=30 uploadCommand= -compressES=false -maxESConns=30 -maxESRequests=500 -packetsPerPoll=50000 -antiSynDrop=false -logEveryXPackets=500000 -logUnknownProtocols=false -logESRequests=false -logFileCreation=true -logHTTPConnections=false +viewPort=8005 ### High Performance settings # https://github.com/arkime/arkime/wiki/Settings#High_Performance_Settings @@ -60,5 +60,4 @@ simpleGzipLevel=3 packetThreads=5 maxPacketsInQueue=300000 dbBulkSize=4000000 -#compressES=true rulesFiles=/dummy/rules.yml diff --git a/shared/bin/pcap_processor.py b/shared/bin/pcap_processor.py index 09ff58a00..8c77cce95 100755 --- a/shared/bin/pcap_processor.py +++ b/shared/bin/pcap_processor.py @@ -48,6 +48,7 @@ PCAP_PROCESSING_MODE_SURICATA = "suricata" ARKIME_CAPTURE_PATH = "/opt/arkime/bin/capture" +ARKIME_AUTOARKIME_TAG = 'AUTOARKIME' SURICATA_PATH = "/usr/bin/suricata" SURICATA_LOG_DIR = os.getenv('SURICATA_LOG_DIR', '/var/log/suricata') @@ -71,6 +72,7 @@ TAGS_NOSHOW = ( USERTAG_TAG, + ARKIME_AUTOARKIME_TAG, SURICATA_AUTOSURICATA_TAG, ZEEK_AUTOZEEK_TAG, ) @@ -112,7 +114,7 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): scanWorkerId = scanWorkersCount.increment() # unique ID for this thread - newFileQueue, pcapBaseDir, arkimeBin, nodeName, nodeHost, autoTag, notLocked, logger = ( + newFileQueue, pcapBaseDir, arkimeBin, nodeName, nodeHost, autoArkime, forceArkime, autoTag, notLocked, logger = ( arkimeWorkerArgs[0], arkimeWorkerArgs[1], arkimeWorkerArgs[2], @@ -121,6 +123,8 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): arkimeWorkerArgs[5], arkimeWorkerArgs[6], arkimeWorkerArgs[7], + arkimeWorkerArgs[8], + arkimeWorkerArgs[9], ) if not logger: @@ -141,49 +145,56 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): fileInfo[FILE_INFO_DICT_NAME] = os.path.join(pcapBaseDir, fileInfo[FILE_INFO_DICT_NAME]) if os.path.isfile(fileInfo[FILE_INFO_DICT_NAME]): - # finalize tags list - fileInfo[FILE_INFO_DICT_TAGS] = ( - [ - x - for x in fileInfo[FILE_INFO_DICT_TAGS] - if (x not in TAGS_NOSHOW) and (not x.startswith(ZEEK_AUTOCARVE_TAG_PREFIX)) - ] - if ((FILE_INFO_DICT_TAGS in fileInfo) and autoTag) - else list() - ) - logger.info(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileInfo}") - - # put together arkime execution command - cmd = [ - arkimeBin, - '--quiet', - '--insecure', - '--node', - fileInfo[FILE_INFO_DICT_NODE] if (FILE_INFO_DICT_NODE in fileInfo) else nodeName, - '-o', - f'ecsEventProvider={arkimeProvider}', - '-o', - f'ecsEventDataset={arkimeDataset}', - '-r', - fileInfo[FILE_INFO_DICT_NAME], - ] - if nodeHost: - cmd.append('--host') - cmd.append(nodeHost) - if notLocked: - cmd.append('--nolockpcap') - cmd.extend(list(chain.from_iterable(zip(repeat('-t'), fileInfo[FILE_INFO_DICT_TAGS])))) - - # execute capture for pcap file - retcode, output = run_process(cmd, logger=logger) - if retcode == 0: - logger.info( - f"{scriptName}[{scanWorkerId}]:\t✅\t{os.path.basename(fileInfo[FILE_INFO_DICT_NAME])}" + # Arkime this PCAP if it's tagged "AUTOARKIME" or if the global autoArkime flag is turned on. + if forceArkime and ( + autoArkime + or ( + (FILE_INFO_DICT_TAGS in fileInfo) and ARKIME_AUTOARKIME_TAG in fileInfo[FILE_INFO_DICT_TAGS] ) - else: - logger.warning( - f"{scriptName}[{scanWorkerId}]:\t❗\t{arkimeBin} {os.path.basename(fileInfo[FILE_INFO_DICT_NAME])} returned {retcode} {output}" + ): + # finalize tags list + fileInfo[FILE_INFO_DICT_TAGS] = ( + [ + x + for x in fileInfo[FILE_INFO_DICT_TAGS] + if (x not in TAGS_NOSHOW) and (not x.startswith(ZEEK_AUTOCARVE_TAG_PREFIX)) + ] + if ((FILE_INFO_DICT_TAGS in fileInfo) and autoTag) + else list() ) + logger.info(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileInfo}") + + # put together arkime execution command + cmd = [ + arkimeBin, + '--quiet', + '--insecure', + '--node', + fileInfo[FILE_INFO_DICT_NODE] if (FILE_INFO_DICT_NODE in fileInfo) else nodeName, + '-o', + f'ecsEventProvider={arkimeProvider}', + '-o', + f'ecsEventDataset={arkimeDataset}', + '-r', + fileInfo[FILE_INFO_DICT_NAME], + ] + if nodeHost: + cmd.append('--host') + cmd.append(nodeHost) + if notLocked: + cmd.append('--nolockpcap') + cmd.extend(list(chain.from_iterable(zip(repeat('-t'), fileInfo[FILE_INFO_DICT_TAGS])))) + + # execute capture for pcap file + retcode, output = run_process(cmd, logger=logger) + if retcode == 0: + logger.info( + f"{scriptName}[{scanWorkerId}]:\t✅\t{os.path.basename(fileInfo[FILE_INFO_DICT_NAME])}" + ) + else: + logger.warning( + f"{scriptName}[{scanWorkerId}]:\t❗\t{arkimeBin} {os.path.basename(fileInfo[FILE_INFO_DICT_NAME])} returned {retcode} {output}" + ) logger.info(f"{scriptName}[{scanWorkerId}]:\tfinished") @@ -543,6 +554,28 @@ def main(): required=True, ) if processingMode == PCAP_PROCESSING_MODE_ARKIME: + parser.add_argument( + '--autoarkime', + dest='autoArkime', + help="Autoanalyze all PCAP file with Arkime", + metavar='true|false', + type=str2bool, + nargs='?', + const=True, + default=False, + required=False, + ) + parser.add_argument( + '--forcearkime', + dest='forceArkime', + help="Force Arkime analysis even on rotated PCAPs", + metavar='true|false', + type=str2bool, + nargs='?', + const=True, + default=False, + required=False, + ) parser.add_argument( '--arkime', required=False, @@ -713,6 +746,8 @@ def main(): args.executable, args.nodeName, args.nodeHost, + args.autoArkime, + args.forceArkime, args.autoTag, args.notLocked, logging, From 05516fdb0bace4600255a5d79bf6d7ee87e5ad18 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 12:15:49 -0700 Subject: [PATCH 02/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- Dockerfiles/arkime.Dockerfile | 2 ++ arkime/supervisord.conf | 22 +++++++++++++++++----- config/arkime-live.env.example | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index d50dfa200..ae46a5bae 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -114,6 +114,7 @@ ARG ARKIME_ECS_DATASET=session ARG ARKIME_INTERFACE=eth0 ARG ARKIME_AUTO_ANALYZE_PCAP_FILES=false ARG ARKIME_AUTO_ANALYZE_PCAP_THREADS=1 +ARG ARKIME_PACKET_THREADS=1 ARG OPENSEARCH_MAX_SHARDS_PER_NODE=2500 ARG WISE=on ARG VIEWER=on @@ -145,6 +146,7 @@ ENV ARKIME_ECS_DATASET $ARKIME_ECS_DATASET ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_AUTO_ANALYZE_PCAP_FILES $ARKIME_AUTO_ANALYZE_PCAP_FILES ENV ARKIME_AUTO_ANALYZE_PCAP_THREADS $ARKIME_AUTO_ANALYZE_PCAP_THREADS +ENV ARKIME_PACKET_THREADS $ARKIME_PACKET_THREADS ENV ARKIME_PCAP_PROCESSOR $ARKIME_PCAP_PROCESSOR ENV ARKIME_LIVE_CAPTURE $ARKIME_LIVE_CAPTURE ENV ARKIME_COMPRESSION_TYPE $ARKIME_COMPRESSION_TYPE diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 013d521a5..1b765a275 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -66,7 +66,7 @@ command=python3 /opt/pcap_arkime_processor.py %(ENV_PCAP_PIPELINE_VERBOSITY)s --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" --host "%(ENV_PCAP_NODE_HOST)s" - --arkime /opt/arkime/bin/capture + --arkime "%(ENV_ARKIME_DIR)s"/bin/capture --autoarkime "%(ENV_ARKIME_AUTO_ANALYZE_PCAP_FILES)s" --forcearkime "%(ENV_ARKIME_ROTATED_PCAP)s" --autotag "%(ENV_AUTO_TAG)s" @@ -84,15 +84,27 @@ redirect_stderr=true user=%(ENV_PUSER)s [program:live-arkime] -command=/opt/zeek/bin/zeekdeploy.sh +command="%(ENV_ARKIME_DIR)s"/bin/capture --insecure + -c "%(ENV_ARKIME_DIR)s"/etc/config.ini + -o pcapDir=/data/pcap/processed + -o bpf="%(ENV_CAPTURE_FILTER)s" + -o packetThreads=%(ENV_ARKIME_PACKET_THREADS)s + -o dropUser=%(ENV_PUSER)s + -o dropGroup=%(ENV_PGROUP)s + -o ecsEventProvider=arkime + -o ecsEventDataset=session + --node "%(ENV_PCAP_NODE_NAME)s" + --host "%(ENV_PCAP_NODE_HOST)s" autostart=%(ENV_ARKIME_LIVE_CAPTURE)s autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s -startsecs=180 +startsecs=30 +startretries=2000000000 stopwaitsecs=15 -startretries=3 stopasgroup=true killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true -user=%(ENV_PUSER)s \ No newline at end of file +user=%(ENV_PUSER)s +directory=/data/pcap/processed + diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example index 74cca0b3e..09b99f041 100644 --- a/config/arkime-live.env.example +++ b/config/arkime-live.env.example @@ -3,5 +3,6 @@ ARKIME_LIVE_CAPTURE=false ARKIME_COMPRESSION_TYPE=none ARKIME_COMPRESSION_LEVEL=0 +ARKIME_PACKET_THREADS=1 ARKIME_PCAP_PROCESSOR=false \ No newline at end of file From a74ad8b7634fe36cb6afe4b6e3eeaec15b8ee31d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 13:01:31 -0700 Subject: [PATCH 03/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- Dockerfiles/arkime.Dockerfile | 8 +- .../Dockerfiles/arkime.Dockerfile | 2 +- arkime/etc/config.ini | 1 + arkime/scripts/docker_entrypoint.sh | 11 +- arkime/scripts/initarkime.sh | 102 +++++++++--------- arkime/supervisord.conf | 7 +- config/arkime-live.env.example | 8 ++ config/arkime-offline.env.example | 1 + .../interface/sensor_ctl/arkime/config.ini | 1 + .../supervisor.init/arkime_config_populate.sh | 5 +- 10 files changed, 88 insertions(+), 58 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index ae46a5bae..67f86b7d6 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -118,10 +118,13 @@ ARG ARKIME_PACKET_THREADS=1 ARG OPENSEARCH_MAX_SHARDS_PER_NODE=2500 ARG WISE=on ARG VIEWER=on +ARG ARKIME_VIEWER_PORT=8005 #Whether or not Arkime is in charge of deleting old PCAP files to reclaim space ARG MANAGE_PCAP_FILES=false ARG ARKIME_PCAP_PROCESSOR=true ARG ARKIME_LIVE_CAPTURE=false +ARG ARKIME_LIVE_NODE_NAME=malcolm +ARG ARKIME_LIVE_NODE_HOST= ARG ARKIME_ROTATED_PCAP=true ARG ARKIME_COMPRESSION_TYPE=none ARG ARKIME_COMPRESSION_LEVEL=0 @@ -149,12 +152,15 @@ ENV ARKIME_AUTO_ANALYZE_PCAP_THREADS $ARKIME_AUTO_ANALYZE_PCAP_THREADS ENV ARKIME_PACKET_THREADS $ARKIME_PACKET_THREADS ENV ARKIME_PCAP_PROCESSOR $ARKIME_PCAP_PROCESSOR ENV ARKIME_LIVE_CAPTURE $ARKIME_LIVE_CAPTURE +ENV ARKIME_LIVE_NODE_NAME $ARKIME_LIVE_NODE_NAME +ENV ARKIME_LIVE_NODE_HOST $ARKIME_LIVE_NODE_HOST ENV ARKIME_COMPRESSION_TYPE $ARKIME_COMPRESSION_TYPE ENV ARKIME_COMPRESSION_LEVEL $ARKIME_COMPRESSION_LEVEL ENV ARKIME_ROTATED_PCAP $ARKIME_ROTATED_PCAP ENV OPENSEARCH_MAX_SHARDS_PER_NODE $OPENSEARCH_MAX_SHARDS_PER_NODE ENV WISE $WISE ENV VIEWER $VIEWER +ENV ARKIME_VIEWER_PORT $ARKIME_VIEWER_PORT ENV MANAGE_PCAP_FILES $MANAGE_PCAP_FILES ENV AUTO_TAG $AUTO_TAG ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY @@ -258,7 +264,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ #Update Path ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}" -EXPOSE 8000 8005 8081 +EXPOSE 8000 8005 8006 8081 WORKDIR $ARKIME_DIR ENTRYPOINT ["/usr/bin/tini", \ diff --git a/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile b/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile index 890787325..c0a51e564 100644 --- a/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile +++ b/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile @@ -66,7 +66,7 @@ RUN git clone --recursive --depth=1 --single-branch -b "$GITHUB_BRANCH" "$GITHUB ENV PATH="/opt:$ARKIME_DIR/bin:$ARKIME_DIR/node-v10.21.0-linux-x64/bin:${PATH}" -EXPOSE 8000 8005 8081 +EXPOSE 8000 8005 8006 8081 WORKDIR $ARKIME_DIR/tests diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 83fa01f82..f9e2e740b 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -17,6 +17,7 @@ geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb httpRealm=Arkime icmpTimeout=10 interface=eth0 +bpf= keyFile=/opt/arkime/etc/viewer.key logESRequests=false logEveryXPackets=500000 diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index bc344b0b0..67495c7c1 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -13,6 +13,7 @@ ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} CAPTURE_INTERFACE=${PCAP_IFACE:-} LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-false} +VIEWER_PORT=${ARKIME_VIEWER_PORT:-8005} MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} OPENSEARCH_URL_FINAL=${OPENSEARCH_URL:-"http://opensearch:9200"} @@ -53,6 +54,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini + sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_DIR}"/etc/config.ini # capture interface(s) if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then @@ -63,6 +65,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then # place capture interfaces in the config file sed -r -i "s|(interface)\s*=\s*.*|\1=$ARKIME_CAPTURE_INTERFACE|" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(readTruncatedPackets=\).*/\1"false"/" "${ARKIME_DIR}"/etc/config.ini + sed -r -i "s/(bpf)\s*=\s*.*/\1=${PCAP_FILTER:-}/" "${ARKIME_DIR}"/etc/config.ini # convert pcap rotation size units (MB to GB) and stick in config file if [[ -n $PCAP_ROTATE_MEGABYTES ]]; then @@ -91,8 +94,8 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capture || true fi - # comment-out features that are only unused in hedgehog run profile mode - if [[ "$MALCOLM_PROFILE" == "hedgehog" ]]; then + # comment-out features that are unused in hedgehog run profile mode and in live-capture mode + if [[ "$MALCOLM_PROFILE" == "hedgehog" ]] || [[ "$LIVE_CAPTURE" == "true" ]]; then sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini sed -i "s/^\(userAutoCreateTmpl=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini @@ -103,7 +106,9 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then sed -i '/^\[custom-fields\]/,$d' "${ARKIME_DIR}"/etc/config.ini fi - chmod 600 "${ARKIME_DIR}"/etc/config.ini + chmod 600 "${ARKIME_DIR}"/etc/config.ini || true + [[ -n ${PUID} ]] && chown -f ${PUID} "${ARKIME_DIR}"/etc/config.ini || true + [[ -n ${PGID} ]] && chown -f :${PGID} "${ARKIME_DIR}"/etc/config.ini || true fi unset OPENSEARCH_URL_FINAL diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh index 06f59eb28..c6207e54f 100755 --- a/arkime/scripts/initarkime.sh +++ b/arkime/scripts/initarkime.sh @@ -3,6 +3,7 @@ # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} +ARKIME_LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-"false"} OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"} @@ -41,70 +42,75 @@ if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then # download and/or update geo updates $ARKIME_DIR/bin/arkime_update_geo.sh - # start and wait patiently for WISE - if [[ "$WISE" = "on" ]] ; then - touch /var/run/arkime/runwise - echo "Giving WISE time to start..." - sleep 5 - until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1" - do - echo "Waiting for WISE to start" - sleep 1 - done - echo "WISE is running!" - echo - fi + # don't do database initialization or run wise in arkime-live mode + if [[ "$ARKIME_LIVE_CAPTURE" == "false" ]]; then - # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created) - if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then + # start and wait patiently for WISE + if [[ "$WISE" = "on" ]] ; then + touch /var/run/arkime/runwise + echo "Giving WISE time to start..." + sleep 5 + until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1" + do + echo "Waiting for WISE to start" + sleep 1 + done + echo "WISE is running!" + echo + fi - echo "Initializing $OPENSEARCH_PRIMARY database..." + # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created) + if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then - $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt + echo "Initializing $OPENSEARCH_PRIMARY database..." - echo "Creating default user..." + $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt - # this password isn't going to be used by Arkime, nginx will do the auth instead - $ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG + echo "Creating default user..." - echo "Initializing fields..." + # this password isn't going to be used by Arkime, nginx will do the auth instead + $ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG - # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S. - touch /tmp/not_a_packet.pcap - $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 - rm -f /tmp/not_a_packet.pcap + echo "Initializing fields..." - echo "Initializing views..." + # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S. + touch /tmp/not_a_packet.pcap + $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 + rm -f /tmp/not_a_packet.pcap - for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do - TEMP_JSON=$(mktemp --suffix=.json) - RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')" - echo "Creating view $(jq '.name' < "${VIEW_FILE}")" - jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}" - curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}" - rm -f "${TEMP_JSON}" - done + echo "Initializing views..." - echo "Setting defaults..." + for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do + TEMP_JSON=$(mktemp --suffix=.json) + RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')" + echo "Creating view $(jq '.name' < "${VIEW_FILE}")" + jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}" + curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}" + rm -f "${TEMP_JSON}" + done - curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json" + echo "Setting defaults..." - echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n" + curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json" - else - echo "$OPENSEARCH_PRIMARY database previously initialized!" - echo + echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n" - $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded - echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!" + else + echo "$OPENSEARCH_PRIMARY database previously initialized!" + echo - fi # if/else OpenSearch database initialized + $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded + echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!" - # increase OpenSearch max shards per node from default if desired - if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then - # see https://github.com/elastic/elasticsearch/issues/40803 - curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }" - fi + fi # if/else OpenSearch database initialized + + # increase OpenSearch max shards per node from default if desired + if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then + # see https://github.com/elastic/elasticsearch/issues/40803 + curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }" + fi + + fi # "$ARKIME_LIVE_CAPTURE" == "false" # before running viewer, call _refresh to make sure everything is available for search first curl "${CURL_CONFIG_PARAMS[@]}" -sS -XPOST "${OPENSEARCH_URL}/_refresh" diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 1b765a275..c67b8cb9b 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -87,14 +87,13 @@ user=%(ENV_PUSER)s command="%(ENV_ARKIME_DIR)s"/bin/capture --insecure -c "%(ENV_ARKIME_DIR)s"/etc/config.ini -o pcapDir=/data/pcap/processed - -o bpf="%(ENV_CAPTURE_FILTER)s" -o packetThreads=%(ENV_ARKIME_PACKET_THREADS)s -o dropUser=%(ENV_PUSER)s -o dropGroup=%(ENV_PGROUP)s -o ecsEventProvider=arkime -o ecsEventDataset=session - --node "%(ENV_PCAP_NODE_NAME)s" - --host "%(ENV_PCAP_NODE_HOST)s" + --node "%(ENV_ARKIME_LIVE_NODE_NAME)s" + --host "%(ENV_ARKIME_LIVE_NODE_HOST)s" autostart=%(ENV_ARKIME_LIVE_CAPTURE)s autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s startsecs=30 @@ -105,6 +104,6 @@ killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true -user=%(ENV_PUSER)s +user=root directory=/data/pcap/processed diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example index 09b99f041..c643faec1 100644 --- a/config/arkime-live.env.example +++ b/config/arkime-live.env.example @@ -1,3 +1,4 @@ +ARKIME_VIEWER_PORT=8006 # Whether or Arkime should monitor live traffic on a local # interface (PCAP_IFACE in pcap-capture.env specifies interface) ARKIME_LIVE_CAPTURE=false @@ -5,4 +6,11 @@ ARKIME_COMPRESSION_TYPE=none ARKIME_COMPRESSION_LEVEL=0 ARKIME_PACKET_THREADS=1 +# The node name (e.g., the hostname of this machine running Malcolm) to associate with +# network traffic metadata +ARKIME_LIVE_NODE_NAME=malcolm +# The node host (e.g., the IP address of the machine running Malcolm) to associate with +# network traffic metadata (optional, defaults to PCAP_NODE_NAME if unspecified) +ARKIME_LIVE_NODE_HOST= + ARKIME_PCAP_PROCESSOR=false \ No newline at end of file diff --git a/config/arkime-offline.env.example b/config/arkime-offline.env.example index 73e441806..f0548cc14 100644 --- a/config/arkime-offline.env.example +++ b/config/arkime-offline.env.example @@ -1,3 +1,4 @@ +ARKIME_VIEWER_PORT=8005 # Whether or not Arkime should analyze uploaded PCAP files ARKIME_AUTO_ANALYZE_PCAP_FILES=true # The number of Arkime processes for analyzing uploaded PCAP files allowed diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/sensor-iso/interface/sensor_ctl/arkime/config.ini index 7d3f6a5b8..fd30ae188 100644 --- a/sensor-iso/interface/sensor_ctl/arkime/config.ini +++ b/sensor-iso/interface/sensor_ctl/arkime/config.ini @@ -12,6 +12,7 @@ geoLite2ASN=/dummy/GeoLite2-ASN.mmdb geoLite2Country=/dummy/GeoLite2-Country.mmdb httpRealm=Arkime icmpTimeout=10 +bpf= interface=enp0s1 logESRequests=false logEveryXPackets=500000 diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index 048e2944e..4f3d0b0b3 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -38,6 +38,10 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t sed -r -i "s|(elasticsearch)\s*=\s*.*|\1=$ARKIME_ELASTICSEARCH|" "$ARKIME_CONFIG_FILE" fi + if [[ -n $ARKIME_VIEWER_PORT ]]; then + sed -r -i "s/(viewPort)\s*=\s*.*/\1=$ARKIME_VIEWER_PORT/" "$ARKIME_CONFIG_FILE" + f + if [[ -n $ARKIME_PASSWORD_SECRET ]]; then # place the Arkime viewer cluster password hash in the config file sed -r -i "s|(passwordSecret)\s*=\s*.*|\1=$ARKIME_PASSWORD_SECRET|" "$ARKIME_CONFIG_FILE" @@ -64,7 +68,6 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t if [[ -n $ARKIME_FREESPACEG ]]; then sed -r -i "s/(freeSpaceG)\s*=\s*.*/\1=$ARKIME_FREESPACEG/" "$ARKIME_CONFIG_FILE" fi - # pcap compression COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" From 3fdfd98295c2066dd23db3b17b37c7d343a99eb4 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 13:36:28 -0700 Subject: [PATCH 04/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- arkime/scripts/initarkime.sh | 106 ++++++++++++++++----------------- arkime/scripts/live_capture.sh | 71 ++++++++++++++++++++++ arkime/supervisord.conf | 13 +--- 3 files changed, 123 insertions(+), 67 deletions(-) create mode 100755 arkime/scripts/live_capture.sh diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh index c6207e54f..e9673e24e 100755 --- a/arkime/scripts/initarkime.sh +++ b/arkime/scripts/initarkime.sh @@ -3,7 +3,6 @@ # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} -ARKIME_LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-"false"} OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"} @@ -36,81 +35,76 @@ fi if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then - echo "Giving $OPENSEARCH_PRIMARY time to start..." - /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" - # download and/or update geo updates $ARKIME_DIR/bin/arkime_update_geo.sh - # don't do database initialization or run wise in arkime-live mode - if [[ "$ARKIME_LIVE_CAPTURE" == "false" ]]; then - - # start and wait patiently for WISE - if [[ "$WISE" = "on" ]] ; then - touch /var/run/arkime/runwise - echo "Giving WISE time to start..." - sleep 5 - until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1" - do - echo "Waiting for WISE to start" - sleep 1 - done - echo "WISE is running!" - echo - fi + echo "Giving $OPENSEARCH_PRIMARY time to start..." + /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" - # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created) - if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then + # start and wait patiently for WISE + if [[ "$WISE" = "on" ]] ; then + touch /var/run/arkime/runwise + echo "Giving WISE time to start..." + sleep 5 + until curl -fsS --output /dev/null "http://127.0.0.1:8081/fields?ver=1" + do + echo "Waiting for WISE to start" + sleep 1 + done + echo "WISE is running!" + echo + fi - echo "Initializing $OPENSEARCH_PRIMARY database..." + # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created) + if (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 )); then - $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt + echo "Initializing $OPENSEARCH_PRIMARY database..." - echo "Creating default user..." + $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" initnoprompt - # this password isn't going to be used by Arkime, nginx will do the auth instead - $ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG + echo "Creating default user..." - echo "Initializing fields..." + # this password isn't going to be used by Arkime, nginx will do the auth instead + $ARKIME_DIR/bin/arkime_add_user.sh "${MALCOLM_USERNAME}" "${MALCOLM_USERNAME}" "ignored" --admin --webauthonly --webauth $DB_SSL_FLAG - # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S. - touch /tmp/not_a_packet.pcap - $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 - rm -f /tmp/not_a_packet.pcap + echo "Initializing fields..." - echo "Initializing views..." + # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S. + touch /tmp/not_a_packet.pcap + $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 + rm -f /tmp/not_a_packet.pcap - for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do - TEMP_JSON=$(mktemp --suffix=.json) - RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')" - echo "Creating view $(jq '.name' < "${VIEW_FILE}")" - jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}" - curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}" - rm -f "${TEMP_JSON}" - done + echo "Initializing views..." - echo "Setting defaults..." + for VIEW_FILE in "$ARKIME_DIR"/etc/views/*.json; do + TEMP_JSON=$(mktemp --suffix=.json) + RANDOM_ID="$(openssl rand -base64 14 | sed -E 's/[^[:alnum:][:space:]]+/_/g')" + echo "Creating view $(jq '.name' < "${VIEW_FILE}")" + jq ". += {\"user\": \"${MALCOLM_USERNAME}\"}" < "${VIEW_FILE}" >"${TEMP_JSON}" + curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_views/_doc/${RANDOM_ID}" -d "@${TEMP_JSON}" + rm -f "${TEMP_JSON}" + done - curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json" + echo "Setting defaults..." - echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n" + curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPOST "${OPENSEARCH_URL}/arkime_users/_update/$MALCOLM_USERNAME" -d "@$ARKIME_DIR/etc/user_settings.json" - else - echo "$OPENSEARCH_PRIMARY database previously initialized!" - echo + echo -e "\n$OPENSEARCH_PRIMARY database initialized!\n" - $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded - echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!" + else + echo "$OPENSEARCH_PRIMARY database previously initialized!" + echo - fi # if/else OpenSearch database initialized + $ARKIME_DIR/db/db.pl $DB_SSL_FLAG "${OPENSEARCH_URL_FULL}" upgradenoprompt --ifneeded + echo "$OPENSEARCH_PRIMARY database is up-to-date for Arkime version $ARKIME_VERSION!" - # increase OpenSearch max shards per node from default if desired - if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then - # see https://github.com/elastic/elasticsearch/issues/40803 - curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }" - fi + fi # if/else OpenSearch database initialized - fi # "$ARKIME_LIVE_CAPTURE" == "false" + # increase OpenSearch max shards per node from default if desired + if [[ -n $OPENSEARCH_MAX_SHARDS_PER_NODE ]]; then + # see https://github.com/elastic/elasticsearch/issues/40803 + curl "${CURL_CONFIG_PARAMS[@]}" -sS --output /dev/null -H'Content-Type: application/json' -XPUT "${OPENSEARCH_URL}/_cluster/settings" -d "{ \"persistent\": { \"cluster.max_shards_per_node\": \"$OPENSEARCH_MAX_SHARDS_PER_NODE\" } }" + fi # before running viewer, call _refresh to make sure everything is available for search first curl "${CURL_CONFIG_PARAMS[@]}" -sS -XPOST "${OPENSEARCH_URL}/_refresh" diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh new file mode 100755 index 000000000..bec4ede55 --- /dev/null +++ b/arkime/scripts/live_capture.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. + +ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} +CERT_FILE="${ARKIME_DIR}"/etc/viewer.crt +KEY_FILE="${ARKIME_DIR}"/etc/viewer.key +ARKIME_PACKET_THREADS=${ARKIME_PACKET_THREADS:-1} +PUSER=${PUSER:-"arkime"} +PGROUP=${PGROUP:-"arkime"} +ARKIME_LIVE_NODE_NAME=${ARKIME_LIVE_NODE_NAME:-"malcolm"} +ARKIME_LIVE_NODE_HOST=${ARKIME_LIVE_NODE_HOST:-""} + +OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} +OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} +OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} +OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"} +OPENSEARCH_CREDS_CONFIG_FILE=${OPENSEARCH_CREDS_CONFIG_FILE:-"/var/local/curlrc/.opensearch.primary.curlrc"} +if ( [[ "$OPENSEARCH_PRIMARY" == "opensearch-remote" ]] || [[ "$OPENSEARCH_PRIMARY" == "elasticsearch-remote" ]] ) && [[ -r "$OPENSEARCH_CREDS_CONFIG_FILE" ]]; then + CURL_CONFIG_PARAMS=( + --config + "$OPENSEARCH_CREDS_CONFIG_FILE" + ) +else + CURL_CONFIG_PARAMS=() +fi + +rm -f /var/run/arkime/initialized /var/run/arkime/runwise + +# make sure TLS certificates exist prior to starting up +CERT_FILE=$ARKIME_DIR/etc/viewer.crt +KEY_FILE=$ARKIME_DIR/etc/viewer.key +if ( [[ ! -f "$CERT_FILE" ]] || [[ ! -f "$KEY_FILE" ]] ) && [[ -x /usr/local/bin/self_signed_key_gen.sh ]]; then + rm -f "$CERT_FILE" "$KEY_FILE" ./newcerts + pushd $ARKIME_DIR/etc/ >/dev/null 2>&1 + /usr/local/bin/self_signed_key_gen.sh -n -o ./newcerts >/dev/null 2>&1 + mv ./newcerts/server.crt "$CERT_FILE" + mv ./newcerts/server.key "$KEY_FILE" + rm -rf ./newcerts + popd >/dev/null 2>&1 +fi + +# download and/or update geo updates +$ARKIME_DIR/bin/arkime_update_geo.sh + +# wait patiently for the non-live Arkime to initialize the database + +echo "Giving $OPENSEARCH_PRIMARY time to start..." +/opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" +echo "Giving Arkime time to initialize..." +sleep 5 +until (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) >= 1 )); do + sleep 1 +done + +# this will also allow viewer to kick off +touch /var/run/arkime/initialized + +echo "Arkime is initialized!" +echo + +"${ARKIME_DIR}"/bin/capture --insecure \ + -c "${ARKIME_DIR}"/etc/config.ini \ + -o pcapDir=/data/pcap/processed \ + -o packetThreads=${ARKIME_PACKET_THREADS} \ + -o dropUser=${PUSER} \ + -o dropGroup=${PGROUP} \ + -o ecsEventProvider=arkime \ + -o ecsEventDataset=session \ + --node "${ARKIME_LIVE_NODE_NAME}" \ + --host "${ARKIME_LIVE_NODE_HOST}" | tee -a "${ARKIME_DIR}"/logs/capture.log 2>&1 diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index c67b8cb9b..0e04da6e6 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -21,7 +21,7 @@ serverurl=unix:///tmp/supervisor.sock command=/opt/initarkime.sh startsecs=0 startretries=0 -autostart=true +autostart=%(ENV_ARKIME_PCAP_PROCESSOR)s autorestart=false stopasgroup=true killasgroup=true @@ -84,16 +84,7 @@ redirect_stderr=true user=%(ENV_PUSER)s [program:live-arkime] -command="%(ENV_ARKIME_DIR)s"/bin/capture --insecure - -c "%(ENV_ARKIME_DIR)s"/etc/config.ini - -o pcapDir=/data/pcap/processed - -o packetThreads=%(ENV_ARKIME_PACKET_THREADS)s - -o dropUser=%(ENV_PUSER)s - -o dropGroup=%(ENV_PGROUP)s - -o ecsEventProvider=arkime - -o ecsEventDataset=session - --node "%(ENV_ARKIME_LIVE_NODE_NAME)s" - --host "%(ENV_ARKIME_LIVE_NODE_HOST)s" +command=/opt/live_capture.sh autostart=%(ENV_ARKIME_LIVE_CAPTURE)s autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s startsecs=30 From e4bd70fd7a5b44c19d62fb3c329bd01860c7ba3b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 14:16:05 -0700 Subject: [PATCH 05/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- arkime-logs/.gitignore | 3 --- arkime-raw/.gitignore | 3 --- arkime/scripts/docker_entrypoint.sh | 2 +- arkime/scripts/live_capture.sh | 6 ++++-- arkime/scripts/viewer_service.sh | 2 +- arkime/scripts/wise_service.sh | 2 +- arkime/supervisord.conf | 5 +++-- docker-compose-standalone.yml | 2 -- docker-compose.yml | 2 -- docs/contributing-local-modifications.md | 2 -- docs/development.md | 1 - docs/kubernetes.md | 2 -- kubernetes/07-arkime.yml | 10 +--------- malcolm-iso/build.sh | 2 -- scripts/control.py | 4 ---- scripts/malcolm_appliance_packager.sh | 2 -- 16 files changed, 11 insertions(+), 39 deletions(-) delete mode 100644 arkime-logs/.gitignore delete mode 100644 arkime-raw/.gitignore diff --git a/arkime-logs/.gitignore b/arkime-logs/.gitignore deleted file mode 100644 index a5baada18..000000000 --- a/arkime-logs/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore - diff --git a/arkime-raw/.gitignore b/arkime-raw/.gitignore deleted file mode 100644 index a5baada18..000000000 --- a/arkime-raw/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!.gitignore - diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 67495c7c1..aab652286 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -91,7 +91,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capture || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true fi # comment-out features that are unused in hedgehog run profile mode and in live-capture mode diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh index bec4ede55..55f3e4a2c 100755 --- a/arkime/scripts/live_capture.sh +++ b/arkime/scripts/live_capture.sh @@ -43,8 +43,10 @@ fi # download and/or update geo updates $ARKIME_DIR/bin/arkime_update_geo.sh -# wait patiently for the non-live Arkime to initialize the database +# we haven't dropUser/dropGroup'ed yet, so make sure the regular user owns the files we just touched +[[ -n ${PUID} ]] && [[ -n ${PGID} ]] && chown -f -R ${PUID}:${PGID} "${ARKIME_DIR}"/etc/ || true +# wait patiently for the non-live Arkime to initialize the database echo "Giving $OPENSEARCH_PRIMARY time to start..." /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" echo "Giving Arkime time to initialize..." @@ -68,4 +70,4 @@ echo -o ecsEventProvider=arkime \ -o ecsEventDataset=session \ --node "${ARKIME_LIVE_NODE_NAME}" \ - --host "${ARKIME_LIVE_NODE_HOST}" | tee -a "${ARKIME_DIR}"/logs/capture.log 2>&1 + --host "${ARKIME_LIVE_NODE_HOST}" diff --git a/arkime/scripts/viewer_service.sh b/arkime/scripts/viewer_service.sh index d1785b49b..9cde725c0 100755 --- a/arkime/scripts/viewer_service.sh +++ b/arkime/scripts/viewer_service.sh @@ -7,7 +7,7 @@ while true; do echo "Launch viewer..." rm -f $ARKIME_DIR/logs/viewer* pushd $ARKIME_DIR/viewer >/dev/null 2>&1 - $ARKIME_DIR/bin/node viewer.js --insecure -n "${PCAP_NODE_NAME:-malcolm}" -c $ARKIME_DIR/etc/config.ini | tee -a $ARKIME_DIR/logs/viewer.log 2>&1 + $ARKIME_DIR/bin/node viewer.js --insecure -n "${PCAP_NODE_NAME:-malcolm}" -c $ARKIME_DIR/etc/config.ini popd >/dev/null 2>&1 fi sleep 5 diff --git a/arkime/scripts/wise_service.sh b/arkime/scripts/wise_service.sh index ff9e26b34..3334db619 100755 --- a/arkime/scripts/wise_service.sh +++ b/arkime/scripts/wise_service.sh @@ -7,7 +7,7 @@ while true; do echo "Launch wise..." rm -f $ARKIME_DIR/logs/wise* pushd $ARKIME_DIR/wiseService >/dev/null 2>&1 - $ARKIME_DIR/bin/node wiseService.js --insecure -c $ARKIME_DIR/etc/wise.ini | tee -a $ARKIME_DIR/logs/wise.log 2>&1 + $ARKIME_DIR/bin/node wiseService.js --insecure -c $ARKIME_DIR/etc/wise.ini popd >/dev/null 2>&1 fi sleep 5 diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 0e04da6e6..008dad45d 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -40,7 +40,8 @@ autorestart=true stopasgroup=true killasgroup=true directory=%(ENV_ARKIME_DIR)s/wiseService -stdout_logfile=%(ENV_ARKIME_DIR)s/logs/wise.log +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 redirect_stderr=true user=%(ENV_PUSER)s @@ -95,6 +96,6 @@ killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true -user=root +user=root ; fear not, capture will dropUser/dropGroup to PUSER directory=/data/pcap/processed diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 684e4a808..a60bd754b 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -206,8 +206,6 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap - - ./arkime-logs:/opt/arkime/logs - - ./arkime-raw:/opt/arkime/raw healthcheck: test: ["CMD", "curl", "--insecure", "--silent", "--fail", "https://localhost:8005/_ns_/nstest.html"] interval: 90s diff --git a/docker-compose.yml b/docker-compose.yml index c38dc4b04..6a3bed0d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -228,8 +228,6 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap - - ./arkime-logs:/opt/arkime/logs - - ./arkime-raw:/opt/arkime/raw - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index f449a9c11..7a68dfe0d 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -42,8 +42,6 @@ arkime: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap - - ./arkime-logs:/opt/arkime/logs - - ./arkime-raw:/opt/arkime/raw zeek: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap:/pcap diff --git a/docs/development.md b/docs/development.md index 8207e3582..c4a5a05fa 100644 --- a/docs/development.md +++ b/docs/development.md @@ -8,7 +8,6 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ * `api` - code and configuration for the `api` container, which provides a REST API to query Malcolm * `arkime` - code and configuration for the `arkime` container that processes PCAP files using `capture`, which serves the Viewer application -* `arkime-logs` - an initially empty directory to which the `arkime` container will write some debug log files * `config` - a directory containing the environment variable files that define Malcolm's configuration * `dashboards` - code and configuration for the `dashboards` container for creating additional ad-hoc visualizations and dashboards beyond that which is provided by Arkime Viewer * `Dockerfiles` - a directory containing build instructions for Malcolm's docker images diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 02c62b0d3..33b9a2c5a 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -307,8 +307,6 @@ Here is a basic step-by-step example illustrating how to deploy Malcolm with Kub ``` $ ls -l total 45,056 -drwxr-xr-x 2 user user 6 Apr 24 14:35 arkime-logs -drwxr-xr-x 2 user user 6 Apr 24 14:35 arkime-raw drwxr-xr-x 2 user user 4,096 Apr 24 14:35 config drwxr-xr-x 3 user user 19 Apr 24 14:35 filebeat drwxr-xr-x 2 user user 6 Apr 24 14:35 htadmin diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index cea3ced27..05effbe64 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -78,9 +78,6 @@ spec: name: arkime-opensearch-curlrc-secret-volume - mountPath: "/data/pcap" name: arkime-pcap-volume - - name: arkime-runtime-logs-volume - mountPath: /opt/arkime/logs - subPath: "arkime" initContainers: - name: arkime-dirinit-container image: ghcr.io/idaholab/malcolm/dirinit:v23.11.0 @@ -92,12 +89,10 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/pcap:processed;/data/runtime-logs:arkime" + value: "/data/pcap:processed" volumeMounts: - name: arkime-pcap-volume mountPath: "/data/pcap" - - name: arkime-runtime-logs-volume - mountPath: "/data/runtime-logs" volumes: - name: arkime-var-local-catrust-volume configMap: @@ -108,6 +103,3 @@ spec: - name: arkime-pcap-volume persistentVolumeClaim: claimName: pcap-claim - - name: arkime-runtime-logs-volume - persistentVolumeClaim: - claimName: runtime-logs-claim \ No newline at end of file diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 44f795369..bb5b163b5 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -92,8 +92,6 @@ if [ -d "$WORKDIR" ]; then # grab things from the Malcolm parent directory into /etc/skel so the user's got it set up in their home/Malcolm dir pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 MALCOLM_DEST_DIR="$WORKDIR/work/$IMAGE_NAME-Live-Build/config/includes.chroot/etc/skel/Malcolm" - mkdir -p "$MALCOLM_DEST_DIR/arkime-logs/" - mkdir -p "$MALCOLM_DEST_DIR/arkime-raw/" mkdir -p "$MALCOLM_DEST_DIR/config/" mkdir -p "$MALCOLM_DEST_DIR/filebeat/certs/" mkdir -p "$MALCOLM_DEST_DIR/htadmin/" diff --git a/scripts/control.py b/scripts/control.py index 9b03d6d95..4303155ba 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -843,8 +843,6 @@ def stop(wipe=False): if wipe: # there is some overlap here among some of these containers, but it doesn't matter boundPathsToWipe = ( - BoundPath("arkime", "/opt/arkime/logs", True, None, None), - BoundPath("arkime", "/opt/arkime/raw", True, None, None), BoundPath("filebeat", "/zeek", True, None, None), BoundPath("file-monitor", "/zeek/logs", True, None, None), BoundPath("netbox", "/opt/netbox/netbox/media", True, None, ["."]), @@ -994,8 +992,6 @@ def start(): if orchMode is OrchestrationFramework.DOCKER_COMPOSE: # make sure some directories exist before we start boundPathsToCreate = ( - BoundPath("arkime", "/opt/arkime/logs", False, None, None), - BoundPath("arkime", "/opt/arkime/raw", False, None, None), BoundPath("file-monitor", "/zeek/logs", False, None, None), BoundPath("nginx-proxy", "/var/local/ca-trust", False, None, None), BoundPath("netbox", "/opt/netbox/netbox/media", False, None, None), diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index b4bb406a5..86c4ba85e 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -61,8 +61,6 @@ if mkdir "$DESTDIR"; then # ensure that if we "grabbed a lock", we release it (works for clean exit, SIGTERM, and SIGINT/Ctrl-C) trap "cleanup" EXIT - mkdir $VERBOSE -p "$DESTDIR/arkime-logs/" - mkdir $VERBOSE -p "$DESTDIR/arkime-raw/" mkdir $VERBOSE -p "$DESTDIR/filebeat/certs/" mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" From 7f061e6ee6bfc867a73a986ac93878d25fd4e1f4 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 14:30:54 -0700 Subject: [PATCH 06/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- arkime/scripts/docker_entrypoint.sh | 43 +++++++++++++++-------------- arkime/scripts/live_capture.sh | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index aab652286..8cd9887e2 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -9,6 +9,7 @@ function urlencodeall() { } ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} +ARKIME_CONFIG_FILE="${ARKIME_DIR}"/etc/config.ini ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} CAPTURE_INTERFACE=${PCAP_IFACE:-} @@ -49,12 +50,12 @@ if ( [[ "$OPENSEARCH_PRIMARY" == "opensearch-remote" ]] || [[ "$OPENSEARCH_PRIMA fi if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then - cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_DIR}"/etc/config.ini + cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_CONFIG_FILE}" - sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_DIR}"/etc/config.ini + sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_CONFIG_FILE}" # capture interface(s) if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then @@ -63,20 +64,20 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then ARKIME_CAPTURE_INTERFACE="$(echo "$CAPTURE_INTERFACE" | sed "s/,/;/g")" # place capture interfaces in the config file - sed -r -i "s|(interface)\s*=\s*.*|\1=$ARKIME_CAPTURE_INTERFACE|" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(readTruncatedPackets=\).*/\1"false"/" "${ARKIME_DIR}"/etc/config.ini - sed -r -i "s/(bpf)\s*=\s*.*/\1=${PCAP_FILTER:-}/" "${ARKIME_DIR}"/etc/config.ini + sed -r -i "s|(interface)\s*=\s*.*|\1=$ARKIME_CAPTURE_INTERFACE|" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(readTruncatedPackets=\).*/\1"false"/" "${ARKIME_CONFIG_FILE}" + sed -r -i "s/(bpf)\s*=\s*.*/\1=${PCAP_FILTER:-}/" "${ARKIME_CONFIG_FILE}" # convert pcap rotation size units (MB to GB) and stick in config file if [[ -n $PCAP_ROTATE_MEGABYTES ]]; then PCAP_ROTATE_GIGABYTES=$(echo "($PCAP_ROTATE_MEGABYTES + 1024 - 1)/1024" | bc) - sed -r -i "s/(maxFileSizeG)\s*=\s*.*/\1=$PCAP_ROTATE_GIGABYTES/" "${ARKIME_DIR}"/etc/config.ini + sed -r -i "s/(maxFileSizeG)\s*=\s*.*/\1=$PCAP_ROTATE_GIGABYTES/" "${ARKIME_CONFIG_FILE}" fi # convert pcap rotation time units (sec to min) and stick in config file if [[ -n $PCAP_ROTATE_SECONDS ]]; then PCAP_ROTATE_MINUTES=$(echo "($PCAP_ROTATE_SECONDS + 60 - 1)/60" | bc) - sed -r -i "s/(maxFileTimeM)\s*=\s*.*/\1=$PCAP_ROTATE_MINUTES/" "${ARKIME_DIR}"/etc/config.ini + sed -r -i "s/(maxFileTimeM)\s*=\s*.*/\1=$PCAP_ROTATE_MINUTES/" "${ARKIME_CONFIG_FILE}" fi # pcap compression @@ -96,19 +97,19 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then # comment-out features that are unused in hedgehog run profile mode and in live-capture mode if [[ "$MALCOLM_PROFILE" == "hedgehog" ]] || [[ "$LIVE_CAPTURE" == "true" ]]; then - sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(userAutoCreateTmpl=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(wiseHost=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(wisePort=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(plugins=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(viewerPlugins=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i '/^\[custom-fields\]/,$d' "${ARKIME_DIR}"/etc/config.ini + sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(userAutoCreateTmpl=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(wiseHost=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(wisePort=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(plugins=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(viewerPlugins=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i '/^\[custom-fields\]/,$d' "${ARKIME_CONFIG_FILE}" fi - chmod 600 "${ARKIME_DIR}"/etc/config.ini || true - [[ -n ${PUID} ]] && chown -f ${PUID} "${ARKIME_DIR}"/etc/config.ini || true - [[ -n ${PGID} ]] && chown -f :${PGID} "${ARKIME_DIR}"/etc/config.ini || true + chmod 600 "${ARKIME_CONFIG_FILE}" || true + [[ -n ${PUID} ]] && chown -f ${PUID} "${ARKIME_CONFIG_FILE}" || true + [[ -n ${PGID} ]] && chown -f :${PGID} "${ARKIME_CONFIG_FILE}" || true fi unset OPENSEARCH_URL_FINAL diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh index 55f3e4a2c..d7b9a897d 100755 --- a/arkime/scripts/live_capture.sh +++ b/arkime/scripts/live_capture.sh @@ -48,7 +48,7 @@ $ARKIME_DIR/bin/arkime_update_geo.sh # wait patiently for the non-live Arkime to initialize the database echo "Giving $OPENSEARCH_PRIMARY time to start..." -/opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" +/opt/opensearch_status.sh -t malcolm_template 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" echo "Giving Arkime time to initialize..." sleep 5 until (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) >= 1 )); do From 24a19ae823e15d730fdb9972aceff3b8a5c5225d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 28 Nov 2023 15:44:12 -0700 Subject: [PATCH 07/78] Work in progress of idaholab/Malcolm#281, arkime-live container; done in arkime-capture branch --- Dockerfiles/arkime.Dockerfile | 2 +- .../Dockerfiles/arkime.Dockerfile | 2 +- config/arkime-live.env.example | 1 - config/arkime-offline.env.example | 1 - config/arkime.env.example | 1 + docker-compose-standalone.yml | 2 - docker-compose.yml | 2 - scripts/install.py | 82 ++++++++++++++----- 8 files changed, 65 insertions(+), 28 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 67f86b7d6..f4d4325f1 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -264,7 +264,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ #Update Path ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}" -EXPOSE 8000 8005 8006 8081 +EXPOSE 8000 8005 8081 WORKDIR $ARKIME_DIR ENTRYPOINT ["/usr/bin/tini", \ diff --git a/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile b/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile index c0a51e564..890787325 100644 --- a/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile +++ b/arkime/arkime_regression_test_harness/Dockerfiles/arkime.Dockerfile @@ -66,7 +66,7 @@ RUN git clone --recursive --depth=1 --single-branch -b "$GITHUB_BRANCH" "$GITHUB ENV PATH="/opt:$ARKIME_DIR/bin:$ARKIME_DIR/node-v10.21.0-linux-x64/bin:${PATH}" -EXPOSE 8000 8005 8006 8081 +EXPOSE 8000 8005 8081 WORKDIR $ARKIME_DIR/tests diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example index c643faec1..bfd8e6656 100644 --- a/config/arkime-live.env.example +++ b/config/arkime-live.env.example @@ -1,4 +1,3 @@ -ARKIME_VIEWER_PORT=8006 # Whether or Arkime should monitor live traffic on a local # interface (PCAP_IFACE in pcap-capture.env specifies interface) ARKIME_LIVE_CAPTURE=false diff --git a/config/arkime-offline.env.example b/config/arkime-offline.env.example index f0548cc14..73e441806 100644 --- a/config/arkime-offline.env.example +++ b/config/arkime-offline.env.example @@ -1,4 +1,3 @@ -ARKIME_VIEWER_PORT=8005 # Whether or not Arkime should analyze uploaded PCAP files ARKIME_AUTO_ANALYZE_PCAP_FILES=true # The number of Arkime processes for analyzing uploaded PCAP files allowed diff --git a/config/arkime.env.example b/config/arkime.env.example index 2d935df51..3df6ecfc1 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -1,3 +1,4 @@ +ARKIME_VIEWER_PORT=8005 # Whether or not Arkime is allowed to delete uploaded/captured PCAP (see # https://arkime.com/faq#pcap-deletion) MANAGE_PCAP_FILES=false diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index fbe200595..29549986c 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -214,8 +214,6 @@ services: hard: -1 depends_on: - opensearch - ports: - - "127.0.0.1:8005:8005" volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro diff --git a/docker-compose.yml b/docker-compose.yml index ee0863b80..c0b3df962 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -236,8 +236,6 @@ services: hard: -1 depends_on: - opensearch - ports: - - "127.0.0.1:8005:8005" volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro diff --git a/scripts/install.py b/scripts/install.py index 8408144ed..042b762a7 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1044,6 +1044,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): if arkimeFreeSpaceGTmp: arkimeFreeSpaceG = arkimeFreeSpaceGTmp + autoArkime = InstallerYesOrNo('Automatically analyze all PCAP files with Arkime?', default=args.autoArkime) autoSuricata = InstallerYesOrNo( 'Automatically analyze all PCAP files with Suricata?', default=args.autoSuricata ) @@ -1084,7 +1085,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: if malcolmProfile == PROFILE_MALCOLM: - arkimeViewerOpen = False openPortsOptions = ('no', 'yes', 'customize') loopBreaker = CountUntilException(MaxAskForValueCount) while openPortsSelection not in [x[0] for x in openPortsOptions] and loopBreaker.increment(): @@ -1116,16 +1116,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path): openPortsSelection = 'n' logstashOpen = False filebeatTcpOpen = False - arkimeViewerOpen = InstallerYesOrNo( - 'Expose Arkime viewer to external hosts for PCAP payload retrieval?', - default=args.exposeArkimeViewer, - ) + else: opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal openPortsSelection = 'y' logstashOpen = True filebeatTcpOpen = True - arkimeViewerOpen = malcolmProfile == PROFILE_HEDGEHOG filebeatTcpFormat = 'json' filebeatTcpSourceField = 'message' @@ -1284,6 +1280,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): # input packet capture parameters pcapNetSniff = False pcapTcpDump = False + liveArkime = False liveZeek = False liveSuricata = False pcapIface = 'lo' @@ -1309,17 +1306,30 @@ def tweak_malcolm_runtime(self, malcolm_install_path): choices=[(x, '', x == captureOptions[0]) for x in captureOptions], )[0] if captureSelection == 'y': - pcapNetSniff = True + liveArkime = malcolmProfile == PROFILE_HEDGEHOG + pcapNetSniff = not liveArkime liveSuricata = True liveZeek = True elif captureSelection == 'c': if InstallerYesOrNo( 'Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?', - default=args.pcapNetSniff or args.pcapTcpDump or (malcolmProfile == PROFILE_HEDGEHOG), + default=args.pcapNetSniff + or args.pcapTcpDump + or args.liveArkime + or (malcolmProfile == PROFILE_HEDGEHOG), ): - pcapNetSniff = InstallerYesOrNo('Capture packets using netsniff-ng?', default=args.pcapNetSniff) - if not pcapNetSniff: - pcapTcpDump = InstallerYesOrNo('Capture packets using tcpdump?', default=args.pcapTcpDump) + liveArkime = (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) and ( + (malcolmProfile == PROFILE_HEDGEHOG) + or InstallerYesOrNo('Capture packets using Arkime capture?', default=args.liveArkime) + ) + pcapNetSniff = (not liveArkime) and InstallerYesOrNo( + 'Capture packets using netsniff-ng?', default=args.pcapNetSniff + ) + pcapTcpDump = ( + (not liveArkime) + and (not pcapNetSniff) + and InstallerYesOrNo('Capture packets using tcpdump?', default=args.pcapTcpDump) + ) liveSuricata = InstallerYesOrNo( 'Should Malcolm analyze live network traffic with Suricata?', default=args.liveSuricata ) @@ -1334,7 +1344,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): default=args.tweakIface, ) - if pcapNetSniff or pcapTcpDump or liveZeek or liveSuricata: + if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: pcapIface = '' loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid capture interface(s)') while (len(pcapIface) <= 0) and loopBreaker.increment(): @@ -1348,6 +1358,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): and (not pcapTcpDump) and (not liveZeek) and (not liveSuricata) + and (not liveArkime) ): InstallerDisplayMessage( f'Warning: Running with the {malcolmProfile} profile but no capture methods are enabled.', @@ -1393,6 +1404,24 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'ARKIME_FREESPACEG', arkimeFreeSpaceG, ), + # live traffic analysis with Arkime capture (only available with remote opensearch or elasticsearch) + EnvValue( + os.path.join(args.configDir, 'arkime-live.env'), + 'ARKIME_LIVE_CAPTURE', + TrueOrFalseNoQuote(liveArkime), + ), + # rotated captured PCAP analysis with Arkime (not live capture) + EnvValue( + os.path.join(args.configDir, 'arkime-offline.env'), + 'ARKIME_ROTATED_PCAP', + TrueOrFalseNoQuote(autoArkime and (not liveArkime)), + ), + # automatic uploaded pcap analysis with Arkime + EnvValue( + os.path.join(args.configDir, 'arkime-offline.env'), + 'ARKIME_AUTO_ANALYZE_PCAP_FILES', + TrueOrFalseNoQuote(autoArkime), + ), # authentication method: basic (true), ldap (false) or no_authentication EnvValue( os.path.join(args.configDir, 'auth-common.env'), @@ -1901,13 +1930,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): pcapDir, sectionIndents[currentSection] * 3, ) - elif re.match(r'^[\s#]*-\s*"([\d\.]+:)?\d+:\d+"\s*$', line): - # set bind IP based on whether it should be externally exposed or not - line = re.sub( - r'^([\s#]*-\s*")([\d\.]+:)?(\d+:\d+"\s*)$', - fr"\g<1>{'0.0.0.0' if arkimeViewerOpen else '127.0.0.1'}:\g<3>", - line, - ) elif currentService == 'filebeat': # stuff specifically in the filebeat section @@ -3508,6 +3530,16 @@ def main(): ) analysisArgGroup = parser.add_argument_group('Analysis options') + analysisArgGroup.add_argument( + '--auto-arkime', + dest='autoArkime', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=True, + help="Automatically analyze all PCAP files with Arkime", + ) analysisArgGroup.add_argument( '--auto-suricata', dest='autoSuricata', @@ -3759,6 +3791,16 @@ def main(): ) captureArgGroup.add_argument( '--live-capture-arkime', + dest='liveArkime', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help=f"Capture live network traffic with Arkime capture (not available with --opensearch {DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]})", + ) + captureArgGroup.add_argument( + '--live-capture-netsniff', dest='pcapNetSniff', type=str2bool, metavar="true|false", @@ -3768,7 +3810,7 @@ def main(): help="Capture live network traffic with netsniff-ng for Arkime", ) captureArgGroup.add_argument( - '--live-capture-arkime-tcpdump', + '--live-capture-tcpdump', dest='pcapTcpDump', type=str2bool, metavar="true|false", From 4a24fed04c2411f952ee76da636e93580919701f Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 5 Dec 2023 07:34:27 -0700 Subject: [PATCH 08/78] bump for v23.12.1 development --- .trigger_iso_workflow_build | 2 +- .trigger_workflow_build | 2 +- docker-compose-standalone.yml | 44 +++++----- docker-compose.yml | 44 +++++----- docs/contributing-pcap.md | 2 +- docs/download.md | 4 +- docs/hedgehog-iso-build.md | 2 +- docs/kubernetes.md | 88 +++++++++---------- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 38 ++++---- docs/ubuntu-install-example.md | 38 ++++---- kubernetes/03-opensearch.yml | 4 +- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 +- kubernetes/06-pcap-monitor.yml | 4 +- kubernetes/07-arkime.yml | 4 +- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 +- kubernetes/11-suricata.yml | 4 +- kubernetes/12-file-monitor.yml | 4 +- kubernetes/13-filebeat.yml | 4 +- kubernetes/14-logstash.yml | 4 +- kubernetes/15-netbox-redis.yml | 4 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 +- kubernetes/18-netbox.yml | 4 +- kubernetes/19-htadmin.yml | 4 +- kubernetes/20-pcap-capture.yml | 4 +- kubernetes/21-zeek-live.yml | 4 +- kubernetes/22-suricata-live.yml | 4 +- kubernetes/23-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 +- .../aws/ami/packer_vars.json.example | 2 +- 34 files changed, 173 insertions(+), 173 deletions(-) diff --git a/.trigger_iso_workflow_build b/.trigger_iso_workflow_build index 4ade3f725..bf8dee067 100644 --- a/.trigger_iso_workflow_build +++ b/.trigger_iso_workflow_build @@ -1,2 +1,2 @@ # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action -1 \ No newline at end of file +2 \ No newline at end of file diff --git a/.trigger_workflow_build b/.trigger_workflow_build index 4ade3f725..bf8dee067 100644 --- a/.trigger_workflow_build +++ b/.trigger_workflow_build @@ -1,2 +1,2 @@ # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action -1 \ No newline at end of file +2 \ No newline at end of file diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 1963a0f74..8b1333305 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -12,7 +12,7 @@ x-logging: services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:23.12.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.12.1 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -51,7 +51,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -81,7 +81,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -109,7 +109,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -153,7 +153,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -188,7 +188,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:23.12.0 + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -228,7 +228,7 @@ services: retries: 3 start_period: 210s zeek: - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -269,7 +269,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -300,7 +300,7 @@ services: - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -337,7 +337,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -366,7 +366,7 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -394,7 +394,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -418,7 +418,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -446,7 +446,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:23.12.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -473,7 +473,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.12.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -500,7 +500,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.12.0 + image: ghcr.io/idaholab/malcolm/freq:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -524,7 +524,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.12.0 + image: ghcr.io/idaholab/malcolm/netbox:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -558,7 +558,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.12.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -584,7 +584,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -614,7 +614,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -643,7 +643,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.12.0 + image: ghcr.io/idaholab/malcolm/api:23.12.1 profiles: ["malcolm"] logging: *default-logging command: gunicorn --bind 0:5000 manage:app @@ -670,7 +670,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index 85958b908..51b4cc688 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:23.12.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.12.1 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -57,7 +57,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -90,7 +90,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -121,7 +121,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -172,7 +172,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -210,7 +210,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:23.12.0 + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -256,7 +256,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -301,7 +301,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -336,7 +336,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -376,7 +376,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -408,7 +408,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -439,7 +439,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -466,7 +466,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -497,7 +497,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:23.12.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -524,7 +524,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.12.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -554,7 +554,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.12.0 + image: ghcr.io/idaholab/malcolm/freq:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -581,7 +581,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.12.0 + image: ghcr.io/idaholab/malcolm/netbox:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -618,7 +618,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.12.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -647,7 +647,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -680,7 +680,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -712,7 +712,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.12.0 + image: ghcr.io/idaholab/malcolm/api:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -745,7 +745,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index d8f0edd61..f9ed21c54 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v23.12.0 release]({{ site.github.repository_url }}/releases/tag/v23.12.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v23.12.1 release]({{ site.github.repository_url }}/releases/tag/v23.12.1)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/download.md b/docs/download.md index 1aca19bca..190220c55 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.12.0.iso](/iso/malcolm-23.12.0.iso) (5.1GiB) | [`3e836d09cd79a4e3f54c6fc365b032385312ad885b8483a0df156b59175d4909`](/iso/malcolm-23.12.0.iso.sha256.txt) | +| [malcolm-23.12.1.iso](/iso/malcolm-23.12.1.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-23.12.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.12.0.iso](/iso/hedgehog-23.12.0.iso) (2.4GiB) | [`835160cc0d2e3608754736989088d912c17372c49764244742e0572af9295d4b`](/iso/hedgehog-23.12.0.iso.sha256.txt) | +| [hedgehog-23.12.1.iso](/iso/hedgehog-23.12.1.iso) (2.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-23.12.1.iso.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index eb05c0ccb..a97d0d283 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-23.12.0.iso" +Finished, created "/sensor-build/hedgehog-23.12.1.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index da570cb96..d7c7088ad 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.12.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.12.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.12.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.12.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.12.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.12.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.12.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.12.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.12.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.12.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.12.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.12.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.12.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.12.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.12.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.12.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.12.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.12.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.12.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.12.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.12.1 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.1 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.12.1 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.12.1 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.12.1 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.12.1 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.12.1 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.12.1 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.12.1 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.12.1 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.12.1 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.12.1 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.12.1 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.12.1 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.12.1 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.1 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.12.1 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.12.1 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.12.1 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.12.1 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.12.1 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.12.1 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -557,28 +557,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.12.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.12.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.12.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.12.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.12.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.12.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.12.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.12.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.12.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.12.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.12.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.12.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.12.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.12.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.12.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.12.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.12.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.12.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.12.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.12.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.12.1 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.12.1 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.12.1 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.12.1 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.12.1 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.12.1 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.12.1 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.12.1 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.12.1 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.12.1 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.12.1 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.12.1 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.12.1 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.1 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.12.1 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.12.1 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.12.1 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.12.1 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.12.1 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.1 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.12.1 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.12.1 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index 309c48309..640276d77 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-23.12.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-23.12.1.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 0a6accdb5..91adf3d93 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.12.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.12.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.12.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.12.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.12.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.12.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.12.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.12.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.12.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.12.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.12.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.12.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.12.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.12.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.12.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.12.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.12.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.12.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.12.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.12.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.12.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.12.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.12.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.12.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.12.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.12.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.12.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.12.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.12.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.12.1 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 9278935e1..339ed20ec 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -250,25 +250,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.12.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.12.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.12.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.12.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.12.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.12.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.12.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.12.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.12.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.12.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.12.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.12.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.12.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.12.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.12.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.12.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.12.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.12.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.12.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.12.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.12.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.12.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.12.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.12.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.12.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.12.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.12.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.12.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.12.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.12.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 2f0ff97b0..dc55a9830 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/opensearch:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -69,7 +69,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 4817da142..6c1774a8b 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dashboards:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index c2152f0d9..2266cdba3 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/file-upload:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 9a2287e41..283baeb62 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/pcap-monitor:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 863c46f73..8a8b911da 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/arkime:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "arkime" initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index c158439fc..fee194155 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/api:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index a210c1b8d..8bdaf4c2d 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dashboards-helper:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 3e5c25046..4e898a593 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/zeek:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -72,7 +72,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index b7ed63050..64f1f3d4e 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/suricata:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index a8e9afd94..ee9879e08 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/file-monitor:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -81,7 +81,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index c38697906..fc6823a91 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/filebeat-oss:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 4b2d03bb7..d90ff6775 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/logstash-oss:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -113,7 +113,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 6fc358ecc..ebe65715e 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/redis:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index d8c7dc2f5..171a64176 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/redis:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 8bd333ede..18a6076ea 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/postgresql:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 8ca9d1fde..19c901803 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/netbox:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index d402c9e1b..b3761a565 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/htadmin:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index d82fd6274..40b06e941 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/pcap-capture:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -46,7 +46,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 725a21b10..eab494cf8 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/zeek:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 19e5763c8..999bc13bc 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/suricata:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-freq.yml b/kubernetes/23-freq.yml index 5173b8d2a..6c26785db 100644 --- a/kubernetes/23-freq.yml +++ b/kubernetes/23-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/freq:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 1f293bd64..ec26aef8b 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/nginx-proxy:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -95,7 +95,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 88031bc28..f6a3d87fd 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,7 +2,7 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", - "malcolm_tag": "v23.12.0", + "malcolm_tag": "v23.12.1", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", From ac58ba3fb06dac978206ed652363cbfb7d65d3c4 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 5 Dec 2023 07:39:34 -0700 Subject: [PATCH 09/78] copyright year update --- Dockerfiles/api.Dockerfile | 2 +- Dockerfiles/arkime.Dockerfile | 2 +- Dockerfiles/dirinit.Dockerfile | 2 +- Dockerfiles/file-monitor.Dockerfile | 2 +- Dockerfiles/file-upload.Dockerfile | 2 +- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/freq.Dockerfile | 2 +- Dockerfiles/htadmin.Dockerfile | 2 +- Dockerfiles/netbox.Dockerfile | 2 +- Dockerfiles/nginx.Dockerfile | 2 +- Dockerfiles/opensearch.Dockerfile | 2 +- Dockerfiles/pcap-capture.Dockerfile | 2 +- Dockerfiles/pcap-monitor.Dockerfile | 2 +- Dockerfiles/postgresql.Dockerfile | 2 +- Dockerfiles/redis.Dockerfile | 2 +- Dockerfiles/zeek.Dockerfile | 2 +- LICENSE.txt | 2 +- NOTICE.txt | 2 +- README.md | 2 +- arkime/scripts/arkime_update_geo.sh | 2 +- arkime/scripts/initarkime.sh | 2 +- arkime/scripts/viewer_service.sh | 2 +- arkime/scripts/wipearkime.sh | 2 +- arkime/scripts/wise_service.sh | 2 +- arkime/supervisord.conf | 2 +- arkime/wise/source.zeeklogs.js | 2 +- dashboards/scripts/create-arkime-sessions-index.sh | 2 +- dashboards/supervisord.conf | 2 +- docker-compose-standalone.yml | 2 +- docker-compose.yml | 2 +- file-monitor/docker-entrypoint.sh | 2 +- file-monitor/supervisord.conf | 2 +- file-upload/docker-entrypoint.sh | 2 +- file-upload/supervisord.conf | 2 +- filebeat/filebeat-nginx.yml | 2 +- filebeat/filebeat-tcp.yml | 2 +- filebeat/filebeat.yml | 2 +- filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py | 2 +- filebeat/scripts/filebeat-process-zeek-folder-functions.sh | 2 +- filebeat/scripts/filebeat-process-zeek-folder.sh | 2 +- filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py | 2 +- filebeat/scripts/zeek-log-field-bitmap.py | 2 +- filebeat/supervisord.conf | 2 +- freq-server/supervisord.conf | 2 +- htadmin/supervisord.conf | 2 +- logstash/pipelines/beats/11_beats_logs.conf | 2 +- logstash/pipelines/enrichment/20_enriched_to_ecs.conf | 2 +- logstash/pipelines/enrichment/23_severity.conf | 2 +- logstash/pipelines/enrichment/96_make_unique.conf | 2 +- logstash/pipelines/enrichment/97_arkimize.conf | 2 +- logstash/pipelines/enrichment/98_finalize.conf | 2 +- logstash/pipelines/suricata/19_severity.conf | 2 +- logstash/pipelines/zeek/10_zeek_prep.conf | 2 +- logstash/pipelines/zeek/11_zeek_parse.conf | 2 +- logstash/pipelines/zeek/12_zeek_mutate.conf | 2 +- logstash/pipelines/zeek/13_zeek_normalize.conf | 2 +- logstash/pipelines/zeek/14_zeek_convert.conf | 2 +- logstash/pipelines/zeek/19_severity.conf | 2 +- logstash/scripts/logstash-start.sh | 2 +- logstash/supervisord.conf | 2 +- malcolm-iso/Dockerfile | 2 +- .../config/hooks/normal/0168-firefox-install.hook.chroot | 2 +- .../config/hooks/normal/0911-get-stig-scripts.hook.chroot | 2 +- malcolm-iso/config/includes.binary/install/preseed_base.cfg | 2 +- malcolm-iso/config/includes.binary/install/preseed_multipar.cfg | 2 +- malcolm-iso/config/includes.binary/install/preseed_vmware.cfg | 2 +- malcolm-iso/vagrant/Vagrantfile | 2 +- netbox/scripts/netbox_init.py | 2 +- netbox/scripts/netbox_library_import.py | 2 +- netbox/supervisord.conf | 2 +- nginx/nginx.conf | 2 +- nginx/nginx_readonly.conf | 2 +- nginx/supervisord.conf | 2 +- pcap-capture/scripts/netsniff-roll.sh | 2 +- pcap-capture/scripts/supervisor.sh | 2 +- pcap-capture/supervisord.conf | 2 +- pcap-monitor/scripts/watch-pcap-uploads-folder.py | 2 +- pcap-monitor/supervisord.conf | 2 +- scripts/build.sh | 2 +- scripts/control.py | 2 +- scripts/demo/Vagrantfile | 2 +- scripts/demo/amazon_linux_2_malcolm_demo_setup.sh | 2 +- scripts/demo/reset_and_auto_populate.sh | 2 +- scripts/documentation_build.sh | 2 +- scripts/install.py | 2 +- scripts/malcolm_appliance_packager.sh | 2 +- scripts/malcolm_common.py | 2 +- scripts/malcolm_kubernetes.py | 2 +- scripts/malcolm_utils.py | 2 +- scripts/package_zeek_logs.sh | 2 +- .../aws/ami/scripts/Malcolm_AMI_Setup.sh | 2 +- scripts/third-party-logs/fluent-bit-setup.ps1 | 2 +- scripts/third-party-logs/fluent-bit-setup.sh | 2 +- scripts/zeek_script_to_malcolm_boilerplate.py | 2 +- sensor-iso/Dockerfile | 2 +- sensor-iso/arkime/Dockerfile | 2 +- sensor-iso/arkime/build-arkime-deb.sh | 2 +- sensor-iso/arkime/build-docker-image.sh | 2 +- sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot | 2 +- sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot | 2 +- sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot | 2 +- sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot | 2 +- .../config/hooks/normal/0911-get-stig-scripts.hook.chroot | 2 +- .../config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot | 2 +- .../config/hooks/normal/0991-security-performance.hook.chroot | 2 +- sensor-iso/config/hooks/normal/0992-login.hook.chroot | 2 +- sensor-iso/config/includes.binary/install/preseed_base.cfg | 2 +- sensor-iso/config/includes.binary/install/preseed_multipar.cfg | 2 +- sensor-iso/config/includes.binary/install/preseed_vmware.cfg | 2 +- .../includes.chroot/opt/zeek/share/zeek/site/extractor.zeek | 2 +- .../opt/zeek/share/zeek/site/extractor_params.zeek | 2 +- sensor-iso/interface/sensor_ctl/clean.sh | 2 +- sensor-iso/interface/sensor_ctl/control.sh | 2 +- sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml | 2 +- .../interface/sensor_ctl/filebeat/sensor_filebeat_local.sh | 2 +- sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml | 2 +- .../interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh | 2 +- sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh | 2 +- .../sensor_ctl/supervisor.init/arkime_config_populate.sh | 2 +- .../sensor_ctl/supervisor.init/supercronic_populate.sh | 2 +- sensor-iso/interface/sensor_ctl/supervisor.sh | 2 +- .../sensor_ctl/zeek/extractor_override.interesting.zeek | 2 +- sensor-iso/interface/sensor_interface/routes.py | 2 +- sensor-iso/interface/sensor_interface/static/js/custom.js | 2 +- sensor-iso/interface/sensor_interface/sysquery/sys_service.py | 2 +- sensor-iso/vagrant/Vagrantfile | 2 +- sensor-iso/yara/Dockerfile | 2 +- sensor-iso/yara/build-docker-image.sh | 2 +- sensor-iso/yara/build-yara-deb.sh | 2 +- sensor-iso/zeek/Dockerfile | 2 +- sensor-iso/zeek/build-docker-image.sh | 2 +- sensor-iso/zeek/build-zeek-deb.sh | 2 +- shared/bin/agg-init.sh | 2 +- shared/bin/capture-format-wait.sh | 2 +- shared/bin/common-init.sh | 2 +- shared/bin/configure-capture.py | 2 +- shared/bin/configure-interfaces.py | 2 +- shared/bin/docker-load-wait.sh | 2 +- shared/bin/fstab.py | 2 +- shared/bin/keystore-bootstrap.sh | 2 +- shared/bin/malcolm-first-run-configure.sh | 2 +- shared/bin/opensearch_read_only.py | 2 +- shared/bin/opensearch_status.sh | 2 +- shared/bin/pcap_processor.py | 2 +- shared/bin/pcap_utils.py | 2 +- shared/bin/pcap_watcher.py | 2 +- shared/bin/preseed_late_user_config.sh | 2 +- shared/bin/prune_files.sh | 2 +- shared/bin/sensor-capture-disk-config.py | 2 +- shared/bin/sensor-init.sh | 2 +- shared/bin/sensorcommon.py | 2 +- shared/bin/service_check_passthrough.sh | 2 +- shared/bin/set-dconf-screen-lock-defaults.sh | 2 +- shared/bin/set-malcolm-gtk-bookmark.sh | 2 +- shared/bin/suricata_config_populate.py | 2 +- shared/bin/suricata_update_config_populate.py | 2 +- shared/bin/therm-sensors-json.py | 2 +- shared/bin/ufw_allow_viewer.sh | 2 +- shared/bin/zeek_carve_logger.py | 2 +- shared/bin/zeek_carve_scanner.py | 2 +- shared/bin/zeek_carve_utils.py | 2 +- shared/bin/zeek_carve_watcher.py | 2 +- shared/bin/zeek_install_plugins.sh | 2 +- shared/bin/zeek_intel_from_threat_feed.py | 2 +- shared/bin/zeek_intel_setup.sh | 2 +- shared/bin/zeek_threat_feed_utils.py | 2 +- shared/bin/zeekdeploy.sh | 2 +- suricata/scripts/eve-clean-logs.sh | 2 +- suricata/supervisord.conf | 2 +- zeek/config/extractor.zeek | 2 +- zeek/config/extractor_override.interesting.zeek | 2 +- zeek/config/extractor_params.zeek | 2 +- zeek/supervisord.conf | 2 +- 173 files changed, 173 insertions(+), 173 deletions(-) diff --git a/Dockerfiles/api.Dockerfile b/Dockerfiles/api.Dockerfile index 08809510e..7023d633a 100644 --- a/Dockerfiles/api.Dockerfile +++ b/Dockerfiles/api.Dockerfile @@ -20,7 +20,7 @@ RUN python3 -m pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheel FROM python:3-slim-bookworm -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 96701db93..1a0b2ece3 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim AS build -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ENV DEBIAN_FRONTEND noninteractive ENV TERM xterm diff --git a/Dockerfiles/dirinit.Dockerfile b/Dockerfiles/dirinit.Dockerfile index 806252251..a17ecafc2 100644 --- a/Dockerfiles/dirinit.Dockerfile +++ b/Dockerfiles/dirinit.Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.18 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 6bfd6b86e..34b429a91 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/file-upload.Dockerfile b/Dockerfiles/file-upload.Dockerfile index 35175bc9a..cb194a588 100644 --- a/Dockerfiles/file-upload.Dockerfile +++ b/Dockerfiles/file-upload.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim AS npmget -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ENV DEBIAN_FRONTEND noninteractive diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index b8c4fcdbc..f4aad2064 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,6 +1,6 @@ FROM docker.elastic.co/beats/filebeat-oss:8.11.1 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/freq.Dockerfile b/Dockerfiles/freq.Dockerfile index a07318cef..82f9067b4 100644 --- a/Dockerfiles/freq.Dockerfile +++ b/Dockerfiles/freq.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile index 35c3728e9..ffcc27bd9 100644 --- a/Dockerfiles/htadmin.Dockerfile +++ b/Dockerfiles/htadmin.Dockerfile @@ -1,6 +1,6 @@ FROM debian:11-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index c075c03a2..4d18a2339 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -1,6 +1,6 @@ FROM netboxcommunity/netbox:v3.6.6 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index 5765943bd..32428d198 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. #################################################################################### # thanks to: nginx - https://github.com/nginxinc/docker-nginx/blob/master/mainline/alpine/Dockerfile diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index 95ea31bff..bdfccaf61 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,6 +1,6 @@ FROM opensearchproject/opensearch:2.11.1 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index 6244eaad9..c582e9e26 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index e8a3cd894..b1baabbee 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/postgresql.Dockerfile b/Dockerfiles/postgresql.Dockerfile index 757d13684..dde2e3b4d 100644 --- a/Dockerfiles/postgresql.Dockerfile +++ b/Dockerfiles/postgresql.Dockerfile @@ -1,6 +1,6 @@ FROM postgres:15-alpine -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/redis.Dockerfile b/Dockerfiles/redis.Dockerfile index 6585f2ea2..a7ce82e61 100644 --- a/Dockerfiles/redis.Dockerfile +++ b/Dockerfiles/redis.Dockerfile @@ -1,6 +1,6 @@ FROM redis:7-alpine -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 6c5c0cfc4..3395035bd 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get -q update && \ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/LICENSE.txt b/LICENSE.txt index db2310bed..e4db15ef7 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright 2023, Battelle Energy Alliance, LLC +Copyright 2024 Battelle Energy Alliance, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/NOTICE.txt b/NOTICE.txt index 55d121c11..4365ece5a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -4,7 +4,7 @@ https://github.com/idaholab/Malcolm See LICENSE.txt for license terms. -Malcolm is Copyright (c) 2023 Battelle Energy Alliance, LLC, and is developed +Malcolm is Copyright (c) 2024 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the Cybersecurity and Infrastructure Security Agency of the U.S. Department of Homeland Security. All rights reserved. diff --git a/README.md b/README.md index 31b2adab6..fb4a574d1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ You can help steer Malcolm's development by sharing your ideas and feedback. Ple ## Copyright and License -Malcolm is Copyright 2023 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the [Cybersecurity and Infrastructure Security Agency](https://www.cisa.gov/) of the [U.S. Department of Homeland Security](https://www.dhs.gov/). +Malcolm is Copyright 2024 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the [Cybersecurity and Infrastructure Security Agency](https://www.cisa.gov/) of the [U.S. Department of Homeland Security](https://www.dhs.gov/). Malcolm is licensed under the Apache License, version 2.0. See `LICENSE.txt` for the terms of its release. diff --git a/arkime/scripts/arkime_update_geo.sh b/arkime/scripts/arkime_update_geo.sh index 64be5973e..9b33f8b44 100755 --- a/arkime/scripts/arkime_update_geo.sh +++ b/arkime/scripts/arkime_update_geo.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. cd "${ARKIME_DIR:-/opt/arkime}"/etc diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh index 06f59eb28..6b861fa9a 100755 --- a/arkime/scripts/initarkime.sh +++ b/arkime/scripts/initarkime.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} diff --git a/arkime/scripts/viewer_service.sh b/arkime/scripts/viewer_service.sh index d1785b49b..54fe2209a 100755 --- a/arkime/scripts/viewer_service.sh +++ b/arkime/scripts/viewer_service.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. while true; do if [[ -f /var/run/arkime/initialized && "$VIEWER" == "on" ]]; then diff --git a/arkime/scripts/wipearkime.sh b/arkime/scripts/wipearkime.sh index 99b010118..6779dd6d1 100755 --- a/arkime/scripts/wipearkime.sh +++ b/arkime/scripts/wipearkime.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [[ ${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"} != "true" ]] && DB_SSL_FLAG="--insecure" || DB_SSL_FLAG="" OPENSEARCH_URL_FULL="$(grep -Pi '^elasticsearch\s*=' $ARKIME_DIR/etc/config.ini | cut -d'=' -f2-)" diff --git a/arkime/scripts/wise_service.sh b/arkime/scripts/wise_service.sh index ff9e26b34..5a952afae 100755 --- a/arkime/scripts/wise_service.sh +++ b/arkime/scripts/wise_service.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. while true; do if [[ ("$WISE" == "on") && (-f /var/run/arkime/runwise) && (-f $ARKIME_DIR/etc/wise.ini) ]]; then diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 44d0d10ed..dd7353b4c 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index ddd1b0e1c..40a9c4035 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -10,7 +10,7 @@ const WISESource = require('./wiseSource.js'); // Data may be populated with Malcolm's Logstash filters: // (https://github.com/idaholab/Malcolm/tree/main/logstash/pipelines) // -// Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +// Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. // see https://raw.githubusercontent.com/idaholab/Malcolm/main/LICENSE.txt ////////////////////////////////////////////////////////////////////////////////// diff --git a/dashboards/scripts/create-arkime-sessions-index.sh b/dashboards/scripts/create-arkime-sessions-index.sh index c839d0fd9..a118dd46c 100755 --- a/dashboards/scripts/create-arkime-sessions-index.sh +++ b/dashboards/scripts/create-arkime-sessions-index.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -euo pipefail shopt -s nocasematch diff --git a/dashboards/supervisord.conf b/dashboards/supervisord.conf index 81e64278a..b6cee6932 100644 --- a/dashboards/supervisord.conf +++ b/dashboards/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 8b1333305..3f997ab90 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. version: '3.7' diff --git a/docker-compose.yml b/docker-compose.yml index 51b4cc688..8202b7586 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. version: '3.7' diff --git a/file-monitor/docker-entrypoint.sh b/file-monitor/docker-entrypoint.sh index eb9955a1c..0027ecfd6 100755 --- a/file-monitor/docker-entrypoint.sh +++ b/file-monitor/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -z $EXTRACTED_FILE_ENABLE_CLAMAV ]]; then EXTRACTED_FILE_ENABLE_CLAMAV=false diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index d39508348..78cf4d79b 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/file-upload/docker-entrypoint.sh b/file-upload/docker-entrypoint.sh index 85dacd97f..f4b3c3afc 100755 --- a/file-upload/docker-entrypoint.sh +++ b/file-upload/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -z $MALCOLM_USERNAME || -z $MALCOLM_PASSWORD ]]; then echo "Please set the SSH username and (openssl-encrypted then base64-encoded) password by adding the following arguments to docker run/create:" diff --git a/file-upload/supervisord.conf b/file-upload/supervisord.conf index f85dac136..5a6f22aa2 100644 --- a/file-upload/supervisord.conf +++ b/file-upload/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/filebeat/filebeat-nginx.yml b/filebeat/filebeat-nginx.yml index 7bd453f6c..2d1247d13 100644 --- a/filebeat/filebeat-nginx.yml +++ b/filebeat/filebeat-nginx.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. name: "${PCAP_NODE_NAME:malcolm}" diff --git a/filebeat/filebeat-tcp.yml b/filebeat/filebeat-tcp.yml index 6a208eedc..dd5353579 100644 --- a/filebeat/filebeat-tcp.yml +++ b/filebeat/filebeat-tcp.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. name: "${PCAP_NODE_NAME:malcolm}" diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 046c1947f..f4429083a 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. name: "${PCAP_NODE_NAME:malcolm}" diff --git a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py index 736e7c0e7..3c2a38f9e 100755 --- a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py +++ b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import os diff --git a/filebeat/scripts/filebeat-process-zeek-folder-functions.sh b/filebeat/scripts/filebeat-process-zeek-folder-functions.sh index 806e0e44c..17cd2e8ff 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder-functions.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder-functions.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. function in_array() { local haystack="${1}[@]" diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh index 7b0c89c3c..825e26609 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # for files (sort -V (natural)) under /zeek that: diff --git a/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py b/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py index 7139361b6..aca0a0893 100755 --- a/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py +++ b/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for PCAP files for processing (by publishing their filenames to a ZMQ socket) diff --git a/filebeat/scripts/zeek-log-field-bitmap.py b/filebeat/scripts/zeek-log-field-bitmap.py index 8e05898ea..6ee960c40 100755 --- a/filebeat/scripts/zeek-log-field-bitmap.py +++ b/filebeat/scripts/zeek-log-field-bitmap.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # parse the fields names from the header of of the log file and compare them to the diff --git a/filebeat/supervisord.conf b/filebeat/supervisord.conf index 705ebc304..24c402bf6 100644 --- a/filebeat/supervisord.conf +++ b/filebeat/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/freq-server/supervisord.conf b/freq-server/supervisord.conf index e1f73dff7..0fc672422 100644 --- a/freq-server/supervisord.conf +++ b/freq-server/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/htadmin/supervisord.conf b/htadmin/supervisord.conf index c93a6d9b0..c0aba0f9d 100644 --- a/htadmin/supervisord.conf +++ b/htadmin/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index e4cd573f9..17c7a3998 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -2,7 +2,7 @@ # Malcolm and Hedgehog Linux itself (i.e., not captured # network traffic metadata, but operational metadata) # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ####################### filter { diff --git a/logstash/pipelines/enrichment/20_enriched_to_ecs.conf b/logstash/pipelines/enrichment/20_enriched_to_ecs.conf index 24552d683..5a46456a6 100644 --- a/logstash/pipelines/enrichment/20_enriched_to_ecs.conf +++ b/logstash/pipelines/enrichment/20_enriched_to_ecs.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/enrichment/23_severity.conf b/logstash/pipelines/enrichment/23_severity.conf index 2f794df35..6f8506c32 100644 --- a/logstash/pipelines/enrichment/23_severity.conf +++ b/logstash/pipelines/enrichment/23_severity.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/enrichment/96_make_unique.conf b/logstash/pipelines/enrichment/96_make_unique.conf index 22848bd4c..913f54514 100644 --- a/logstash/pipelines/enrichment/96_make_unique.conf +++ b/logstash/pipelines/enrichment/96_make_unique.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # take array fields that are already generic (i.e., ECS or Arkime) and deduplicate them. # there is also a little bit of light normalization that happens here diff --git a/logstash/pipelines/enrichment/97_arkimize.conf b/logstash/pipelines/enrichment/97_arkimize.conf index e5a88aa8c..384f9a455 100644 --- a/logstash/pipelines/enrichment/97_arkimize.conf +++ b/logstash/pipelines/enrichment/97_arkimize.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # take fields that are already generic (i.e., ECS or whatever) and copy them # to their Arkime equivalents if applicable diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf index d9a64f309..f292b9202 100644 --- a/logstash/pipelines/enrichment/98_finalize.conf +++ b/logstash/pipelines/enrichment/98_finalize.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # final adjustments before forwarding diff --git a/logstash/pipelines/suricata/19_severity.conf b/logstash/pipelines/suricata/19_severity.conf index 5c3043642..19d8db1eb 100644 --- a/logstash/pipelines/suricata/19_severity.conf +++ b/logstash/pipelines/suricata/19_severity.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 48c5a2d1e..6e0785a35 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index f6809839d..249450c3d 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -10,7 +10,7 @@ # - get filters where in != out # $ docker compose exec logstash curl -XGET http://localhost:9600/_node/stats/pipelines | jq -r '.. | .filters? // empty | .[] | objects | select (.events.in != .events.out) | [.id, .events.in, .events.out, .events.duration_in_millis] | join (";")' # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ####################### filter { diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 83ed11143..9eebdebc8 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index c4e74423b..48d97d88d 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/14_zeek_convert.conf b/logstash/pipelines/zeek/14_zeek_convert.conf index 51e9b2dfb..3103b0d62 100644 --- a/logstash/pipelines/zeek/14_zeek_convert.conf +++ b/logstash/pipelines/zeek/14_zeek_convert.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/19_severity.conf b/logstash/pipelines/zeek/19_severity.conf index 5851cf21e..aedf410ae 100644 --- a/logstash/pipelines/zeek/19_severity.conf +++ b/logstash/pipelines/zeek/19_severity.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/scripts/logstash-start.sh b/logstash/scripts/logstash-start.sh index be558a9d8..35df19c66 100755 --- a/logstash/scripts/logstash-start.sh +++ b/logstash/scripts/logstash-start.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/logstash/supervisord.conf b/logstash/supervisord.conf index f9734ba3f..a9b91b0bf 100644 --- a/logstash/supervisord.conf +++ b/logstash/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [inet_http_server] port=0.0.0.0:9001 diff --git a/malcolm-iso/Dockerfile b/malcolm-iso/Dockerfile index 7790c54a8..e371b85b0 100644 --- a/malcolm-iso/Dockerfile +++ b/malcolm-iso/Dockerfile @@ -1,6 +1,6 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot b/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot index 18c4941ab..98b7a4782 100755 --- a/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export LC_ALL=C.UTF-8 export LANG=C.UTF-8 diff --git a/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot b/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot index 32dceebc4..4ba6b95fb 100755 --- a/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # clone harbian-audit and clean up some stuff we don't need mkdir -p /opt diff --git a/malcolm-iso/config/includes.binary/install/preseed_base.cfg b/malcolm-iso/config/includes.binary/install/preseed_base.cfg index 1d18bfce4..f25b47593 100644 --- a/malcolm-iso/config/includes.binary/install/preseed_base.cfg +++ b/malcolm-iso/config/includes.binary/install/preseed_base.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i hw-detect/load_firmware boolean true d-i clock-setup/utc boolean true diff --git a/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg b/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg index 43624ac41..96d68233c 100644 --- a/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg +++ b/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg b/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg index 7d433121d..792d0c7a6 100644 --- a/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg +++ b/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/malcolm-iso/vagrant/Vagrantfile b/malcolm-iso/vagrant/Vagrantfile index 0558db616..ecc7852a2 100644 --- a/malcolm-iso/vagrant/Vagrantfile +++ b/malcolm-iso/vagrant/Vagrantfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. unless Vagrant.has_plugin?("vagrant-sshfs") raise 'vagrant-sshfs plugin is not installed!' diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index ac93af265..874776681 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import glob diff --git a/netbox/scripts/netbox_library_import.py b/netbox/scripts/netbox_library_import.py index 034ec0306..fa514c738 100644 --- a/netbox/scripts/netbox_library_import.py +++ b/netbox/scripts/netbox_library_import.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # adapted from minitriga/Netbox-Device-Type-Library-Import (MIT License) # Copyright (c) 2021 Alexander Gittings diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index 1703592bc..4bb797473 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [inet_http_server] port=0.0.0.0:9001 diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 05a138c45..6741c5861 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. daemon off; diff --git a/nginx/nginx_readonly.conf b/nginx/nginx_readonly.conf index 61e6b0012..bbc611d02 100644 --- a/nginx/nginx_readonly.conf +++ b/nginx/nginx_readonly.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. daemon off; diff --git a/nginx/supervisord.conf b/nginx/supervisord.conf index a03b1f7e3..4fa4d85ad 100644 --- a/nginx/supervisord.conf +++ b/nginx/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/pcap-capture/scripts/netsniff-roll.sh b/pcap-capture/scripts/netsniff-roll.sh index ca2ab23f9..1fd48530a 100755 --- a/pcap-capture/scripts/netsniff-roll.sh +++ b/pcap-capture/scripts/netsniff-roll.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. lastmod(){ expr $(date +%s) - $(stat -c %X "$1") diff --git a/pcap-capture/scripts/supervisor.sh b/pcap-capture/scripts/supervisor.sh index 46e612c8d..618e775ff 100755 --- a/pcap-capture/scripts/supervisor.sh +++ b/pcap-capture/scripts/supervisor.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/pcap-capture/supervisord.conf b/pcap-capture/supervisord.conf index 07a7b1af2..1ee0eb6a3 100644 --- a/pcap-capture/supervisord.conf +++ b/pcap-capture/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/pcap-monitor/scripts/watch-pcap-uploads-folder.py b/pcap-monitor/scripts/watch-pcap-uploads-folder.py index 2b7847f0e..8945d937e 100755 --- a/pcap-monitor/scripts/watch-pcap-uploads-folder.py +++ b/pcap-monitor/scripts/watch-pcap-uploads-folder.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for PCAP files for processing (by publishing their filenames to a ZMQ socket) diff --git a/pcap-monitor/supervisord.conf b/pcap-monitor/supervisord.conf index bb3e3bdc1..e1f7b6fc1 100644 --- a/pcap-monitor/supervisord.conf +++ b/pcap-monitor/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] diff --git a/scripts/build.sh b/scripts/build.sh index a4ff97c27..36bd18b5c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/scripts/control.py b/scripts/control.py index c6b39d6c9..665843d41 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import sys diff --git a/scripts/demo/Vagrantfile b/scripts/demo/Vagrantfile index a4859d27a..6ba20e272 100644 --- a/scripts/demo/Vagrantfile +++ b/scripts/demo/Vagrantfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. class VagrantPlugins::ProviderVirtualBox::Action::Network def dhcp_server_matches_config?(dhcp_server, config) diff --git a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh index ac074e5c0..41dd9bd90 100755 --- a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh +++ b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################### # for setting up a Malcolm demo instance on an Amazon Linux 2 instance from scratch diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh index 38dea9057..900b2a43a 100755 --- a/scripts/demo/reset_and_auto_populate.sh +++ b/scripts/demo/reset_and_auto_populate.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### # diff --git a/scripts/documentation_build.sh b/scripts/documentation_build.sh index d9a230878..7510d1b57 100755 --- a/scripts/documentation_build.sh +++ b/scripts/documentation_build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/scripts/install.py b/scripts/install.py index 8408144ed..26ca4a7c2 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import sys diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 3f7256549..38db1c456 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 843af368a..b4e0794fa 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import getpass import importlib diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 4bb7bc47c..0bd833bcf 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import base64 import glob diff --git a/scripts/malcolm_utils.py b/scripts/malcolm_utils.py index cab6703d1..febc46df1 100644 --- a/scripts/malcolm_utils.py +++ b/scripts/malcolm_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import contextlib import enum diff --git a/scripts/package_zeek_logs.sh b/scripts/package_zeek_logs.sh index b06d6dcff..8a484e4da 100755 --- a/scripts/package_zeek_logs.sh +++ b/scripts/package_zeek_logs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # package up Zeek logs in a format more suitable for upload to Malcolm # diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index a1c4d1f80..dcc032fe8 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # Configure Amazon Linux 2 and install Malcolm diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1 index 2fa6f40dd..8a35575dd 100644 --- a/scripts/third-party-logs/fluent-bit-setup.ps1 +++ b/scripts/third-party-logs/fluent-bit-setup.ps1 @@ -5,7 +5,7 @@ # configuration of fluent-bit (https://fluentbit.io/) for forwarding logs to # an instance of Malcolm (https://github.com/idaholab/malcolm). # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### $fluent_bit_version = '2.2' diff --git a/scripts/third-party-logs/fluent-bit-setup.sh b/scripts/third-party-logs/fluent-bit-setup.sh index 69499b873..6ab0b8366 100755 --- a/scripts/third-party-logs/fluent-bit-setup.sh +++ b/scripts/third-party-logs/fluent-bit-setup.sh @@ -7,7 +7,7 @@ # configuration of fluent-bit (https://fluentbit.io/) for forwarding logs to # an instance of Malcolm (https://github.com/idaholab/malcolm). # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### # force bash diff --git a/scripts/zeek_script_to_malcolm_boilerplate.py b/scripts/zeek_script_to_malcolm_boilerplate.py index 11c9857d3..f254099e0 100755 --- a/scripts/zeek_script_to_malcolm_boilerplate.py +++ b/scripts/zeek_script_to_malcolm_boilerplate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # # This script takes as input the filenames of one or more .zeek scripts which diff --git a/sensor-iso/Dockerfile b/sensor-iso/Dockerfile index 1510f7e1a..774c62213 100644 --- a/sensor-iso/Dockerfile +++ b/sensor-iso/Dockerfile @@ -1,6 +1,6 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index f6d5ab354..80c6074ea 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/arkime/build-arkime-deb.sh b/sensor-iso/arkime/build-arkime-deb.sh index c20eee583..b556741ab 100755 --- a/sensor-iso/arkime/build-arkime-deb.sh +++ b/sensor-iso/arkime/build-arkime-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ARKIME_URL="https://github.com/arkime/arkime.git" OUTPUT_DIR="/tmp" diff --git a/sensor-iso/arkime/build-docker-image.sh b/sensor-iso/arkime/build-docker-image.sh index 808a5641b..e071e492d 100755 --- a/sensor-iso/arkime/build-docker-image.sh +++ b/sensor-iso/arkime/build-docker-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot b/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot index 18c4941ab..98b7a4782 100755 --- a/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot +++ b/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export LC_ALL=C.UTF-8 export LANG=C.UTF-8 diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot index a1b67dde1..3a217625c 100755 --- a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export LC_ALL=C.UTF-8 export LANG=C.UTF-8 diff --git a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot b/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot index 7eeb00e8b..a97c39031 100755 --- a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot +++ b/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. sed -i 's/^exit 0//' /etc/rc.local 2>/dev/null diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 161106cea..31804abf0 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # some environment variables needed for build export CCACHE_DIR="/var/spool/ccache" diff --git a/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot b/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot index f5bc55347..2d9862133 100755 --- a/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot +++ b/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # clone harbian-audit and clean up some stuff we don't need mkdir -p /opt diff --git a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 09adb6273..ee8baa4ae 100755 --- a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # remove development packages not necessary for building dynamic Zeek plugins apt-get -y --purge remove \ diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot index 2a462a350..95349d90f 100755 --- a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # configure firewall sed -i "s/LOGLEVEL=.*/LOGLEVEL=off/" /etc/ufw/ufw.conf diff --git a/sensor-iso/config/hooks/normal/0992-login.hook.chroot b/sensor-iso/config/hooks/normal/0992-login.hook.chroot index d8dc2fff3..c8cd4a337 100755 --- a/sensor-iso/config/hooks/normal/0992-login.hook.chroot +++ b/sensor-iso/config/hooks/normal/0992-login.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. sed -i 's/^#autologin-user=.*/autologin-user=sensor/' /etc/lightdm/lightdm.conf sed -i 's/^#autologin-user-timeout=.*/autologin-user-timeout=0/' /etc/lightdm/lightdm.conf diff --git a/sensor-iso/config/includes.binary/install/preseed_base.cfg b/sensor-iso/config/includes.binary/install/preseed_base.cfg index 81b5ba435..bc3c87ca2 100644 --- a/sensor-iso/config/includes.binary/install/preseed_base.cfg +++ b/sensor-iso/config/includes.binary/install/preseed_base.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i netcfg/enable boolean false d-i netcfg/choose_interface select auto diff --git a/sensor-iso/config/includes.binary/install/preseed_multipar.cfg b/sensor-iso/config/includes.binary/install/preseed_multipar.cfg index 4f5c10499..9189b6932 100644 --- a/sensor-iso/config/includes.binary/install/preseed_multipar.cfg +++ b/sensor-iso/config/includes.binary/install/preseed_multipar.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg b/sensor-iso/config/includes.binary/install/preseed_vmware.cfg index 7d433121d..792d0c7a6 100644 --- a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg +++ b/sensor-iso/config/includes.binary/install/preseed_vmware.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek index 0418f23e9..07e745368 100644 --- a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek +++ b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. @load ./extractor_params diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek index a169bd0d9..2ffb8a8a2 100644 --- a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek +++ b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { const extractor_extract_none = "none" &redef; diff --git a/sensor-iso/interface/sensor_ctl/clean.sh b/sensor-iso/interface/sensor_ctl/clean.sh index e4f696d9c..047b1d220 100755 --- a/sensor-iso/interface/sensor_ctl/clean.sh +++ b/sensor-iso/interface/sensor_ctl/clean.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/sensor-iso/interface/sensor_ctl/control.sh b/sensor-iso/interface/sensor_ctl/control.sh index eb020d25d..84a645b1f 100755 --- a/sensor-iso/interface/sensor_ctl/control.sh +++ b/sensor-iso/interface/sensor_ctl/control.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml b/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml index 6b6ec2fda..0816bb59f 100644 --- a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml +++ b/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. logging.metrics.enabled: false diff --git a/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh b/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh index 47de02598..212a1d1ad 100755 --- a/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh +++ b/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -z "$ZEEK_CAPTURE_PATH" ]]; then ZEEK_CAPTURE_PATH="$HOME/bro_logs" diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml b/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml index de1d9eb84..f3a0dbae7 100644 --- a/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml +++ b/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filebeat.inputs: - type: tcp diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh b/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh index e650773dc..a62c80487 100755 --- a/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh +++ b/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory (containing config file) [[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath diff --git a/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh b/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh index 28d09a96b..798987cde 100755 --- a/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh +++ b/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. SPACE_STRING="$(/bin/df -lh --output=source,target,avail,size,pcent | tail -n +2 | grep '^/dev' | tr -s ' ' ',' | cut -d, -f2,3,4,5 | sed 's/^/\[/' | sed 's/$/\]/' | tr '\n' '.')" logger "${SPACE_STRING}" diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index 048e2944e..56c86b784 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export ARKIME_HTTPS_FLAG="" diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh index ee81c84ba..a4eb73573 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -n $SUPERVISOR_PATH ]] && [[ -d "$SUPERVISOR_PATH"/supercronic ]]; then diff --git a/sensor-iso/interface/sensor_ctl/supervisor.sh b/sensor-iso/interface/sensor_ctl/supervisor.sh index 7f1dc9241..cfe2562c5 100755 --- a/sensor-iso/interface/sensor_ctl/supervisor.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek b/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek index eaf220289..290a8948c 100644 --- a/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek +++ b/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { redef extractor_always_extract_unknown = F; diff --git a/sensor-iso/interface/sensor_interface/routes.py b/sensor-iso/interface/sensor_interface/routes.py index 2f4c3d66e..4a0b0c5ce 100644 --- a/sensor-iso/interface/sensor_interface/routes.py +++ b/sensor-iso/interface/sensor_interface/routes.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import psutil import time diff --git a/sensor-iso/interface/sensor_interface/static/js/custom.js b/sensor-iso/interface/sensor_interface/static/js/custom.js index 592b584cb..9e7373347 100644 --- a/sensor-iso/interface/sensor_interface/static/js/custom.js +++ b/sensor-iso/interface/sensor_interface/static/js/custom.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +// Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. function start_all() { var xhttp = new XMLHttpRequest(); diff --git a/sensor-iso/interface/sensor_interface/sysquery/sys_service.py b/sensor-iso/interface/sensor_interface/sysquery/sys_service.py index a1a432bf2..781882084 100644 --- a/sensor-iso/interface/sensor_interface/sysquery/sys_service.py +++ b/sensor-iso/interface/sensor_interface/sysquery/sys_service.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import subprocess import json diff --git a/sensor-iso/vagrant/Vagrantfile b/sensor-iso/vagrant/Vagrantfile index 500ab7083..04cfb86f6 100644 --- a/sensor-iso/vagrant/Vagrantfile +++ b/sensor-iso/vagrant/Vagrantfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. unless Vagrant.has_plugin?("vagrant-sshfs") raise 'vagrant-sshfs plugin is not installed!' diff --git a/sensor-iso/yara/Dockerfile b/sensor-iso/yara/Dockerfile index 4afe51c14..98b9c4674 100644 --- a/sensor-iso/yara/Dockerfile +++ b/sensor-iso/yara/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/yara/build-docker-image.sh b/sensor-iso/yara/build-docker-image.sh index 190dd2454..31060166b 100755 --- a/sensor-iso/yara/build-docker-image.sh +++ b/sensor-iso/yara/build-docker-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/sensor-iso/yara/build-yara-deb.sh b/sensor-iso/yara/build-yara-deb.sh index dd8415742..1fd9f7989 100755 --- a/sensor-iso/yara/build-yara-deb.sh +++ b/sensor-iso/yara/build-yara-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. YARA_URL="https://github.com/VirusTotal/YARA" YARA_VER="$(curl -sqI "$YARA_URL/releases/latest" | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}' | sed 's/^v//')" diff --git a/sensor-iso/zeek/Dockerfile b/sensor-iso/zeek/Dockerfile index 6086825ff..c4e69ce8e 100644 --- a/sensor-iso/zeek/Dockerfile +++ b/sensor-iso/zeek/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/zeek/build-docker-image.sh b/sensor-iso/zeek/build-docker-image.sh index 320226c35..3538212ff 100755 --- a/sensor-iso/zeek/build-docker-image.sh +++ b/sensor-iso/zeek/build-docker-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/sensor-iso/zeek/build-zeek-deb.sh b/sensor-iso/zeek/build-zeek-deb.sh index 4be53b1b8..0ea95d4dc 100755 --- a/sensor-iso/zeek/build-zeek-deb.sh +++ b/sensor-iso/zeek/build-zeek-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export CCACHE_DIR="/var/spool/ccache" export CCACHE_COMPRESS=1 diff --git a/shared/bin/agg-init.sh b/shared/bin/agg-init.sh index 3f8d0857f..b0215f97c 100755 --- a/shared/bin/agg-init.sh +++ b/shared/bin/agg-init.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. SCRIPT_PATH="$(dirname $(realpath -e "${BASH_SOURCE[0]}"))" diff --git a/shared/bin/capture-format-wait.sh b/shared/bin/capture-format-wait.sh index e2ba88095..1c4362e7c 100755 --- a/shared/bin/capture-format-wait.sh +++ b/shared/bin/capture-format-wait.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/common-init.sh b/shared/bin/common-init.sh index 645e0d181..19bbd3f9b 100755 --- a/shared/bin/common-init.sh +++ b/shared/bin/common-init.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. declare -A IFACES diff --git a/shared/bin/configure-capture.py b/shared/bin/configure-capture.py index e948c2bcb..c67729d81 100755 --- a/shared/bin/configure-capture.py +++ b/shared/bin/configure-capture.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # script for configuring sensor capture and forwarding parameters diff --git a/shared/bin/configure-interfaces.py b/shared/bin/configure-interfaces.py index f3f4c5bfc..56eafb508 100755 --- a/shared/bin/configure-interfaces.py +++ b/shared/bin/configure-interfaces.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # script for configuring sensor network interface controller(s) diff --git a/shared/bin/docker-load-wait.sh b/shared/bin/docker-load-wait.sh index c6cd55c8f..5f177c912 100755 --- a/shared/bin/docker-load-wait.sh +++ b/shared/bin/docker-load-wait.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/fstab.py b/shared/bin/fstab.py index 7f13f45ce..b6750cd84 100644 --- a/shared/bin/fstab.py +++ b/shared/bin/fstab.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # fstab interpreter diff --git a/shared/bin/keystore-bootstrap.sh b/shared/bin/keystore-bootstrap.sh index f24243523..b519db285 100644 --- a/shared/bin/keystore-bootstrap.sh +++ b/shared/bin/keystore-bootstrap.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # make sure the keystore file used by the tool (e.g., foobar) is copied or created # into the correct location before the tool. starts up. diff --git a/shared/bin/malcolm-first-run-configure.sh b/shared/bin/malcolm-first-run-configure.sh index 1f82c876e..3a81f572f 100755 --- a/shared/bin/malcolm-first-run-configure.sh +++ b/shared/bin/malcolm-first-run-configure.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/opensearch_read_only.py b/shared/bin/opensearch_read_only.py index 7d28bf4b3..38dac8715 100755 --- a/shared/bin/opensearch_read_only.py +++ b/shared/bin/opensearch_read_only.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import json diff --git a/shared/bin/opensearch_status.sh b/shared/bin/opensearch_status.sh index 47f7e5772..72883939b 100755 --- a/shared/bin/opensearch_status.sh +++ b/shared/bin/opensearch_status.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/shared/bin/pcap_processor.py b/shared/bin/pcap_processor.py index 09ff58a00..b1ab10951 100755 --- a/shared/bin/pcap_processor.py +++ b/shared/bin/pcap_processor.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Process queued files reported by pcap_watcher.py, using either arkime's capture or zeek to process diff --git a/shared/bin/pcap_utils.py b/shared/bin/pcap_utils.py index 2c9bf4f57..39fd5d313 100644 --- a/shared/bin/pcap_utils.py +++ b/shared/bin/pcap_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import os import re diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 821986ffd..8d70f322e 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for PCAP files for processing (by publishing their filenames to a ZMQ socket) diff --git a/shared/bin/preseed_late_user_config.sh b/shared/bin/preseed_late_user_config.sh index f5036608a..439c01a72 100755 --- a/shared/bin/preseed_late_user_config.sh +++ b/shared/bin/preseed_late_user_config.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################## # prompt whether to autologin or not diff --git a/shared/bin/prune_files.sh b/shared/bin/prune_files.sh index 56fe3a9dd..9d4c63f99 100755 --- a/shared/bin/prune_files.sh +++ b/shared/bin/prune_files.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # recursion depth (1 = not recursive) DEPTH=1 diff --git a/shared/bin/sensor-capture-disk-config.py b/shared/bin/sensor-capture-disk-config.py index b6ac3dd52..40c05b323 100755 --- a/shared/bin/sensor-capture-disk-config.py +++ b/shared/bin/sensor-capture-disk-config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Detect, partition, and format devices to be used for sensor packet/log captures. diff --git a/shared/bin/sensor-init.sh b/shared/bin/sensor-init.sh index 7a081f9ef..4deda1256 100755 --- a/shared/bin/sensor-init.sh +++ b/shared/bin/sensor-init.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. SCRIPT_PATH="$(dirname $(realpath -e "${BASH_SOURCE[0]}"))" diff --git a/shared/bin/sensorcommon.py b/shared/bin/sensorcommon.py index 19d97f3ff..8d5b3ebb3 100644 --- a/shared/bin/sensorcommon.py +++ b/shared/bin/sensorcommon.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import ipaddress diff --git a/shared/bin/service_check_passthrough.sh b/shared/bin/service_check_passthrough.sh index 6b98aed5c..53005ca41 100755 --- a/shared/bin/service_check_passthrough.sh +++ b/shared/bin/service_check_passthrough.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # This script will check command-line parameters and environment variables to see # if the service (determined by the hostname, unless otherwise specified) is diff --git a/shared/bin/set-dconf-screen-lock-defaults.sh b/shared/bin/set-dconf-screen-lock-defaults.sh index c5864c1fd..9ccb4c0c1 100755 --- a/shared/bin/set-dconf-screen-lock-defaults.sh +++ b/shared/bin/set-dconf-screen-lock-defaults.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/set-malcolm-gtk-bookmark.sh b/shared/bin/set-malcolm-gtk-bookmark.sh index f324b94dc..5b16afc88 100755 --- a/shared/bin/set-malcolm-gtk-bookmark.sh +++ b/shared/bin/set-malcolm-gtk-bookmark.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if ! grep -q Malcolm$ "$HOME"/.config/gtk-3.0/bookmarks && [[ -d "$HOME"/Malcolm ]]; then mkdir -p "$HOME"/.config/gtk-3.0/ diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index 612e30a15..eeb870077 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # modify suricata.yaml according to many environment variables diff --git a/shared/bin/suricata_update_config_populate.py b/shared/bin/suricata_update_config_populate.py index edcad9155..529bfce14 100755 --- a/shared/bin/suricata_update_config_populate.py +++ b/shared/bin/suricata_update_config_populate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # modify suricata's update.yaml according to many environment variables diff --git a/shared/bin/therm-sensors-json.py b/shared/bin/therm-sensors-json.py index 48f7d6b2f..87ff591ea 100755 --- a/shared/bin/therm-sensors-json.py +++ b/shared/bin/therm-sensors-json.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import json diff --git a/shared/bin/ufw_allow_viewer.sh b/shared/bin/ufw_allow_viewer.sh index ae581a314..90c06fbaf 100755 --- a/shared/bin/ufw_allow_viewer.sh +++ b/shared/bin/ufw_allow_viewer.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # manage a UFW rule for allowing a remote Arkime viewer instance (on the same host # to which arkime's capture is forwarding session logs) to connect to and diff --git a/shared/bin/zeek_carve_logger.py b/shared/bin/zeek_carve_logger.py index 0f2e0f884..2ab20dadd 100755 --- a/shared/bin/zeek_carve_logger.py +++ b/shared/bin/zeek_carve_logger.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for files extracted by zeek for processing diff --git a/shared/bin/zeek_carve_scanner.py b/shared/bin/zeek_carve_scanner.py index 35d56f7bd..c472b84b2 100755 --- a/shared/bin/zeek_carve_scanner.py +++ b/shared/bin/zeek_carve_scanner.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Process queued files reported by zeek_carve_watcher.py, scanning them with the specified diff --git a/shared/bin/zeek_carve_utils.py b/shared/bin/zeek_carve_utils.py index 1ac1129ea..21550aeaa 100644 --- a/shared/bin/zeek_carve_utils.py +++ b/shared/bin/zeek_carve_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import clamd import logging diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py index e2b93590a..960464905 100755 --- a/shared/bin/zeek_carve_watcher.py +++ b/shared/bin/zeek_carve_watcher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for files extracted by zeek for processing diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index d1588bf8b..70ce9b01f 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index 45184e838..456387238 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. from collections import deque from dateparser import parse as ParseDate diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index 077dade07..d4a8ad80b 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # set up intel files prior to running zeek # - https://idaholab.github.io/Malcolm/docs/zeek-intel.html#ZeekIntel diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index e5446a6aa..a7b0c21a7 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # adapted some code from tenzir/threatbus # - https://github.com/tenzir/threatbus diff --git a/shared/bin/zeekdeploy.sh b/shared/bin/zeekdeploy.sh index 0e2b2999e..f5a88d49f 100755 --- a/shared/bin/zeekdeploy.sh +++ b/shared/bin/zeekdeploy.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # get utilities for finding default zeek path and executable [[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath diff --git a/suricata/scripts/eve-clean-logs.sh b/suricata/scripts/eve-clean-logs.sh index e38a0e00b..33b0127e0 100755 --- a/suricata/scripts/eve-clean-logs.sh +++ b/suricata/scripts/eve-clean-logs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # Clean up suricata log files that have reached a certain age. If we can # verify they've been parsed and logged at least one event to the database, diff --git a/suricata/supervisord.conf b/suricata/supervisord.conf index 49e10393e..8a245f445 100644 --- a/suricata/supervisord.conf +++ b/suricata/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/zeek/config/extractor.zeek b/zeek/config/extractor.zeek index 0418f23e9..07e745368 100644 --- a/zeek/config/extractor.zeek +++ b/zeek/config/extractor.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. @load ./extractor_params diff --git a/zeek/config/extractor_override.interesting.zeek b/zeek/config/extractor_override.interesting.zeek index eaf220289..290a8948c 100644 --- a/zeek/config/extractor_override.interesting.zeek +++ b/zeek/config/extractor_override.interesting.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { redef extractor_always_extract_unknown = F; diff --git a/zeek/config/extractor_params.zeek b/zeek/config/extractor_params.zeek index 1d82162fe..653e91503 100644 --- a/zeek/config/extractor_params.zeek +++ b/zeek/config/extractor_params.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { const extractor_extract_none = "none" &redef; diff --git a/zeek/supervisord.conf b/zeek/supervisord.conf index 23db15fbe..4c5270253 100644 --- a/zeek/supervisord.conf +++ b/zeek/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) From 13298156640816e2905e806cad825bcf95029306 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 5 Dec 2023 08:16:22 -0700 Subject: [PATCH 10/78] update workflows --- .github/workflows/api-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/arkime-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/dashboards-build-and-push-ghcr.yml | 12 ++++++------ .../dashboards-helper-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/dirinit-build-and-push-ghcr.yml | 12 ++++++------ .../workflows/file-monitor-build-and-push-ghcr.yml | 12 ++++++------ .../workflows/file-upload-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/filebeat-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/freq-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/htadmin-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/logstash-build-and-push-ghcr.yml | 12 ++++++------ .../malcolm-iso-build-docker-wrap-push-ghcr.yml | 12 ++++++------ .github/workflows/netbox-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/nginx-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/opensearch-build-and-push-ghcr.yml | 12 ++++++------ .../workflows/pcap-capture-build-and-push-ghcr.yml | 12 ++++++------ .../workflows/pcap-monitor-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/postgresql-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/redis-build-and-push-ghcr.yml | 12 ++++++------ .../sensor-iso-build-docker-wrap-push-ghcr.yml | 12 ++++++------ .github/workflows/suricata-build-and-push-ghcr.yml | 12 ++++++------ .github/workflows/zeek-build-and-push-ghcr.yml | 12 ++++++------ 22 files changed, 132 insertions(+), 132 deletions(-) diff --git a/.github/workflows/api-build-and-push-ghcr.yml b/.github/workflows/api-build-and-push-ghcr.yml index f1f5a5d5f..5a3ab5eee 100644 --- a/.github/workflows/api-build-and-push-ghcr.yml +++ b/.github/workflows/api-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/api.Dockerfile diff --git a/.github/workflows/arkime-build-and-push-ghcr.yml b/.github/workflows/arkime-build-and-push-ghcr.yml index ca9d674a5..51e258b03 100644 --- a/.github/workflows/arkime-build-and-push-ghcr.yml +++ b/.github/workflows/arkime-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/arkime.Dockerfile diff --git a/.github/workflows/dashboards-build-and-push-ghcr.yml b/.github/workflows/dashboards-build-and-push-ghcr.yml index 20fb0371d..5f4e729ee 100644 --- a/.github/workflows/dashboards-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/dashboards.Dockerfile diff --git a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml index 624d2ac30..2a4fb6a47 100644 --- a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/dashboards-helper.Dockerfile diff --git a/.github/workflows/dirinit-build-and-push-ghcr.yml b/.github/workflows/dirinit-build-and-push-ghcr.yml index af49b0e66..c57ed918a 100644 --- a/.github/workflows/dirinit-build-and-push-ghcr.yml +++ b/.github/workflows/dirinit-build-and-push-ghcr.yml @@ -23,14 +23,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -38,23 +38,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/dirinit.Dockerfile diff --git a/.github/workflows/file-monitor-build-and-push-ghcr.yml b/.github/workflows/file-monitor-build-and-push-ghcr.yml index e056c0393..c74860015 100644 --- a/.github/workflows/file-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/file-monitor-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/file-monitor.Dockerfile diff --git a/.github/workflows/file-upload-build-and-push-ghcr.yml b/.github/workflows/file-upload-build-and-push-ghcr.yml index b04551dd6..eb42c5ee1 100644 --- a/.github/workflows/file-upload-build-and-push-ghcr.yml +++ b/.github/workflows/file-upload-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/file-upload.Dockerfile diff --git a/.github/workflows/filebeat-build-and-push-ghcr.yml b/.github/workflows/filebeat-build-and-push-ghcr.yml index b7cf3a385..c56c1c6a8 100644 --- a/.github/workflows/filebeat-build-and-push-ghcr.yml +++ b/.github/workflows/filebeat-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/filebeat.Dockerfile diff --git a/.github/workflows/freq-build-and-push-ghcr.yml b/.github/workflows/freq-build-and-push-ghcr.yml index a04a6d758..b8e589f19 100644 --- a/.github/workflows/freq-build-and-push-ghcr.yml +++ b/.github/workflows/freq-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/freq.Dockerfile diff --git a/.github/workflows/htadmin-build-and-push-ghcr.yml b/.github/workflows/htadmin-build-and-push-ghcr.yml index b6173f98b..04ac783ce 100644 --- a/.github/workflows/htadmin-build-and-push-ghcr.yml +++ b/.github/workflows/htadmin-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/htadmin.Dockerfile diff --git a/.github/workflows/logstash-build-and-push-ghcr.yml b/.github/workflows/logstash-build-and-push-ghcr.yml index 3151903e4..f358e2226 100644 --- a/.github/workflows/logstash-build-and-push-ghcr.yml +++ b/.github/workflows/logstash-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/logstash.Dockerfile diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml index 9f68aaaa8..488575fe8 100644 --- a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -29,17 +29,17 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master @@ -78,7 +78,7 @@ jobs: sudo rm -rf /tmp/live-build /tmp/live-build*.deb - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -139,14 +139,14 @@ jobs: sarif_file: 'trivy-results.sarif' - name: ghcr.io login - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push ISO image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: ./malcolm-iso push: true diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml index d67a45ec6..65cc11a71 100644 --- a/.github/workflows/netbox-build-and-push-ghcr.yml +++ b/.github/workflows/netbox-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/netbox.Dockerfile diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml index 4c63f216a..7ef3e9a7f 100644 --- a/.github/workflows/nginx-build-and-push-ghcr.yml +++ b/.github/workflows/nginx-build-and-push-ghcr.yml @@ -38,14 +38,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -58,23 +58,23 @@ jobs: id: extract_commit_sha - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/nginx.Dockerfile diff --git a/.github/workflows/opensearch-build-and-push-ghcr.yml b/.github/workflows/opensearch-build-and-push-ghcr.yml index 40b743459..225b89a2d 100644 --- a/.github/workflows/opensearch-build-and-push-ghcr.yml +++ b/.github/workflows/opensearch-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/opensearch.Dockerfile diff --git a/.github/workflows/pcap-capture-build-and-push-ghcr.yml b/.github/workflows/pcap-capture-build-and-push-ghcr.yml index 7ded814d0..e8de170c4 100644 --- a/.github/workflows/pcap-capture-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-capture-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/pcap-capture.Dockerfile diff --git a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml index aea7ac085..ca5138e18 100644 --- a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/pcap-monitor.Dockerfile diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml index 36db42be1..29aefda44 100644 --- a/.github/workflows/postgresql-build-and-push-ghcr.yml +++ b/.github/workflows/postgresql-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/postgresql.Dockerfile diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml index b89103496..a04d0369b 100644 --- a/.github/workflows/redis-build-and-push-ghcr.yml +++ b/.github/workflows/redis-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/redis.Dockerfile diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml index ce522185f..db374d6f2 100644 --- a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml @@ -27,17 +27,17 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master @@ -74,7 +74,7 @@ jobs: sudo rm -rf /tmp/live-build /tmp/live-build*.deb - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -129,14 +129,14 @@ jobs: sarif_file: 'trivy-results.sarif' - name: ghcr.io login - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push ISO image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: ./sensor-iso push: true diff --git a/.github/workflows/suricata-build-and-push-ghcr.yml b/.github/workflows/suricata-build-and-push-ghcr.yml index e76abb523..bd07b7006 100644 --- a/.github/workflows/suricata-build-and-push-ghcr.yml +++ b/.github/workflows/suricata-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/suricata.Dockerfile diff --git a/.github/workflows/zeek-build-and-push-ghcr.yml b/.github/workflows/zeek-build-and-push-ghcr.yml index 2cd12c3ab..4cb14636c 100644 --- a/.github/workflows/zeek-build-and-push-ghcr.yml +++ b/.github/workflows/zeek-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/zeek.Dockerfile From b0f39c37090e238e90f7dd563c3bbf6583b5ba87 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 5 Dec 2023 08:19:29 -0700 Subject: [PATCH 11/78] change URL for downloading manuf list to new wireshark.org URL, idaholab/Malcolm#306 --- Dockerfiles/arkime.Dockerfile | 2 +- arkime/scripts/arkime_update_geo.sh | 2 +- sensor-iso/build.sh | 2 +- shared/bin/manuf-oui-parse.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 1a0b2ece3..3a0d4371e 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -216,7 +216,7 @@ RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do rm -f "GeoLite2-$DB*"; \ done; \ curl -s -S -L -o $ARKIME_DIR/etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" && \ - curl -s -S -L -o $ARKIME_DIR/etc/oui.txt "https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf" + curl -s -S -L -o $ARKIME_DIR/etc/oui.txt "https://www.wireshark.org/download/automated/data/manuf" RUN groupadd --gid $DEFAULT_GID $PGROUP && \ useradd -M --uid $DEFAULT_UID --gid $DEFAULT_GID --home $ARKIME_DIR $PUSER && \ diff --git a/arkime/scripts/arkime_update_geo.sh b/arkime/scripts/arkime_update_geo.sh index 9b33f8b44..aed3c1ccf 100755 --- a/arkime/scripts/arkime_update_geo.sh +++ b/arkime/scripts/arkime_update_geo.sh @@ -8,7 +8,7 @@ wget -nv --no-check-certificate -O ipv4-address-space.csv_new https://www.iana.o mv -f ipv4-address-space.csv_new ipv4-address-space.csv || \ rm -f ipv4-address-space.csv_new -wget -nv -O oui.txt_new https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf && \ +wget -nv -O oui.txt_new https://www.wireshark.org/download/automated/data/manuf && \ mv -f oui.txt_new oui.txt || \ rm -f oui.txt_new diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index 0bc69ef89..4dff09cbd 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -167,7 +167,7 @@ if [ -d "$WORKDIR" ]; then fi fi curl -s -S -L -o ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" - curl -s -S -L -o oui.txt "https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf" + curl -s -S -L -o oui.txt "https://www.wireshark.org/download/automated/data/manuf" popd >/dev/null 2>&1 # clone and build Arkime .deb package in its own clean environment (rather than in hooks/) diff --git a/shared/bin/manuf-oui-parse.py b/shared/bin/manuf-oui-parse.py index e37d65ed0..92920adff 100755 --- a/shared/bin/manuf-oui-parse.py +++ b/shared/bin/manuf-oui-parse.py @@ -17,7 +17,7 @@ except ImportError: import yaml -DEFAULT_MANUF_URL = "https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf" +DEFAULT_MANUF_URL = "https://www.wireshark.org/download/automated/data/manuf" padded_mac_low = '00:00:00:00:00:00' padded_mac_high = 'FF:FF:FF:FF:FF:FF' mac_pattern = re.compile(r"[-:\.]") From 78861db647c6aeb595eab6bbb4d4318091aabbbe Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 5 Dec 2023 12:52:13 -0700 Subject: [PATCH 12/78] surface .env variable for zeek intel refresh threads --- config/zeek.env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/zeek.env.example b/config/zeek.env.example index ca0c9d6c0..7e0e64249 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -12,6 +12,8 @@ ZEEK_INTEL_FEED_SINCE= # Specifies a cron expression indicating the refresh interval for generating the # Zeek Intelligence Framework files ('' disables automatic refresh) ZEEK_INTEL_REFRESH_CRON_EXPRESSION= +# Number of threads to use for querying feeds for generating Zeek Intelligence Framework files +ZEEK_INTEL_REFRESH_THREADS=2 # Determines the file extraction behavior for file transfers detected by Zeek ZEEK_EXTRACTOR_MODE=none # Whether or not to use polling vs. native inotify API to watch for files From 9284881bc451c66cd887c9e54cfd3f39c9af778f Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 6 Dec 2023 14:33:40 -0700 Subject: [PATCH 13/78] for idaholab/Malcolm#282, reviewing capabilities for containers --- docker-compose-standalone.yml | 47 ++++++++++----------------------- docker-compose.yml | 47 ++++++++++----------------------- kubernetes/03-opensearch.yml | 2 ++ kubernetes/07-arkime.yml | 5 ---- kubernetes/10-zeek.yml | 7 ++--- kubernetes/11-suricata.yml | 7 ++--- kubernetes/14-logstash.yml | 2 ++ kubernetes/20-pcap-capture.yml | 4 +-- kubernetes/21-zeek-live.yml | 6 ++--- kubernetes/22-suricata-live.yml | 6 ++--- 10 files changed, 44 insertions(+), 89 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 3f997ab90..38e38f18f 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -36,6 +36,7 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK required in order for OpenSearch to lock memory, preventing swapping - IPC_LOCK volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro @@ -123,6 +124,7 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK required in order for LogStash to lock memory, preventing swapping - IPC_LOCK env_file: - ./config/process.env @@ -207,10 +209,6 @@ services: - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' - ulimits: - memlock: - soft: -1 - hard: -1 depends_on: - opensearch ports: @@ -237,15 +235,9 @@ services: hostname: zeek networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -276,15 +268,12 @@ services: stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values and scheduling policies for capture + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -309,15 +298,9 @@ services: hostname: suricata networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -344,15 +327,12 @@ services: stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values and scheduling policies for capture + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -406,10 +386,11 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK required for some of the memory optimizations netsniff-ng does - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env diff --git a/docker-compose.yml b/docker-compose.yml index 8202b7586..4555f368f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,7 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK required in order for OpenSearch to lock memory, preventing swapping - IPC_LOCK volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro @@ -135,6 +136,7 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK required in order for LogStash to lock memory, preventing swapping - IPC_LOCK env_file: - ./config/process.env @@ -229,10 +231,6 @@ services: - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' - ulimits: - memlock: - soft: -1 - hard: -1 depends_on: - opensearch ports: @@ -265,15 +263,9 @@ services: hostname: zeek networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -308,15 +300,12 @@ services: stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -345,15 +334,9 @@ services: hostname: suricata networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -383,15 +366,12 @@ services: stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values and scheduling policies for capture + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -451,10 +431,11 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK required for some of the memory optimizations netsniff-ng does - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index dc55a9830..1c3b68a00 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -37,7 +37,9 @@ spec: securityContext: capabilities: add: + # IPC_LOCK - for OpenSearch to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limit on the node running Elasticsearch - SYS_RESOURCE ports: - name: opensearch diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 8a8b911da..aaaa74ae2 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -34,11 +34,6 @@ spec: imagePullPolicy: Always stdin: false tty: true - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE ports: - name: http protocol: TCP diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 4e898a593..b71c15f23 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -23,11 +23,8 @@ spec: securityContext: capabilities: add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE envFrom: - configMapRef: name: process-env diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index 64f1f3d4e..deeec963c 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -23,11 +23,8 @@ spec: securityContext: capabilities: add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE envFrom: - configMapRef: name: process-env diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index d90ff6775..524a98f2f 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -56,7 +56,9 @@ spec: securityContext: capabilities: add: + # IPC_LOCK - for LogStash to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limit on the node running LogStash - SYS_RESOURCE ports: - name: lumberjack diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 40b06e941..f1424883b 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -23,11 +23,11 @@ spec: securityContext: capabilities: add: + # IPC_LOCK required for some of the memory optimizations netsniff-ng does - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE envFrom: - configMapRef: name: process-env diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index eab494cf8..9991bbaa0 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -23,11 +23,11 @@ spec: securityContext: capabilities: add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE + # SYS_NICE - to set process nice values and scheduling policies for capture + - SYS_NICE envFrom: - configMapRef: name: process-env diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 999bc13bc..3f05373e6 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -23,11 +23,11 @@ spec: securityContext: capabilities: add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE envFrom: - configMapRef: name: process-env From 3b437b93d60c394a7864c39986a4f9a76f03324c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 6 Dec 2023 14:45:42 -0700 Subject: [PATCH 14/78] for idaholab/Malcolm#282, reviewing capabilities for containers --- docker-compose-standalone.yml | 12 +++++++++--- docker-compose.yml | 12 +++++++++--- kubernetes/03-opensearch.yml | 4 ++-- kubernetes/14-logstash.yml | 4 ++-- kubernetes/20-pcap-capture.yml | 2 ++ 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 38e38f18f..6d2c4e415 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -36,8 +36,10 @@ services: soft: -1 hard: -1 cap_add: - # IPC_LOCK required in order for OpenSearch to lock memory, preventing swapping + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro @@ -124,8 +126,10 @@ services: soft: -1 hard: -1 cap_add: - # IPC_LOCK required in order for LogStash to lock memory, preventing swapping + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE env_file: - ./config/process.env - ./config/ssl.env @@ -386,8 +390,10 @@ services: soft: -1 hard: -1 cap_add: - # IPC_LOCK required for some of the memory optimizations netsniff-ng does + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/docker-compose.yml b/docker-compose.yml index 4555f368f..2f9ec451c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,8 +39,10 @@ services: soft: -1 hard: -1 cap_add: - # IPC_LOCK required in order for OpenSearch to lock memory, preventing swapping + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro @@ -136,8 +138,10 @@ services: soft: -1 hard: -1 cap_add: - # IPC_LOCK required in order for LogStash to lock memory, preventing swapping + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE env_file: - ./config/process.env - ./config/ssl.env @@ -431,8 +435,10 @@ services: soft: -1 hard: -1 cap_add: - # IPC_LOCK required for some of the memory optimizations netsniff-ng does + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 1c3b68a00..1d8370e8a 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -37,9 +37,9 @@ spec: securityContext: capabilities: add: - # IPC_LOCK - for OpenSearch to lock memory, preventing swapping + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK - # SYS_RESOURCE - for increasing memlock limit on the node running Elasticsearch + # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE ports: - name: opensearch diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 524a98f2f..22f7fa97f 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -56,9 +56,9 @@ spec: securityContext: capabilities: add: - # IPC_LOCK - for LogStash to lock memory, preventing swapping + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK - # SYS_RESOURCE - for increasing memlock limit on the node running LogStash + # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE ports: - name: lumberjack diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index f1424883b..7eb5c83ec 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -25,6 +25,8 @@ spec: add: # IPC_LOCK required for some of the memory optimizations netsniff-ng does - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW From 23e7ea46e49a9633a87813a06f1a7e5ed30fbf7c Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 6 Dec 2023 14:47:04 -0700 Subject: [PATCH 15/78] for idaholab/Malcolm#282, reviewing capabilities for containers --- Dockerfiles/arkime.Dockerfile | 1 - Dockerfiles/suricata.Dockerfile | 1 - Dockerfiles/zeek.Dockerfile | 1 - 3 files changed, 3 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 3a0d4371e..a3d18c805 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -93,7 +93,6 @@ ENV DEFAULT_GID $DEFAULT_GID ENV PUSER "arkime" ENV PGROUP "arkime" ENV PUSER_PRIV_DROP true -ENV PUSER_RLIMIT_UNLOCK true ENV DEBIAN_FRONTEND noninteractive ENV TERM xterm diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index ae4539970..53306d55f 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -28,7 +28,6 @@ ENV PGROUP "suricata" # docker-uid-gid-setup.sh will cause them to be lost, so we need # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false -ENV PUSER_RLIMIT_UNLOCK true ENV SUPERCRONIC_VERSION "0.2.28" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 3395035bd..7440a3327 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -92,7 +92,6 @@ ENV PGROUP "zeeker" # docker-uid-gid-setup.sh will cause them to be lost, so we need # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false -ENV PUSER_RLIMIT_UNLOCK true ENV SUPERCRONIC_VERSION "0.2.28" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" From e4a4387218adf30b97ca1ae20a9243d57cb6efbf Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 07:35:39 -0700 Subject: [PATCH 16/78] beats to v8.11.2 (https://www.elastic.co/guide/en/beats/libbeat/current/release-notes-8.11.2.html) and logstash to v8.11.2 (https://www.elastic.co/guide/en/logstash/current/logstash-8-11-2.html) --- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/logstash.Dockerfile | 2 +- sensor-iso/build.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index f4aad2064..e11a366af 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.11.1 +FROM docker.elastic.co/beats/filebeat-oss:8.11.2 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index ef0777b55..f4424705d 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.11.1 +FROM docker.elastic.co/logstash/logstash-oss:8.11.2 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index 4dff09cbd..b5841ce94 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.11.1" +BEATS_VER="8.11.2" BEATS_OSS="-oss" BUILD_ERROR_CODE=1 From f345c7b06534911b3f3ec41d79f04f28cccea53c Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 07:50:55 -0700 Subject: [PATCH 17/78] comments for idaholab/Malcolm#282 --- docker-compose-standalone.yml | 6 +++--- docker-compose.yml | 4 ++-- kubernetes/10-zeek.yml | 2 +- kubernetes/11-suricata.yml | 2 +- kubernetes/21-zeek-live.yml | 2 +- kubernetes/22-suricata-live.yml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 6d2c4e415..2a729d9ca 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -240,7 +240,7 @@ services: networks: - default cap_add: - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env @@ -276,7 +276,7 @@ services: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values and scheduling policies for capture + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env @@ -303,7 +303,7 @@ services: networks: - default cap_add: - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env diff --git a/docker-compose.yml b/docker-compose.yml index 2f9ec451c..9ffc6a6b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -268,7 +268,7 @@ services: networks: - default cap_add: - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env @@ -374,7 +374,7 @@ services: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values and scheduling policies for capture + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling - SYS_NICE env_file: - ./config/process.env diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index b71c15f23..c4af201cd 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -23,7 +23,7 @@ spec: securityContext: capabilities: add: - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE envFrom: - configMapRef: diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index deeec963c..574da3fea 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -23,7 +23,7 @@ spec: securityContext: capabilities: add: - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE envFrom: - configMapRef: diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 9991bbaa0..f7bde1e28 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -26,7 +26,7 @@ spec: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values and scheduling policies for capture + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE envFrom: - configMapRef: diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 3f05373e6..a37847fc2 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -26,7 +26,7 @@ spec: # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE envFrom: - configMapRef: From 7c67665a68af9fb27f9ec1c25214137e4ca7f8e0 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 08:46:41 -0700 Subject: [PATCH 18/78] goStatic -vhost '' --- shared/bin/service_check_passthrough.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/service_check_passthrough.sh b/shared/bin/service_check_passthrough.sh index 53005ca41..3e63a57a4 100755 --- a/shared/bin/service_check_passthrough.sh +++ b/shared/bin/service_check_passthrough.sh @@ -159,7 +159,7 @@ EOF fi # json vs http if command -v goStatic >/dev/null 2>&1; then - goStatic -path "$(pwd)" -fallback "index.html" -port $PORT + goStatic -vhost "" -path "$(pwd)" -fallback "index.html" -port $PORT elif command -v python3 >/dev/null 2>&1; then python3 -m http.server --bind 0.0.0.0 $PORT elif command -v python >/dev/null 2>&1; then From ce2b7b8ad38b4d7c6d4765b24177ef34517db455 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 09:39:34 -0700 Subject: [PATCH 19/78] added python3-dotenv as an install script dependency --- scripts/install.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 8eced2ba5..3a9aa26ac 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -2339,9 +2339,27 @@ def __init__(self, orchMode, debug=False, configOnly=False): # determine packages required by Malcolm itself (not docker, those will be done later) if (self.distro == PLATFORM_LINUX_UBUNTU) or (self.distro == PLATFORM_LINUX_DEBIAN): - self.requiredPackages.extend(['apache2-utils', 'make', 'openssl', 'python3-dialog', 'xz-utils']) + self.requiredPackages.extend( + [ + 'apache2-utils', + 'make', + 'openssl', + 'python3-dialog', + 'python3-dotenv', + 'xz-utils', + ] + ) elif (self.distro == PLATFORM_LINUX_FEDORA) or (self.distro == PLATFORM_LINUX_CENTOS): - self.requiredPackages.extend(['httpd-tools', 'make', 'openssl', 'python3-dialog', 'xz']) + self.requiredPackages.extend( + [ + 'httpd-tools', + 'make', + 'openssl', + 'python3-dialog', + 'python3-dotenv', + 'xz', + ] + ) # on Linux this script requires root, or sudo, unless we're in local configuration-only mode if os.getuid() == 0: From dead7761b63cb8058be14144231d296c2a783c18 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 09:45:56 -0700 Subject: [PATCH 20/78] Revert "added python3-dotenv as an install script dependency" This reverts commit ce2b7b8ad38b4d7c6d4765b24177ef34517db455. --- scripts/install.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 3a9aa26ac..8eced2ba5 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -2339,27 +2339,9 @@ def __init__(self, orchMode, debug=False, configOnly=False): # determine packages required by Malcolm itself (not docker, those will be done later) if (self.distro == PLATFORM_LINUX_UBUNTU) or (self.distro == PLATFORM_LINUX_DEBIAN): - self.requiredPackages.extend( - [ - 'apache2-utils', - 'make', - 'openssl', - 'python3-dialog', - 'python3-dotenv', - 'xz-utils', - ] - ) + self.requiredPackages.extend(['apache2-utils', 'make', 'openssl', 'python3-dialog', 'xz-utils']) elif (self.distro == PLATFORM_LINUX_FEDORA) or (self.distro == PLATFORM_LINUX_CENTOS): - self.requiredPackages.extend( - [ - 'httpd-tools', - 'make', - 'openssl', - 'python3-dialog', - 'python3-dotenv', - 'xz', - ] - ) + self.requiredPackages.extend(['httpd-tools', 'make', 'openssl', 'python3-dialog', 'xz']) # on Linux this script requires root, or sudo, unless we're in local configuration-only mode if os.getuid() == 0: From 7591fbcf6ff0785709876aea7964840b09f9d5d7 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 09:58:46 -0700 Subject: [PATCH 21/78] some tweaks to installer --- scripts/install.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 26ca4a7c2..ed9b1b328 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -434,8 +434,10 @@ def tweak_malcolm_runtime(self, malcolm_install_path): defaultUid = '1000' defaultGid = '1000' if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): - defaultUid = str(os.stat(malcolm_install_path).st_uid) - defaultGid = str(os.stat(malcolm_install_path).st_gid) + if pathUid := os.stat(malcolm_install_path).st_uid: + defaultUid = str(pathUid) + if pathGid := os.stat(malcolm_install_path).st_gid: + defaultGid = str(pathGid) puid = defaultUid pgid = defaultGid From c60ae37100d3ec51573378fe7afe1d7a939d47f0 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 11:00:24 -0700 Subject: [PATCH 22/78] idaholab/Malcolm#310, have install.py offer to pull the docker images --- scripts/install.py | 102 +++++++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 32 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index ed9b1b328..8a480e1cf 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -235,6 +235,7 @@ def __init__(self, orchMode, debug=False, configOnly=False): self.checkPackageCmds = [] self.installPackageCmds = [] self.requiredPackages = [] + self.dockerComposeCmd = None self.pipCmd = 'pip3' if not which(self.pipCmd, debug=self.debug): @@ -302,15 +303,16 @@ def install_required_packages(self): return self.install_package(self.requiredPackages) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - def install_docker_images(self, docker_image_file): + def install_docker_images(self, docker_image_file, malcolm_install_path): result = False + composeFile = os.path.join(malcolm_install_path, 'docker-compose.yml') if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: if ( docker_image_file and os.path.isfile(docker_image_file) and InstallerYesOrNo( - f'Load Malcolm Docker images from {docker_image_file}', default=True, forceInteraction=True + f'Load Malcolm Docker images from {docker_image_file}?', default=True, forceInteraction=True ) ): ecode, out = self.run_process(['docker', 'load', '-q', '-i', docker_image_file], privileged=True) @@ -319,6 +321,31 @@ def install_docker_images(self, docker_image_file): else: eprint(f"Loading Malcolm Docker images failed: {out}") + elif ( + os.path.isfile(composeFile) + and self.dockerComposeCmd + and InstallerYesOrNo(f'Pull Malcolm Docker images?', default=True, forceInteraction=True) + ): + for priv in (False, True): + ecode, out = self.run_process( + [ + self.dockerComposeCmd, + '-f', + composeFile, + '--profile=malcolm', + 'pull', + '--quiet', + ], + privileged=priv, + ) + if ecode == 0: + break + + if ecode == 0: + result = True + else: + eprint(f"Pulling Malcolm Docker images failed: {out}") + return result # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -331,7 +358,7 @@ def install_malcolm_files(self, malcolm_install_file, default_config_dir): malcolm_install_file and os.path.isfile(malcolm_install_file) and InstallerYesOrNo( - f'Extract Malcolm runtime files from {malcolm_install_file}', default=True, forceInteraction=True + f'Extract Malcolm runtime files from {malcolm_install_file}?', default=True, forceInteraction=True ) ): # determine and create destination path for installation @@ -2593,24 +2620,28 @@ def install_docker_compose(self): if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # first see if docker compose/docker-compose is already installed and runnable # (try non-root and root) - dockerComposeCmd = ('docker', 'compose') - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) + tmpComposeCmd = ('docker', 'compose') + + for priv in (False, True): + err, out = self.run_process([tmpComposeCmd, 'version'], privileged=priv) + if err == 0: + break if err != 0: - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=True) - if err != 0: - dockerComposeCmd = 'docker-compose' - if not which(dockerComposeCmd, debug=self.debug): - if os.path.isfile('/usr/libexec/docker/cli-plugins/docker-compose'): - dockerComposeCmd = '/usr/libexec/docker/cli-plugins/docker-compose' - elif os.path.isfile('/usr/local/bin/docker-compose'): - dockerComposeCmd = '/usr/local/bin/docker-compose' - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) - if err != 0: - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=True) + tmpComposeCmd = 'docker-compose' + if not which(tmpComposeCmd, debug=self.debug): + if os.path.isfile('/usr/libexec/docker/cli-plugins/docker-compose'): + tmpComposeCmd = '/usr/libexec/docker/cli-plugins/docker-compose' + elif os.path.isfile('/usr/local/bin/docker-compose'): + tmpComposeCmd = '/usr/local/bin/docker-compose' + for priv in (False, True): + err, out = self.run_process([tmpComposeCmd, 'version'], privileged=priv) + if err == 0: + break - if (err != 0) and InstallerYesOrNo( - 'docker compose failed, attempt to install docker compose?', default=True - ): + if err == 0: + self.dockerComposeCmd = tmpComposeCmd + + elif InstallerYesOrNo('docker compose failed, attempt to install docker compose?', default=True): if InstallerYesOrNo('Install docker compose directly from docker github?', default=True): # download docker-compose from github and put it in /usr/local/bin @@ -2624,7 +2655,7 @@ def install_docker_compose(self): unames.append(out[0].lower()) if len(unames) == 2: # download docker-compose from github and save it to a temporary file - tempFileName = os.path.join(self.tempDirName, dockerComposeCmd) + tempFileName = os.path.join(self.tempDirName, tmpComposeCmd) dockerComposeUrl = f"https://github.com/docker/compose/releases/download/v{DOCKER_COMPOSE_INSTALL_VERSION}/docker-compose-{unames[0]}-{unames[1]}" if DownloadToFile(dockerComposeUrl, tempFileName, debug=self.debug): os.chmod(tempFileName, 493) # 493 = 0o755, mark as executable @@ -2634,7 +2665,7 @@ def install_docker_compose(self): ) if err == 0: eprint("Download and installation of docker-compose apparently succeeded") - dockerComposeCmd = '/usr/local/bin/docker-compose' + tmpComposeCmd = '/usr/local/bin/docker-compose' else: raise Exception(f'Error copying {tempFileName} to /usr/local/bin: {out}') @@ -2658,11 +2689,13 @@ def install_docker_compose(self): eprint(f"Install docker-compose via pip failed with {err}, {out}") # see if docker-compose is now installed and runnable (try non-root and root) - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) - if err != 0: - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=True) + for priv in (False, True): + err, out = self.run_process([tmpComposeCmd, 'version'], privileged=priv) + if err == 0: + break if err == 0: + self.dockerComposeCmd = tmpComposeCmd result = True if self.debug: eprint('docker compose succeeded') @@ -3908,8 +3941,6 @@ def main(): success = installer.install_docker_compose() if hasattr(installer, 'tweak_system_files'): success = installer.tweak_system_files() - if (orchMode is OrchestrationFramework.DOCKER_COMPOSE) and hasattr(installer, 'install_docker_images'): - success = installer.install_docker_images(imageFile) if (orchMode is OrchestrationFramework.DOCKER_COMPOSE) and hasattr(installer, 'install_malcolm_files'): success, installPath = installer.install_malcolm_files(malcolmFile, args.configDir is None) @@ -3960,12 +3991,19 @@ def main(): if args.debug: eprint(f"Malcolm installation detected at {installPath}") - if (installPath is not None) and os.path.isdir(installPath) and hasattr(installer, 'tweak_malcolm_runtime'): - installer.tweak_malcolm_runtime(installPath) - eprint(f"\nMalcolm has been installed to {installPath}. See README.md for more information.") - eprint( - f"Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in {os.path.join(installPath, 'scripts')}." - ) + if (installPath is not None) and os.path.isdir(installPath): + if hasattr(installer, 'tweak_malcolm_runtime'): + installer.tweak_malcolm_runtime(installPath) + eprint(f"\nMalcolm has been installed to {installPath}. See README.md for more information.") + eprint( + f"Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in {os.path.join(installPath, 'scripts')}." + ) + if ( + (not args.configOnly) + and (orchMode is OrchestrationFramework.DOCKER_COMPOSE) + and hasattr(installer, 'install_docker_images') + ): + success = installer.install_docker_images(imageFile, installPath) if __name__ == '__main__': From d544339938463958069526c9a371c9d3607b9392 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 11:14:23 -0700 Subject: [PATCH 23/78] idaholab/Malcolm#310, have install.py offer to pull the docker images --- docs/ubuntu-install-example.md | 8 +++++--- scripts/install.py | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 339ed20ec..e47cc726f 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -179,7 +179,7 @@ Scan extracted PE files with Capa? (y / N): y Lookup extracted file hashes with VirusTotal? (y / N): n -Download updated file scanner signatures periodically? (n/Y): y +Download updated file scanner signatures periodically? (Y / n): n Should Malcolm run and maintain an instance of NetBox, an infrastructure resource modeling tool? (y / N): n @@ -190,7 +190,9 @@ Should Malcolm capture live network traffic? 2 Specify capture interface(s) (comma-separated): eth0 -Enable dark mode for OpenSearch Dashboards? (n/Y): y +Enable dark mode for OpenSearch Dashboards? (Y / n): y + +Pull Malcolm Docker images (y / N): y Malcolm has been installed to /home/user/Malcolm. See README.md for more information. Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in /home/user/Malcolm/scripts. @@ -227,7 +229,7 @@ As an alternative to manually copying the files to the sensor, Malcolm can facil In this example, rather than [building Malcolm from scratch](development.md#Build), images may be pulled from [GitHub](https://github.com/orgs/idaholab/packages?repo_name=Malcolm): ``` -user@host:~/Malcolm$ docker compose pull +user@host:~/Malcolm$ docker compose --profile=malcolm pull Pulling api ... done Pulling arkime ... done Pulling dashboards ... done diff --git a/scripts/install.py b/scripts/install.py index 8a480e1cf..06ec0d891 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -3994,10 +3994,7 @@ def main(): if (installPath is not None) and os.path.isdir(installPath): if hasattr(installer, 'tweak_malcolm_runtime'): installer.tweak_malcolm_runtime(installPath) - eprint(f"\nMalcolm has been installed to {installPath}. See README.md for more information.") - eprint( - f"Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in {os.path.join(installPath, 'scripts')}." - ) + if ( (not args.configOnly) and (orchMode is OrchestrationFramework.DOCKER_COMPOSE) @@ -4005,6 +4002,10 @@ def main(): ): success = installer.install_docker_images(imageFile, installPath) + InstallerDisplayMessage( + f"Malcolm has been installed to {installPath}. See README.md for more information.\nScripts for starting and stopping Malcolm and changing authentication-related settings can be found in {os.path.join(installPath, 'scripts')}." + ) + if __name__ == '__main__': main() From 0bf719de72785797893779d82e5f4a04fac80565 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 11:17:35 -0700 Subject: [PATCH 24/78] idaholab/Malcolm#310, have install.py offer to pull the docker images --- scripts/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.py b/scripts/install.py index 06ec0d891..2cf62d771 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -324,7 +324,7 @@ def install_docker_images(self, docker_image_file, malcolm_install_path): elif ( os.path.isfile(composeFile) and self.dockerComposeCmd - and InstallerYesOrNo(f'Pull Malcolm Docker images?', default=True, forceInteraction=True) + and InstallerYesOrNo(f'Pull Malcolm Docker images?', default=False, forceInteraction=False) ): for priv in (False, True): ecode, out = self.run_process( From 47f770f4f7323f86ed3bb0c3e8a8284e42a8340e Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 12:26:50 -0700 Subject: [PATCH 25/78] work in progress for idaholab/Malcolm#281, clean up env. variable names for arkime live capture mode --- Dockerfiles/arkime.Dockerfile | 6 --- Dockerfiles/filebeat.Dockerfile | 2 + Dockerfiles/suricata.Dockerfile | 3 ++ Dockerfiles/zeek.Dockerfile | 2 + arkime/scripts/live_capture.sh | 8 ++-- arkime/scripts/viewer_service.sh | 4 +- arkime/supervisord.conf | 1 - config/arkime-live.env.example | 6 +-- config/upload-common.env.example | 3 -- docs/malcolm-config.md | 1 - scripts/install.py | 68 +++++++++++++++++++------------- shared/bin/pcap_processor.py | 2 +- 12 files changed, 57 insertions(+), 49 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 869619344..c8e4544ab 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -122,8 +122,6 @@ ARG ARKIME_VIEWER_PORT=8005 ARG MANAGE_PCAP_FILES=false ARG ARKIME_PCAP_PROCESSOR=true ARG ARKIME_LIVE_CAPTURE=false -ARG ARKIME_LIVE_NODE_NAME=malcolm -ARG ARKIME_LIVE_NODE_HOST= ARG ARKIME_ROTATED_PCAP=true ARG ARKIME_COMPRESSION_TYPE=none ARG ARKIME_COMPRESSION_LEVEL=0 @@ -133,7 +131,6 @@ ARG AUTO_TAG=true ARG PCAP_PIPELINE_VERBOSITY="" ARG PCAP_MONITOR_HOST=pcap-monitor ARG PCAP_NODE_NAME=malcolm -ARG PCAP_NODE_HOST= ARG MAXMIND_GEOIP_DB_LICENSE_KEY="" # Declare envs vars for each arg @@ -151,8 +148,6 @@ ENV ARKIME_AUTO_ANALYZE_PCAP_THREADS $ARKIME_AUTO_ANALYZE_PCAP_THREADS ENV ARKIME_PACKET_THREADS $ARKIME_PACKET_THREADS ENV ARKIME_PCAP_PROCESSOR $ARKIME_PCAP_PROCESSOR ENV ARKIME_LIVE_CAPTURE $ARKIME_LIVE_CAPTURE -ENV ARKIME_LIVE_NODE_NAME $ARKIME_LIVE_NODE_NAME -ENV ARKIME_LIVE_NODE_HOST $ARKIME_LIVE_NODE_HOST ENV ARKIME_COMPRESSION_TYPE $ARKIME_COMPRESSION_TYPE ENV ARKIME_COMPRESSION_LEVEL $ARKIME_COMPRESSION_LEVEL ENV ARKIME_ROTATED_PCAP $ARKIME_ROTATED_PCAP @@ -165,7 +160,6 @@ ENV AUTO_TAG $AUTO_TAG ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST ENV PCAP_NODE_NAME $PCAP_NODE_NAME -ENV PCAP_NODE_HOST $PCAP_NODE_HOST COPY --from=build $ARKIME_DIR $ARKIME_DIR diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index e11a366af..37226950b 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -60,6 +60,7 @@ ARG FILEBEAT_TCP_PARSE_SOURCE_FIELD="message" ARG FILEBEAT_TCP_PARSE_TARGET_FIELD="" ARG FILEBEAT_TCP_PARSE_DROP_FIELD="" ARG FILEBEAT_TCP_TAG="_malcolm_beats" +ARG PCAP_NODE_NAME=malcolm ENV SUPERCRONIC_VERSION "0.2.28" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" @@ -163,6 +164,7 @@ ENV FILEBEAT_TCP_PARSE_DROP_FIELD $FILEBEAT_TCP_PARSE_DROP_FIELD ENV FILEBEAT_TCP_TAG $FILEBEAT_TCP_TAG ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json" ENV FILEBEAT_ZEEK_DIR "/zeek/" +ENV PCAP_NODE_NAME $PCAP_NODE_NAME VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data", "/usr/share/filebeat-tcp/data"] diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 53306d55f..a835601bf 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -160,6 +160,7 @@ ARG SURICATA_ROTATED_PCAP=false ARG PCAP_IFACE=lo ARG PCAP_IFACE_TWEAK=false ARG PCAP_FILTER= +ARG PCAP_NODE_NAME=malcolm ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST @@ -178,6 +179,8 @@ ENV SURICATA_ROTATED_PCAP $SURICATA_ROTATED_PCAP ENV PCAP_IFACE $PCAP_IFACE ENV PCAP_IFACE_TWEAK $PCAP_IFACE_TWEAK ENV PCAP_FILTER $PCAP_FILTER +ENV PCAP_NODE_NAME $PCAP_NODE_NAME + ENV PUSER_CHOWN "$SURICATA_CONFIG_DIR;$SURICATA_MANAGED_DIR;$SURICATA_LOG_DIR;$SURICATA_RUN_DIR" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 7440a3327..716d6fff8 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -276,6 +276,7 @@ ARG ZEEK_ROTATED_PCAP=false ARG PCAP_IFACE=lo ARG PCAP_IFACE_TWEAK=false ARG PCAP_FILTER= +ARG PCAP_NODE_NAME=malcolm ENV AUTO_TAG $AUTO_TAG ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR @@ -295,6 +296,7 @@ ENV ZEEK_ROTATED_PCAP $ZEEK_ROTATED_PCAP ENV PCAP_IFACE $PCAP_IFACE ENV PCAP_IFACE_TWEAK $PCAP_IFACE_TWEAK ENV PCAP_FILTER $PCAP_FILTER +ENV PCAP_NODE_NAME $PCAP_NODE_NAME # environment variables for zeek runtime tweaks (used in local.zeek) ARG ZEEK_DISABLE_HASH_ALL_FILES= diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh index d7b9a897d..85a4700f9 100755 --- a/arkime/scripts/live_capture.sh +++ b/arkime/scripts/live_capture.sh @@ -8,8 +8,8 @@ KEY_FILE="${ARKIME_DIR}"/etc/viewer.key ARKIME_PACKET_THREADS=${ARKIME_PACKET_THREADS:-1} PUSER=${PUSER:-"arkime"} PGROUP=${PGROUP:-"arkime"} -ARKIME_LIVE_NODE_NAME=${ARKIME_LIVE_NODE_NAME:-"malcolm"} -ARKIME_LIVE_NODE_HOST=${ARKIME_LIVE_NODE_HOST:-""} +NODE_NAME=${PCAP_NODE_NAME:-"malcolm"}-live +NODE_HOST=${ARKIME_LIVE_NODE_HOST:-""} OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} @@ -69,5 +69,5 @@ echo -o dropGroup=${PGROUP} \ -o ecsEventProvider=arkime \ -o ecsEventDataset=session \ - --node "${ARKIME_LIVE_NODE_NAME}" \ - --host "${ARKIME_LIVE_NODE_HOST}" + --node "${NODE_NAME}" \ + --host "${NODE_HOST}" diff --git a/arkime/scripts/viewer_service.sh b/arkime/scripts/viewer_service.sh index 0c71a04ef..00314c460 100755 --- a/arkime/scripts/viewer_service.sh +++ b/arkime/scripts/viewer_service.sh @@ -2,12 +2,14 @@ # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. +[[ "${ARKIME_LIVE_CAPTURE:-false}" == "true" ]] && LIVE_NODE_FLAG=-live || LIVE_NODE_FLAG= + while true; do if [[ -f /var/run/arkime/initialized && "$VIEWER" == "on" ]]; then echo "Launch viewer..." rm -f $ARKIME_DIR/logs/viewer* pushd $ARKIME_DIR/viewer >/dev/null 2>&1 - $ARKIME_DIR/bin/node viewer.js --insecure -n "${PCAP_NODE_NAME:-malcolm}" -c $ARKIME_DIR/etc/config.ini + $ARKIME_DIR/bin/node viewer.js --insecure -n "${PCAP_NODE_NAME:-malcolm}${LIVE_NODE_FLAG}" -c $ARKIME_DIR/etc/config.ini popd >/dev/null 2>&1 fi sleep 5 diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 101c1611a..532275976 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -66,7 +66,6 @@ command=python3 /opt/pcap_arkime_processor.py %(ENV_PCAP_PIPELINE_VERBOSITY)s --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --host "%(ENV_PCAP_NODE_HOST)s" --arkime "%(ENV_ARKIME_DIR)s"/bin/capture --autoarkime "%(ENV_ARKIME_AUTO_ANALYZE_PCAP_FILES)s" --forcearkime "%(ENV_ARKIME_ROTATED_PCAP)s" diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example index bfd8e6656..9dd7cd303 100644 --- a/config/arkime-live.env.example +++ b/config/arkime-live.env.example @@ -5,11 +5,9 @@ ARKIME_COMPRESSION_TYPE=none ARKIME_COMPRESSION_LEVEL=0 ARKIME_PACKET_THREADS=1 -# The node name (e.g., the hostname of this machine running Malcolm) to associate with -# network traffic metadata -ARKIME_LIVE_NODE_NAME=malcolm # The node host (e.g., the IP address of the machine running Malcolm) to associate with -# network traffic metadata (optional, defaults to PCAP_NODE_NAME if unspecified) +# network traffic metadata when ARKIME_LIVE_CAPTURE is true +# (optional, defaults to PCAP_NODE_NAME if unspecified) ARKIME_LIVE_NODE_HOST= ARKIME_PCAP_PROCESSOR=false \ No newline at end of file diff --git a/config/upload-common.env.example b/config/upload-common.env.example index 17a89fd95..ad55df213 100644 --- a/config/upload-common.env.example +++ b/config/upload-common.env.example @@ -4,9 +4,6 @@ AUTO_TAG=true # The node name (e.g., the hostname of this machine running Malcolm) to associate with # network traffic metadata PCAP_NODE_NAME=malcolm -# The node host (e.g., the IP address of the machine running Malcolm) to associate with -# network traffic metadata (optional, defaults to PCAP_NODE_NAME if unspecified) -PCAP_NODE_HOST= # Verbosity flag for pcap pipeline debugging (e.g., -v, -vv, -vvv, etc.) PCAP_PIPELINE_VERBOSITY= # Whether or not PCAP files extant in ./pcap/ will be ignored on startup diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 9a02bcd2d..2df1292a4 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -74,7 +74,6 @@ Although the configuration script automates many of the following configuration * **`upload-common.env`** - settings for dealing with PCAP files [uploaded](upload.md#Upload) to Malcolm for analysis - `AUTO_TAG` – if set to `true`, Malcolm will automatically create Arkime sessions and Zeek logs with tags based on the filename, as described in [Tagging](upload.md#Tagging) (default `true`) - `PCAP_NODE_NAME` - specifies the node name to associate with network traffic metadata - - `PCAP_NODE_HOST` - specifies the node host or IP address to associate with network traffic metadata (optional, defaults to the value of `PCAP_NODE_NAME`; generally this value *does not* need to be specified) * **`zeek.env`**, **`zeek-secret.env`**, **`zeek-live.env`** and **`zeek-offline.env`** - settings for [Zeek](https://www.zeek.org/index.html) and for scanning [extracted files](file-scanning.md#ZeekFileExtraction) Zeek observes in network traffic - `EXTRACTED_FILE_CAPA_VERBOSE` – if set to `true`, all Capa rule hits will be logged; otherwise (`false`) only [MITRE ATT&CK® technique](https://attack.mitre.org/techniques) classifications will be logged - `EXTRACTED_FILE_ENABLE_CAPA` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) determined to be PE (portable executable) files will be scanned with [Capa](https://github.com/fireeye/capa) diff --git a/scripts/install.py b/scripts/install.py index 0b362ab86..ea19fd5c2 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -499,7 +499,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): f'Enter the node name to associate with network traffic metadata', default=args.pcapNodeName, ) - pcapNodeHost = '' if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # guestimate how much memory we should use based on total system memory @@ -655,18 +654,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): f'Enter Logstash host and port (e.g., 192.168.1.123:5044)', default=args.logstashHost, ) - pcapNodeHost = InstallerAskForString( - f"Enter this node's hostname or IP to associate with network traffic metadata", - default=args.pcapNodeHost, - ) - if not pcapNodeHost and not InstallerYesOrNo( - f'Node hostname or IP is required for Arkime session retrieval under the {malcolmProfile} profile. Are you sure?', - default=False, - ): - pcapNodeHost = InstallerAskForString( - f"Enter this node's hostname or IP to associate with network traffic metadata", - default=args.pcapNodeHost, - ) if (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( 'Forward Logstash logs to a secondary remote document store?', @@ -1310,6 +1297,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): pcapNetSniff = False pcapTcpDump = False liveArkime = False + liveArkimeNodeHost = '' liveZeek = False liveSuricata = False pcapIface = 'lo' @@ -1381,6 +1369,20 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'Specify capture interface(s) (comma-separated)', default=args.pcapIface ) + if liveArkime: + liveArkimeNodeHost = InstallerAskForString( + f"Enter this node's hostname or IP to associate with network traffic metadata", + default=args.liveArkimeNodeHost, + ) + if not liveArkimeNodeHost and not InstallerYesOrNo( + f'With live Arkime capture node hostname or IP is required for viewer session retrieval. Are you sure?', + default=False, + ): + liveArkimeNodeHost = InstallerAskForString( + f"Enter this node's hostname or IP to associate with network traffic metadata", + default=args.liveArkimeNodeHost, + ) + if ( (malcolmProfile == PROFILE_HEDGEHOG) and (not pcapNetSniff) @@ -1439,6 +1441,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'ARKIME_LIVE_CAPTURE', TrueOrFalseNoQuote(liveArkime), ), + # capture source "node host" for live Arkime capture + EnvValue( + os.path.join(args.configDir, 'arkime-live.env'), + 'ARKIME_LIVE_NODE_HOST', + liveArkimeNodeHost, + ), # rotated captured PCAP analysis with Arkime (not live capture) EnvValue( os.path.join(args.configDir, 'arkime-offline.env'), @@ -1732,12 +1740,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'PCAP_NODE_NAME', pcapNodeName, ), - # capture source "node host" for locally processed PCAP files - EnvValue( - os.path.join(args.configDir, 'upload-common.env'), - 'PCAP_NODE_HOST', - pcapNodeHost, - ), # zeek file extraction mode EnvValue( os.path.join(args.configDir, 'zeek.env'), @@ -3834,6 +3836,25 @@ def main(): default=False, help=f"Capture live network traffic with Arkime capture (not available with --opensearch {DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]})", ) + captureArgGroup.add_argument( + '--live-capture-arkime', + dest='liveArkime', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help=f"Capture live network traffic with Arkime capture (not available with --opensearch {DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]})", + ) + captureArgGroup.add_argument( + '--live-capture-arkime-node-host', + dest='liveArkimeNodeHost', + required=False, + metavar='', + type=str, + default='', + help='The node hostname or IP address to associate with live network traffic observed by Arkime capture', + ) captureArgGroup.add_argument( '--live-capture-netsniff', dest='pcapNetSniff', @@ -3883,15 +3904,6 @@ def main(): default=os.getenv('HOSTNAME', os.getenv('COMPUTERNAME', platform.node())).split('.')[0], help='The node name to associate with network traffic metadata', ) - captureArgGroup.add_argument( - '--node-host', - dest='pcapNodeHost', - required=False, - metavar='', - type=str, - default='', - help='The node hostname or IP address to associate with network traffic metadata', - ) try: parser.error = parser.exit diff --git a/shared/bin/pcap_processor.py b/shared/bin/pcap_processor.py index e4af20af8..a27029703 100755 --- a/shared/bin/pcap_processor.py +++ b/shared/bin/pcap_processor.py @@ -542,7 +542,7 @@ def main(): help="PCAP source node host (for Arkime viewer reachback)", metavar='', type=str, - default=os.getenv('PCAP_NODE_HOST', ''), + default='', ) requiredNamed = parser.add_argument_group('required arguments') requiredNamed.add_argument( From d4175c644f5423153473e3cb2e1f167c12aac560 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 13:23:49 -0700 Subject: [PATCH 26/78] idaholab/Malcolm#281 --- scripts/install.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index ea19fd5c2..5949551be 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -3836,16 +3836,6 @@ def main(): default=False, help=f"Capture live network traffic with Arkime capture (not available with --opensearch {DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]})", ) - captureArgGroup.add_argument( - '--live-capture-arkime', - dest='liveArkime', - type=str2bool, - metavar="true|false", - nargs='?', - const=True, - default=False, - help=f"Capture live network traffic with Arkime capture (not available with --opensearch {DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]})", - ) captureArgGroup.add_argument( '--live-capture-arkime-node-host', dest='liveArkimeNodeHost', From 66e1525aa6000dd731e480b53cd85f569c4ed18d Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 13:35:58 -0700 Subject: [PATCH 27/78] idaholab/Malcolm#281 --- docker-compose-standalone.yml | 28 ++++++++++++++++++++++++++++ docker-compose.yml | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 9862981e9..7f44387d9 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -210,6 +210,7 @@ services: - ./config/upload-common.env - ./config/auth.env - ./config/arkime.env + - ./config/arkime-offline.env - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' @@ -225,6 +226,33 @@ services: timeout: 30s retries: 3 start_period: 210s + arkime-live: + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 + profiles: ["malcolm", "hedgehog"] + logging: *default-logging + restart: "no" + stdin_open: false + tty: true + network_mode: host + cap_add: + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets + - NET_ADMIN + - NET_RAW + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE + env_file: + - ./config.elastic/process.env + - ./config.elastic/ssl.env + - ./config.elastic/opensearch.env + - ./config.elastic/upload-common.env + - ./config.elastic/auth.env + - ./config.elastic/arkime.env + - ./config.elastic/arkime-live.env + - ./config.elastic/arkime-secret.env + volumes: + - ./nginx/ca-trust:/var/local/ca-trust:ro + - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./pcap:/data/pcap zeek: image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] diff --git a/docker-compose.yml b/docker-compose.yml index 8e219ca92..0fd1c7759 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -250,6 +250,39 @@ services: timeout: 30s retries: 3 start_period: 210s + arkime-live: + build: + context: . + dockerfile: Dockerfiles/arkime.Dockerfile + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 + profiles: ["malcolm", "hedgehog"] + logging: *default-logging + restart: "no" + stdin_open: false + tty: true + network_mode: host + cap_add: + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets + - NET_ADMIN + - NET_RAW + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE + env_file: + - ./config.elastic/process.env + - ./config.elastic/ssl.env + - ./config.elastic/opensearch.env + - ./config.elastic/upload-common.env + - ./config.elastic/auth.env + - ./config.elastic/arkime.env + - ./config.elastic/arkime-live.env + - ./config.elastic/arkime-secret.env + volumes: + - ./nginx/ca-trust:/var/local/ca-trust:ro + - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./pcap:/data/pcap + - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro + - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro + - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro zeek: build: context: . From 1c13f59d17ac16a9042e7cf32f1539292a2444f9 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 13:43:54 -0700 Subject: [PATCH 28/78] idaholab/Malcolm#281 --- config/arkime-live.env.example | 4 +++- config/arkime-offline.env.example | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example index 9dd7cd303..7b7273709 100644 --- a/config/arkime-live.env.example +++ b/config/arkime-live.env.example @@ -10,4 +10,6 @@ ARKIME_PACKET_THREADS=1 # (optional, defaults to PCAP_NODE_NAME if unspecified) ARKIME_LIVE_NODE_HOST= -ARKIME_PCAP_PROCESSOR=false \ No newline at end of file +ARKIME_PCAP_PROCESSOR=false +VIEWER=on +WISE=off \ No newline at end of file diff --git a/config/arkime-offline.env.example b/config/arkime-offline.env.example index 73e441806..aa53cdc87 100644 --- a/config/arkime-offline.env.example +++ b/config/arkime-offline.env.example @@ -9,4 +9,6 @@ ARKIME_AUTO_ANALYZE_PCAP_THREADS=1 # Arkime will see duplicate traffic. ARKIME_ROTATED_PCAP=true -ARKIME_PCAP_PROCESSOR=true \ No newline at end of file +ARKIME_PCAP_PROCESSOR=true +VIEWER=on +WISE=on \ No newline at end of file From 0c9874aaca9237d8acad2874a0a70956702b816c Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 13:57:51 -0700 Subject: [PATCH 29/78] for idaholab/Malcolm#282, reviewing capabilities for containers --- Dockerfiles/pcap-capture.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- Dockerfiles/zeek.Dockerfile | 4 ++-- .../docker-compose.yml | 6 ------ docs/hedgehog-upgrade.md | 12 ++++++------ docs/live-analysis.md | 2 +- pcap-capture/scripts/supervisor.sh | 2 +- .../hooks/normal/0910-sensor-build.hook.chroot | 8 ++++---- suricata/scripts/docker_entrypoint.sh | 2 +- zeek/scripts/docker_entrypoint.sh | 4 ++-- 10 files changed, 19 insertions(+), 25 deletions(-) diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index c582e9e26..b13b7b009 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -87,7 +87,7 @@ RUN apt-get -q update && \ chown root:${PGROUP} /usr/bin/tcpdump && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump && \ chown root:${PGROUP} /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng && \ chmod 755 /usr/local/bin/*.sh WORKDIR "$PCAP_PATH" diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 53306d55f..a8059c56b 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -125,7 +125,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour suricata-update update --fail --verbose --etopen --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ chown root:${PGROUP} /sbin/ethtool /usr/bin/suricata && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/suricata && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 7440a3327..ab6a5565a 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -246,8 +246,8 @@ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ usermod -a -G tty ${PUSER} && \ chown root:${PGROUP} /sbin/ethtool "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats && \ touch "${SUPERCRONIC_CRONTAB}" && \ chown -R ${DEFAULT_UID}:${DEFAULT_GID} "${ZEEK_DIR}"/share/zeek/site/intel "${SUPERCRONIC_CRONTAB}" && \ ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_zeek_processor.py && \ diff --git a/arkime/arkime_regression_test_harness/docker-compose.yml b/arkime/arkime_regression_test_harness/docker-compose.yml index c205d0a7a..1e112a7a6 100644 --- a/arkime/arkime_regression_test_harness/docker-compose.yml +++ b/arkime/arkime_regression_test_harness/docker-compose.yml @@ -13,12 +13,6 @@ services: cluster.routing.allocation.node_initial_primaries_recoveries : 8 expose: - 9200 - ulimits: - memlock: - soft: -1 - hard: -1 - cap_add: - - IPC_LOCK healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] interval: 30s diff --git a/docs/hedgehog-upgrade.md b/docs/hedgehog-upgrade.md index 6c7e69e29..f106bb030 100644 --- a/docs/hedgehog-upgrade.md +++ b/docs/hedgehog-upgrade.md @@ -208,9 +208,9 @@ commands: ``` chown root:netdev /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng chown root:netdev /opt/zeek/bin/zeek && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/zeek/bin/zeek + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/zeek/bin/zeek chown root:netdev /sbin/ethtool && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool chown root:netdev /opt/zeek/bin/capstats && \ @@ -218,7 +218,7 @@ chown root:netdev /opt/zeek/bin/capstats && \ chown root:netdev /usr/bin/tcpdump && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump chown root:netdev /opt/arkime/bin/capture && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/arkime/bin/capture + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture ln -s -f /opt/zeek/bin/zeek /usr/local/bin/ ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/ @@ -233,9 +233,9 @@ example: ``` root@hedgehog:/tmp# chown root:netdev /usr/sbin/netsniff-ng && \ -> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng +> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng root@hedgehog:/tmp# chown root:netdev /opt/zeek/bin/zeek && \ -> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/zeek/bin/zeek +> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/zeek/bin/zeek root@hedgehog:/tmp# chown root:netdev /sbin/ethtool && \ > setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool root@hedgehog:/tmp# chown root:netdev /opt/zeek/bin/capstats && \ @@ -243,7 +243,7 @@ root@hedgehog:/tmp# chown root:netdev /opt/zeek/bin/capstats && \ root@hedgehog:/tmp# chown root:netdev /usr/bin/tcpdump && \ > setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump root@hedgehog:/tmp# chown root:netdev /opt/arkime/bin/capture && \ -> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/arkime/bin/capture +> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture root@hedgehog:/tmp# ln -s -f /opt/zeek/bin/zeek /usr/local/bin/ root@hedgehog:/tmp# ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/ root@hedgehog:/tmp# ln -s -f /usr/bin/tcpdump /usr/local/bin/ diff --git a/docs/live-analysis.md b/docs/live-analysis.md index 4e8dd7453..2095b8649 100644 --- a/docs/live-analysis.md +++ b/docs/live-analysis.md @@ -18,7 +18,7 @@ Please see the [Hedgehog Linux README](hedgehog.md) for more information. ## Monitoring local network interfaces -Malcolm's `pcap-capture`, `suricata-live` and `zeek-live` containers can monitor one or more local network interfaces, specified by the `PCAP_IFACE` environment variable in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). These containers are started with additional privileges (`IPC_LOCK`, `NET_ADMIN`, `NET_RAW`, and `SYS_ADMIN`) to allow opening network interfaces in promiscuous mode for capture. +Malcolm's `pcap-capture`, `suricata-live` and `zeek-live` containers can monitor one or more local network interfaces, specified by the `PCAP_IFACE` environment variable in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). These containers are started with additional privileges (`IPC_LOCK`, `NET_ADMIN`, `NET_RAW`) to allow opening network interfaces in promiscuous mode for capture. The instances of Zeek and Suricata (in the `suricata-live` and `zeek-live` containers when the `SURICATA_LIVE_CAPTURE` and `ZEEK_LIVE_CAPTURE` [environment variables](malcolm-config.md#MalcolmConfigEnvVars) are set to `true`, respectively) analyze traffic on-the-fly and generate log files containing network session metadata. These log files are in turn scanned by [Filebeat](https://www.elastic.co/products/beats/filebeat) and forwarded to [Logstash](https://www.elastic.co/products/logstash) for enrichment and indexing into the [OpenSearch](https://opensearch.org/) document store. diff --git a/pcap-capture/scripts/supervisor.sh b/pcap-capture/scripts/supervisor.sh index 618e775ff..07e1096e3 100755 --- a/pcap-capture/scripts/supervisor.sh +++ b/pcap-capture/scripts/supervisor.sh @@ -14,7 +14,7 @@ function join_by { local IFS="$1"; shift; echo "$*"; } function SetCaptureCapabilities() { setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump || true - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng || true } # Create config files for each capture interface for the various capture programs (tcpdump, netsniff) diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 31804abf0..514a0a6d9 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -224,9 +224,9 @@ freshclam --stdout --quiet --no-warnings # set up capabilities for network-related tools chown root:netdev /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng chown root:netdev "${ZEEK_DIR}"/bin/zeek && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek chown root:netdev /sbin/ethtool && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool chown root:netdev "${ZEEK_DIR}"/bin/capstats && \ @@ -234,9 +234,9 @@ chown root:netdev "${ZEEK_DIR}"/bin/capstats && \ chown root:netdev /usr/bin/tcpdump && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump chown root:netdev /usr/bin/suricata && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/suricata chown root:netdev /opt/arkime/bin/capture && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/arkime/bin/capture + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture ln -s -f "${ZEEK_DIR}"/bin/zeek /usr/local/bin/ ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/ diff --git a/suricata/scripts/docker_entrypoint.sh b/suricata/scripts/docker_entrypoint.sh index a3d3e0200..caf3b3eff 100755 --- a/suricata/scripts/docker_entrypoint.sh +++ b/suricata/scripts/docker_entrypoint.sh @@ -2,7 +2,7 @@ # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/suricata || true # modify suricata.yaml according to environment variables (as non-root) if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh index 2d9d4f972..5c3d7932a 100755 --- a/zeek/scripts/docker_entrypoint.sh +++ b/zeek/scripts/docker_entrypoint.sh @@ -4,8 +4,8 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek || true -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats || true if [[ "${ZEEK_LIVE_CAPTURE:-false}" != "true" ]] && [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then sleep 15 # give the "live" instance, if there is one, a chance to go first From 2d77b8f547f3c48453a98630cc9ec7ce65f50e22 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 14:57:42 -0700 Subject: [PATCH 30/78] for idaholab/Malcolm#282, reviewing capabilities for containers --- Dockerfiles/suricata.Dockerfile | 3 ++- docker-compose-standalone.yml | 8 ++++++++ docker-compose.yml | 8 ++++++++ kubernetes/11-suricata.yml | 4 ++++ kubernetes/22-suricata-live.yml | 4 ++++ .../config/hooks/normal/0910-sensor-build.hook.chroot | 2 +- suricata/scripts/docker_entrypoint.sh | 2 +- 7 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index a8059c56b..ae4539970 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -28,6 +28,7 @@ ENV PGROUP "suricata" # docker-uid-gid-setup.sh will cause them to be lost, so we need # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false +ENV PUSER_RLIMIT_UNLOCK true ENV SUPERCRONIC_VERSION "0.2.28" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" @@ -125,7 +126,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour suricata-update update --fail --verbose --etopen --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ chown root:${PGROUP} /sbin/ethtool /usr/bin/suricata && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/suricata && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 2a729d9ca..182fc968c 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -303,6 +303,10 @@ services: networks: - default cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: @@ -332,6 +336,10 @@ services: tty: true network_mode: host cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/docker-compose.yml b/docker-compose.yml index 9ffc6a6b7..499fd6570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -339,6 +339,10 @@ services: networks: - default cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling - SYS_NICE env_file: @@ -371,6 +375,10 @@ services: tty: true network_mode: host cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index 574da3fea..eb9cf0f0f 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -23,6 +23,10 @@ spec: securityContext: capabilities: add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE envFrom: diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index a37847fc2..38f2b1655 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -23,6 +23,10 @@ spec: securityContext: capabilities: add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 514a0a6d9..144e70778 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -234,7 +234,7 @@ chown root:netdev "${ZEEK_DIR}"/bin/capstats && \ chown root:netdev /usr/bin/tcpdump && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump chown root:netdev /usr/bin/suricata && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/suricata + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata chown root:netdev /opt/arkime/bin/capture && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture diff --git a/suricata/scripts/docker_entrypoint.sh b/suricata/scripts/docker_entrypoint.sh index caf3b3eff..a3d3e0200 100755 --- a/suricata/scripts/docker_entrypoint.sh +++ b/suricata/scripts/docker_entrypoint.sh @@ -2,7 +2,7 @@ # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/suricata || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata || true # modify suricata.yaml according to environment variables (as non-root) if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then From 692f6586b6f6154e0d2d5ff1c9e16e05428b68ce Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 15:11:44 -0700 Subject: [PATCH 31/78] work in progress for idaholab/Malcolm#281, clean up env. variable names for arkime live capture mode --- Dockerfiles/arkime.Dockerfile | 2 +- arkime/scripts/docker_entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index c8e4544ab..e329fac9b 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -251,7 +251,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ chmod u+s $ARKIME_DIR/bin/capture && \ chown root:${PGROUP} /sbin/ethtool $ARKIME_DIR/bin/capture && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' $ARKIME_DIR/bin/capture && \ mkdir -p /var/run/arkime && \ chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime #Update Path diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 8cd9887e2..395db3527 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -92,7 +92,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ARKIME_DIR}"/bin/capture || true fi # comment-out features that are unused in hedgehog run profile mode and in live-capture mode From 8c0975f35ca6f23563cfd7fff75e27dee4d24035 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 15:31:31 -0700 Subject: [PATCH 32/78] idaholab/Malcolm#281 --- Dockerfiles/arkime.Dockerfile | 3 ++- arkime/scripts/docker_entrypoint.sh | 2 +- docker-compose-standalone.yml | 5 +++++ docker-compose.yml | 5 +++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index e329fac9b..57c96f82c 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -99,6 +99,7 @@ ENV PGROUP "arkime" # docker-uid-gid-setup.sh will cause them to be lost, so we need # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false +ENV PUSER_RLIMIT_UNLOCK true ENV DEBIAN_FRONTEND noninteractive ENV TERM xterm @@ -251,7 +252,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ chmod u+s $ARKIME_DIR/bin/capture && \ chown root:${PGROUP} /sbin/ethtool $ARKIME_DIR/bin/capture && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' $ARKIME_DIR/bin/capture && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ mkdir -p /var/run/arkime && \ chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime #Update Path diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 395db3527..8cd9887e2 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -92,7 +92,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ARKIME_DIR}"/bin/capture || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true fi # comment-out features that are unused in hedgehog run profile mode and in live-capture mode diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 961c8aef1..9caad00cb 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -235,6 +235,10 @@ services: tty: true network_mode: host cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW @@ -245,6 +249,7 @@ services: - ./config.elastic/ssl.env - ./config.elastic/opensearch.env - ./config.elastic/upload-common.env + - ./config.seth/pcap-capture.env - ./config.elastic/auth.env - ./config.elastic/arkime.env - ./config.elastic/arkime-live.env diff --git a/docker-compose.yml b/docker-compose.yml index 6012565c6..1404fdc6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -262,6 +262,10 @@ services: tty: true network_mode: host cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW @@ -272,6 +276,7 @@ services: - ./config.elastic/ssl.env - ./config.elastic/opensearch.env - ./config.elastic/upload-common.env + - ./config.seth/pcap-capture.env - ./config.elastic/auth.env - ./config.elastic/arkime.env - ./config.elastic/arkime-live.env From 676400f69b2c95f41b12a2412369514ac7379507 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 7 Dec 2023 15:52:10 -0700 Subject: [PATCH 33/78] idaholab/Malcolm#281 --- arkime/scripts/docker_entrypoint.sh | 1 + arkime/scripts/live_capture.sh | 2 +- arkime/supervisord.conf | 2 +- malcolm-iso/build.sh | 1 + pcap/arkime-live/.gitignore | 3 +++ scripts/control.py | 4 ++-- scripts/install.py | 1 + scripts/malcolm_appliance_packager.sh | 1 + 8 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 pcap/arkime-live/.gitignore diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 8cd9887e2..70fc3974b 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -56,6 +56,7 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(pcapDir=\).*/\1\/data\/pcap\/arkime-live/" "${ARKIME_CONFIG_FILE}" # capture interface(s) if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh index 85a4700f9..bfb32604b 100755 --- a/arkime/scripts/live_capture.sh +++ b/arkime/scripts/live_capture.sh @@ -63,7 +63,7 @@ echo "${ARKIME_DIR}"/bin/capture --insecure \ -c "${ARKIME_DIR}"/etc/config.ini \ - -o pcapDir=/data/pcap/processed \ + -o pcapDir=/data/pcap/arkime-live \ -o packetThreads=${ARKIME_PACKET_THREADS} \ -o dropUser=${PUSER} \ -o dropGroup=${PGROUP} \ diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 532275976..cb42cf4ce 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -96,5 +96,5 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true user=root ; fear not, capture will dropUser/dropGroup to PUSER -directory=/data/pcap/processed +directory=/data/pcap/arkime-live diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 99f69b9df..cf5a39973 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -106,6 +106,7 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/kubernetes/" mkdir -p "$MALCOLM_DEST_DIR/opensearch-backup/" mkdir -p "$MALCOLM_DEST_DIR/opensearch/nodes/" + mkdir -p "$MALCOLM_DEST_DIR/pcap/arkime-live/" mkdir -p "$MALCOLM_DEST_DIR/pcap/processed/" mkdir -p "$MALCOLM_DEST_DIR/pcap/upload/tmp/spool/" mkdir -p "$MALCOLM_DEST_DIR/pcap/upload/variants/" diff --git a/pcap/arkime-live/.gitignore b/pcap/arkime-live/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/pcap/arkime-live/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/scripts/control.py b/scripts/control.py index abc44234b..3b90076a2 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -849,7 +849,7 @@ def stop(wipe=False): BoundPath("netbox-postgres", "/var/lib/postgresql/data", True, None, ["."]), BoundPath("netbox-redis", "/data", True, None, ["."]), BoundPath("opensearch", "/usr/share/opensearch/data", True, ["nodes"], None), - BoundPath("pcap-monitor", "/pcap", True, ["processed", "upload"], None), + BoundPath("pcap-monitor", "/pcap", True, ["arkime-live", "processed", "upload"], None), BoundPath("suricata", "/var/log/suricata", True, None, ["."]), BoundPath( "upload", @@ -1000,7 +1000,7 @@ def start(): BoundPath("netbox-redis", "/data", False, None, None), BoundPath("opensearch", "/usr/share/opensearch/data", False, ["nodes"], None), BoundPath("opensearch", "/opt/opensearch/backup", False, None, None), - BoundPath("pcap-monitor", "/pcap", False, ["processed", "upload"], None), + BoundPath("pcap-monitor", "/pcap", False, ["arkime-live", "processed", "upload"], None), BoundPath("suricata", "/var/log/suricata", False, ["live"], None), BoundPath( "upload", diff --git a/scripts/install.py b/scripts/install.py index 5949551be..e4875f194 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -979,6 +979,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): for pathToCreate in ( indexDirFull, indexSnapshotDirFull, + os.path.join(pcapDirFull, 'arkime-live'), os.path.join(pcapDirFull, 'processed'), os.path.join(pcapDirFull, os.path.join('upload', os.path.join('tmp', 'spool'))), os.path.join(pcapDirFull, os.path.join('upload', 'variants')), diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 8a2c15142..a0360483f 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -73,6 +73,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/nginx/certs/" mkdir $VERBOSE -p "$DESTDIR/opensearch-backup/" mkdir $VERBOSE -p "$DESTDIR/opensearch/nodes/" + mkdir $VERBOSE -p "$DESTDIR/pcap/arkime-live/" mkdir $VERBOSE -p "$DESTDIR/pcap/processed/" mkdir $VERBOSE -p "$DESTDIR/pcap/upload/tmp/spool" mkdir $VERBOSE -p "$DESTDIR/pcap/upload/variants/" From 5399cef1bf1aa2e7e8d6f6f01fa53a1b18081602 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 07:43:18 -0700 Subject: [PATCH 34/78] bump elasticsearch-py to v8.11.1 --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index 19243e4ca..f76b2a87f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,5 +5,5 @@ opensearch-py==2.4.2 requests==2.31.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.11.0 +elasticsearch==8.11.1 elasticsearch-dsl==8.11.0 \ No newline at end of file From 5ba889e9fb815124732efc3f2c83ccd2b2649162 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 07:59:21 -0700 Subject: [PATCH 35/78] idaholab/Malcolm#281, don't try to surface opensearch port unless primary is opensearch-local --- scripts/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.py b/scripts/install.py index e4875f194..4547ca2ef 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1114,7 +1114,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): logstashOpen = False filebeatTcpOpen = False elif openPortsSelection == 'y': - opensearchOpen = True + opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal logstashOpen = True filebeatTcpOpen = True else: From bf648614fefcf9c5556d09f1b86d64af340e47d8 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 08:17:12 -0700 Subject: [PATCH 36/78] idaholab/Malcolm#281, arkime capture work in progress --- kubernetes/07-arkime.yml | 2 + kubernetes/21-zeek-live.yml | 3 - kubernetes/22-suricata-live.yml | 3 - kubernetes/23-arkime-live.yml | 85 +++++++++++++++++++++++++ kubernetes/{23-freq.yml => 24-freq.yml} | 0 scripts/malcolm_kubernetes.py | 2 + 6 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 kubernetes/23-arkime-live.yml rename kubernetes/{23-freq.yml => 24-freq.yml} (100%) diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index b3b01920f..bbc9bf8dd 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -51,6 +51,8 @@ spec: name: upload-common-env - configMapRef: name: arkime-env + - configMapRef: + name: arkime-offline-env - secretRef: name: arkime-secret-env env: diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index f7bde1e28..f145bca23 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -43,9 +43,6 @@ spec: name: zeek-live-env - configMapRef: name: pcap-capture-env - env: - - name: ZEEK_DISABLED - value: "true" volumeMounts: - mountPath: /var/local/ca-trust/configmap name: zeek-live-var-local-catrust-volume diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 38f2b1655..44df37747 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -45,9 +45,6 @@ spec: name: suricata-live-env - configMapRef: name: pcap-capture-env - env: - - name: SURICATA_DISABLED - value: "true" volumeMounts: - mountPath: /var/local/ca-trust/configmap name: suricata-live-var-local-catrust-volume diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml new file mode 100644 index 000000000..5e3fff84d --- /dev/null +++ b/kubernetes/23-arkime-live.yml @@ -0,0 +1,85 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: arkime-live-deployment + namespace: malcolm +spec: + selector: + matchLabels: + name: arkime-live-deployment + replicas: 1 + template: + metadata: + labels: + name: arkime-live-deployment + spec: + containers: + - name: arkime-live-container + image: ghcr.io/mmguero-dev/malcolm/arkime:23.12.1 + imagePullPolicy: Always + stdin: false + tty: true + securityContext: + capabilities: + add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets + - NET_ADMIN + - NET_RAW + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE + envFrom: + - configMapRef: + name: process-env + - configMapRef: + name: ssl-env + - configMapRef: + name: opensearch-env + - secretRef: + name: auth-env + - configMapRef: + name: upload-common-env + - configMapRef: + name: pcap-capture-env + - configMapRef: + name: arkime-env + - configMapRef: + name: arkime-live-env + - secretRef: + name: arkime-secret-env + volumeMounts: + - mountPath: /var/local/ca-trust/configmap + name: arkime-live-var-local-catrust-volume + - mountPath: /var/local/curlrc/secretmap + name: arkime-live-opensearch-curlrc-secret-volume + - mountPath: "/data/pcap" + name: arkime-live-pcap-volume + initContainers: + - name: arkime-live-dirinit-container + image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + imagePullPolicy: Always + stdin: false + tty: true + envFrom: + - configMapRef: + name: process-env + env: + - name: PUSER_MKDIR + value: "/data/pcap:arkime-live" + volumeMounts: + - name: arkime-live-pcap-volume + mountPath: "/data/pcap" + volumes: + - name: arkime-live-var-local-catrust-volume + configMap: + name: var-local-catrust + - name: arkime-live-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc + - name: arkime-live-pcap-volume + persistentVolumeClaim: + claimName: pcap-claim diff --git a/kubernetes/23-freq.yml b/kubernetes/24-freq.yml similarity index 100% rename from kubernetes/23-freq.yml rename to kubernetes/24-freq.yml diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 0bd833bcf..aa6ac8557 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -197,6 +197,7 @@ MALCOLM_PROFILES_CONTAINERS[PROFILE_MALCOLM] = [ 'api', 'arkime', + 'arkime-live', 'dashboards', 'dashboards-helper', 'filebeat', @@ -220,6 +221,7 @@ ] MALCOLM_PROFILES_CONTAINERS[PROFILE_HEDGEHOG] = [ 'arkime', + 'arkime-live', 'file-monitor', 'filebeat', 'pcap-capture', From dd9794e193c4c95061451c8b938eb5d663f8fdad Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 08:32:11 -0700 Subject: [PATCH 37/78] idaholab/Malcolm#281, arkime capture work in progress --- docs/kubernetes.md | 2 +- docs/malcolm-hedgehog-e2e-iso-install.md | 2 +- scripts/install.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 23b244c17..b1b351d50 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -375,7 +375,7 @@ Enter index threshold (e.g., 250GB, 1TB, 60%, etc.): 250G Determine oldest indices by name (instead of creation time)? (Y / n): y -Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)? (y / N): y +Should Arkime delete uploaded PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)? (y / N): y Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.): 10% diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index a537f9f65..ba5511a5f 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -177,7 +177,7 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest - Choose **Y** to proceed to the following related questions about managing the data storage used by Malcolm. - **Delete the oldest indices when the database exceeds a certain size?** - Most of the configuration around OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Snapshot Management](https://opensearch.org/docs/latest/opensearch/snapshots/sm-dashboards/) can be done in OpenSearch Dashboards. In addition to (or instead of) the OpenSearch index state management operations, Malcolm can also be configured to delete the oldest network session metadata indices when the database exceeds a certain size to prevent filling up all available storage with OpenSearch indices. - - **Should Arkime delete PCAP files based on available storage?** + - **Should Arkime delete uploaded PCAP files based on available storage?** - Answering **Y** allows Arkime to prune (delete) old PCAP files based on available disk space (see https://arkime.com/faq#pcap-deletion). - **Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)** - If [Arkime PCAP-deletion](https://arkime.com/faq#pcap-deletion) is enabled, Arkime will delete PCAP files when **free space** is lower than this value, specified as integer gigabytes (e.g., `500`) or a percentage (e.g., `10%`) diff --git a/scripts/install.py b/scripts/install.py index 4547ca2ef..6a6e37d08 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1046,7 +1046,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) or (malcolmProfile != PROFILE_MALCOLM) or InstallerYesOrNo( - 'Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', + 'Should Arkime delete uploaded PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', default=args.arkimeManagePCAP, ) ) @@ -1953,7 +1953,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): # whether or not to restart services automatically (on boot, etc.) line = f"{sectionIndents[currentSection] * 2}restart: {restartMode}" - elif currentService == 'arkime': + elif (currentService == 'arkime') or (currentService == 'arkime-live'): # stuff specifically in the arkime section if re.match(r'^\s*-.+:/data/pcap(:.+)?\s*$', line): # Arkime's reference to the PCAP directory From 0f129fee9bcdebed596edb519164d9b04b8aef82 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 09:43:57 -0700 Subject: [PATCH 38/78] work in progress for idaholab/Malcolm#281 (arkime capture) and idaholab/Malcolm#282 (container capabilities) --- Dockerfiles/arkime.Dockerfile | 6 ++---- Dockerfiles/suricata.Dockerfile | 6 ++---- Dockerfiles/zeek.Dockerfile | 6 ++---- arkime/scripts/docker_entrypoint.sh | 1 - arkime/scripts/initarkime.sh | 2 +- arkime/supervisord.conf | 2 +- docker-compose-standalone.yml | 10 ---------- docker-compose.yml | 10 ---------- kubernetes/10-zeek.yml | 5 ----- kubernetes/11-suricata.yml | 9 --------- shared/bin/pcap_processor.py | 11 ++++++----- suricata/scripts/docker_entrypoint.sh | 1 - suricata/supervisord.conf | 2 +- zeek/scripts/docker_entrypoint.sh | 1 - zeek/supervisord.conf | 2 +- 15 files changed, 16 insertions(+), 58 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 57c96f82c..c0fc4d2dd 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -170,7 +170,6 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour apt-get install -q -y --no-install-recommends \ bc \ curl \ - ethtool \ file \ geoip-bin \ gettext \ @@ -249,9 +248,8 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ ln -sfr /opt/pcap_processor.py /opt/pcap_arkime_processor.py && \ cp -f /opt/arkime_update_geo.sh $ARKIME_DIR/bin/arkime_update_geo.sh && \ mv $ARKIME_DIR/etc/config.ini $ARKIME_DIR/etc/config.orig.ini && \ - chmod u+s $ARKIME_DIR/bin/capture && \ - chown root:${PGROUP} /sbin/ethtool $ARKIME_DIR/bin/capture && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + cp $ARKIME_DIR/bin/capture $ARKIME_DIR/bin/capture-offline && \ + chown root:${PGROUP} $ARKIME_DIR/bin/capture && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ mkdir -p /var/run/arkime && \ chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 5d20921d0..fdf5a5d68 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -60,7 +60,6 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour apt-get install -q -y --no-install-recommends \ bc \ curl \ - ethtool \ file \ inotify-tools \ iproute2 \ @@ -124,8 +123,8 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour find /tmp/default-rules/ -not -path '*/.gitignore' -type f -exec cp "{}" "$SURICATA_CONFIG_DIR"/rules/ \; && \ suricata-update update-sources --verbose --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ suricata-update update --fail --verbose --etopen --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ - chown root:${PGROUP} /sbin/ethtool /usr/bin/suricata && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + cp /usr/bin/suricata /usr/bin/suricata-offline && \ + chown root:${PGROUP} /usr/bin/suricata && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -135,7 +134,6 @@ COPY --chmod=644 suricata/supervisord.conf /etc/supervisord.conf COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic -COPY --chmod=755 shared/bin/nic-capture-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/pcap_processor.py /usr/local/bin/ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=755 shared/bin/suricata_config_populate.py /usr/local/bin/ diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 39a2cbee3..534fb5021 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -132,7 +132,6 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ clang \ cmake \ curl \ - ethtool \ file \ flex \ git \ @@ -223,7 +222,6 @@ ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/ ADD zeek/scripts/docker_entrypoint.sh /usr/local/bin/ ADD shared/bin/zeek_intel_setup.sh ${ZEEK_DIR}/bin/ ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ -ADD shared/bin/nic-capture-setup.sh /usr/local/bin/ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh @@ -244,8 +242,8 @@ RUN mkdir -p /tmp/logs && \ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \ usermod -a -G tty ${PUSER} && \ - chown root:${PGROUP} /sbin/ethtool "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + cp "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/zeek-offline && \ + chown root:${PGROUP} "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats && \ touch "${SUPERCRONIC_CRONTAB}" && \ diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 70fc3974b..4825f1a63 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -92,7 +92,6 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then fi # ensure capabilities for capture - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true fi diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh index f30295986..a47caab03 100755 --- a/arkime/scripts/initarkime.sh +++ b/arkime/scripts/initarkime.sh @@ -71,7 +71,7 @@ if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S. touch /tmp/not_a_packet.pcap - $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 + $ARKIME_DIR/bin/capture-offline $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 rm -f /tmp/not_a_packet.pcap echo "Initializing views..." diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index cb42cf4ce..57ba0656d 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -66,7 +66,7 @@ command=python3 /opt/pcap_arkime_processor.py %(ENV_PCAP_PIPELINE_VERBOSITY)s --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --arkime "%(ENV_ARKIME_DIR)s"/bin/capture + --arkime "%(ENV_ARKIME_DIR)s"/bin/capture-offline --autoarkime "%(ENV_ARKIME_AUTO_ANALYZE_PCAP_FILES)s" --forcearkime "%(ENV_ARKIME_ROTATED_PCAP)s" --autotag "%(ENV_AUTO_TAG)s" diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 9caad00cb..dfbb1f9a5 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -268,9 +268,6 @@ services: hostname: zeek networks: - default - cap_add: - # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -331,13 +328,6 @@ services: hostname: suricata networks: - default - cap_add: - # IPC_LOCK - to lock memory, preventing swapping - - IPC_LOCK - # SYS_RESOURCE - for increasing memlock limits - - SYS_RESOURCE - # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env diff --git a/docker-compose.yml b/docker-compose.yml index 1404fdc6c..365f29cd4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -301,9 +301,6 @@ services: hostname: zeek networks: - default - cap_add: - # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -372,13 +369,6 @@ services: hostname: suricata networks: - default - cap_add: - # IPC_LOCK - to lock memory, preventing swapping - - IPC_LOCK - # SYS_RESOURCE - for increasing memlock limits - - SYS_RESOURCE - # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling - - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index c4af201cd..93223ecf4 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -20,11 +20,6 @@ spec: imagePullPolicy: Always stdin: false tty: true - securityContext: - capabilities: - add: - # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - - SYS_NICE envFrom: - configMapRef: name: process-env diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index eb9cf0f0f..fad8c6ef4 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -20,15 +20,6 @@ spec: imagePullPolicy: Always stdin: false tty: true - securityContext: - capabilities: - add: - # IPC_LOCK - to lock memory, preventing swapping - - IPC_LOCK - # SYS_RESOURCE - for increasing memlock limits - - SYS_RESOURCE - # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - - SYS_NICE envFrom: - configMapRef: name: process-env diff --git a/shared/bin/pcap_processor.py b/shared/bin/pcap_processor.py index a27029703..8f54857c9 100755 --- a/shared/bin/pcap_processor.py +++ b/shared/bin/pcap_processor.py @@ -47,15 +47,15 @@ PCAP_PROCESSING_MODE_ZEEK = "zeek" PCAP_PROCESSING_MODE_SURICATA = "suricata" -ARKIME_CAPTURE_PATH = "/opt/arkime/bin/capture" +ARKIME_CAPTURE_PATH = "/opt/arkime/bin/capture-offline" ARKIME_AUTOARKIME_TAG = 'AUTOARKIME' -SURICATA_PATH = "/usr/bin/suricata" +SURICATA_PATH = "/usr/bin/suricata-offline" SURICATA_LOG_DIR = os.getenv('SURICATA_LOG_DIR', '/var/log/suricata') SURICATA_CONFIG_FILE = os.getenv('SURICATA_CONFIG_FILE', '/etc/suricata/suricata.yaml') SURICATA_AUTOSURICATA_TAG = 'AUTOSURICATA' -ZEEK_PATH = "/opt/zeek/bin/zeek" +ZEEK_PATH = "/opt/zeek/bin/zeek-offline" ZEEK_EXTRACTOR_MODE_INTERESTING = 'interesting' ZEEK_EXTRACTOR_MODE_MAPPED = 'mapped' ZEEK_EXTRACTOR_MODE_NONE = 'none' @@ -146,8 +146,9 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): if os.path.isfile(fileInfo[FILE_INFO_DICT_NAME]): # Arkime this PCAP if it's tagged "AUTOARKIME" or if the global autoArkime flag is turned on. - if forceArkime and ( - autoArkime + if ( + forceArkime + or autoArkime or ( (FILE_INFO_DICT_TAGS in fileInfo) and ARKIME_AUTOARKIME_TAG in fileInfo[FILE_INFO_DICT_TAGS] ) diff --git a/suricata/scripts/docker_entrypoint.sh b/suricata/scripts/docker_entrypoint.sh index a3d3e0200..b3044ed59 100755 --- a/suricata/scripts/docker_entrypoint.sh +++ b/suricata/scripts/docker_entrypoint.sh @@ -1,7 +1,6 @@ #!/bin/bash # ensure capabilities for capture -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata || true # modify suricata.yaml according to environment variables (as non-root) diff --git a/suricata/supervisord.conf b/suricata/supervisord.conf index 8a245f445..8376a91d9 100644 --- a/suricata/supervisord.conf +++ b/suricata/supervisord.conf @@ -24,7 +24,7 @@ command=python3 /usr/local/bin/pcap_suricata_processor.py %(ENV_PCAP_PIPELINE_VE --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --suricata /usr/bin/suricata + --suricata /usr/bin/suricata-offline --autotag "%(ENV_AUTO_TAG)s" --autosuricata "%(ENV_SURICATA_AUTO_ANALYZE_PCAP_FILES)s" --forcesuricata "%(ENV_SURICATA_ROTATED_PCAP)s" diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh index 5c3d7932a..0ed78c9b0 100755 --- a/zeek/scripts/docker_entrypoint.sh +++ b/zeek/scripts/docker_entrypoint.sh @@ -3,7 +3,6 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} # ensure capabilities for capture -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats || true diff --git a/zeek/supervisord.conf b/zeek/supervisord.conf index 4c5270253..5ad1d49b7 100644 --- a/zeek/supervisord.conf +++ b/zeek/supervisord.conf @@ -24,7 +24,7 @@ command=python3 /usr/local/bin/pcap_zeek_processor.py %(ENV_PCAP_PIPELINE_VERBOS --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --zeek /opt/zeek/bin/zeek + --zeek /opt/zeek/bin/zeek-offline --autotag "%(ENV_AUTO_TAG)s" --autozeek "%(ENV_ZEEK_AUTO_ANALYZE_PCAP_FILES)s" --forcezeek "%(ENV_ZEEK_ROTATED_PCAP)s" From c207871510142a40c518c6e89c69f3b5750be3c9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 10:36:57 -0700 Subject: [PATCH 39/78] work in progress for idaholab/Malcolm#281: arkime capture tuning parameters --- arkime/scripts/docker_entrypoint.sh | 26 +++++++++++++++++++++++--- arkime/scripts/live_capture.sh | 1 - config/arkime-live.env.example | 12 +++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 4825f1a63..2a8c5e751 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -58,6 +58,26 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_CONFIG_FILE}" sed -i "s/^\(pcapDir=\).*/\1\/data\/pcap\/arkime-live/" "${ARKIME_CONFIG_FILE}" + # performance tuning parameters + [[ -n "$ARKIME_DB_BULK_SIZE" ]] && \ + sed -r -i "s/(dbBulkSize)\s*=\s*.*/\1=$ARKIME_DB_BULK_SIZE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_MAGIC_MODE" ]] && \ + sed -r -i "s/(magicMode)\s*=\s*.*/\1=$ARKIME_MAGIC_MODE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_MAX_PACKETS_IN_QUEUE" ]] && \ + sed -r -i "s/(maxPacketsInQueue)\s*=\s*.*/\1=$ARKIME_MAX_PACKETS_IN_QUEUE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PACKET_THREADS" ]] && \ + sed -r -i "s/(packetThreads)\s*=\s*.*/\1=$ARKIME_PACKET_THREADS/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PCAP_WRITE_METHOD" ]] && \ + sed -r -i "s/(pcapWriteMethod)\s*=\s*.*/\1=$ARKIME_PCAP_WRITE_METHOD/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PCAP_WRITE_SIZE" ]] && \ + sed -r -i "s/(pcapWriteSize)\s*=\s*.*/\1=$ARKIME_PCAP_WRITE_SIZE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PCAP_READ_METHOD" ]] && \ + sed -r -i "s/(pcapReadMethod)\s*=\s*.*/\1=$ARKIME_PCAP_READ_METHOD/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_TPACKETV3_NUM_THREADS" ]] && \ + sed -r -i "s/(tpacketv3NumThreads)\s*=\s*.*/\1=$ARKIME_TPACKETV3_NUM_THREADS/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_TPACKETV3_BLOCK_SIZE" ]] && \ + sed -r -i "s/(tpacketv3BlockSize)\s*=\s*.*/\1=$ARKIME_TPACKETV3_BLOCK_SIZE/" "${ARKIME_CONFIG_FILE}" + # capture interface(s) if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then @@ -84,11 +104,11 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then # pcap compression COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" - sed -r -i "s/(simpleCompression)\s*=\s*.*/\1=$COMPRESSION_TYPE/" "$ARKIME_CONFIG_FILE" + sed -r -i "s/(simpleCompression)\s*=\s*.*/\1=$COMPRESSION_TYPE/" "${ARKIME_CONFIG_FILE}" if [[ "$COMPRESSION_TYPE" == "zstd" ]]; then - sed -r -i "s/(simpleZstdLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "$ARKIME_CONFIG_FILE" + sed -r -i "s/(simpleZstdLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "${ARKIME_CONFIG_FILE}" elif [[ "$COMPRESSION_TYPE" == "gzip" ]]; then - sed -r -i "s/(simpleGzipLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "$ARKIME_CONFIG_FILE" + sed -r -i "s/(simpleGzipLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "${ARKIME_CONFIG_FILE}" fi # ensure capabilities for capture diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh index bfb32604b..19072706a 100755 --- a/arkime/scripts/live_capture.sh +++ b/arkime/scripts/live_capture.sh @@ -64,7 +64,6 @@ echo "${ARKIME_DIR}"/bin/capture --insecure \ -c "${ARKIME_DIR}"/etc/config.ini \ -o pcapDir=/data/pcap/arkime-live \ - -o packetThreads=${ARKIME_PACKET_THREADS} \ -o dropUser=${PUSER} \ -o dropGroup=${PGROUP} \ -o ecsEventProvider=arkime \ diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example index 7b7273709..da1d54b02 100644 --- a/config/arkime-live.env.example +++ b/config/arkime-live.env.example @@ -1,9 +1,19 @@ # Whether or Arkime should monitor live traffic on a local # interface (PCAP_IFACE in pcap-capture.env specifies interface) ARKIME_LIVE_CAPTURE=false + +# Live capture tuning parameters ARKIME_COMPRESSION_TYPE=none ARKIME_COMPRESSION_LEVEL=0 -ARKIME_PACKET_THREADS=1 +ARKIME_DB_BULK_SIZE=4000000 +ARKIME_MAGIC_MODE=basic +ARKIME_MAX_PACKETS_IN_QUEUE=300000 +ARKIME_PACKET_THREADS=2 +ARKIME_PCAP_WRITE_METHOD=simple +ARKIME_PCAP_WRITE_SIZE=2560000 +ARKIME_PCAP_READ_METHOD=tpacketv3 +ARKIME_TPACKETV3_NUM_THREADS=2 +ARKIME_TPACKETV3_BLOCK_SIZE=8388608 # The node host (e.g., the IP address of the machine running Malcolm) to associate with # network traffic metadata when ARKIME_LIVE_CAPTURE is true From 2bc1c43add333c1a4104d714fe29d210399b7fdc Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 11:14:52 -0700 Subject: [PATCH 40/78] work in progress for idaholab/Malcolm#281: arkime capture rules --- Dockerfiles/arkime.Dockerfile | 5 +++-- arkime/etc/config.ini | 1 + arkime/rules/.gitignore | 3 +++ arkime/rules/single_session_no_spi.yml | 11 +++++++++++ arkime/rules/ssh_trunate.yml | 10 ++++++++++ arkime/rules/tls_trunate.yml | 10 ++++++++++ arkime/scripts/docker_entrypoint.sh | 7 +++++++ docker-compose-standalone.yml | 2 ++ docker-compose.yml | 2 ++ kubernetes/07-arkime.yml | 5 +++++ kubernetes/23-arkime-live.yml | 5 +++++ malcolm-iso/build.sh | 2 ++ scripts/malcolm_appliance_packager.sh | 2 ++ scripts/malcolm_kubernetes.py | 6 ++++++ 14 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 arkime/rules/.gitignore create mode 100644 arkime/rules/single_session_no_spi.yml create mode 100644 arkime/rules/ssh_trunate.yml create mode 100644 arkime/rules/tls_trunate.yml diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index c0fc4d2dd..26007ec75 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -222,6 +222,7 @@ COPY --chmod=644 shared/bin/watch_common.py /opt/ COPY --chmod=644 arkime/supervisord.conf /etc/supervisord.conf ADD arkime/scripts /opt/ ADD arkime/etc $ARKIME_DIR/etc/ +ADD arkime/rules/*.yml $ARKIME_DIR/rules/ ADD arkime/wise/source.*.js $ARKIME_DIR/wiseService/ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic @@ -234,7 +235,7 @@ RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do cd /tmp && \ curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \ tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \ - mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/logs/ && \ + mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/rules/ $ARKIME_DIR/logs/ && \ mv -v "GeoLite2-$DB.mmdb" $ARKIME_DIR/etc/; \ rm -f "GeoLite2-$DB*"; \ done; \ @@ -252,7 +253,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ chown root:${PGROUP} $ARKIME_DIR/bin/capture && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ mkdir -p /var/run/arkime && \ - chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime + chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/rules $ARKIME_DIR/logs /var/run/arkime #Update Path ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}" diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index f9e2e740b..34aed4c57 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -45,6 +45,7 @@ readTruncatedPackets=true reqBodyOnlyUtf8=true rirFile=/opt/arkime/etc/ipv4-address-space.csv rotateIndex=daily +rulesFiles= smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: spiDataMaxIndices=7 supportSha256=false diff --git a/arkime/rules/.gitignore b/arkime/rules/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/arkime/rules/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/arkime/rules/single_session_no_spi.yml b/arkime/rules/single_session_no_spi.yml new file mode 100644 index 000000000..fdfb73069 --- /dev/null +++ b/arkime/rules/single_session_no_spi.yml @@ -0,0 +1,11 @@ +--- +version: 1 +rules: + - name: "Dont save SPI sessions with only 1 source packet" + when: "beforeFinalSave" + fields: + packets.src: 1 + packets.dst: 0 + tcpflags.syn: 1 + ops: + _dontSaveSPI: 1 diff --git a/arkime/rules/ssh_trunate.yml b/arkime/rules/ssh_trunate.yml new file mode 100644 index 000000000..4d5b099d9 --- /dev/null +++ b/arkime/rules/ssh_trunate.yml @@ -0,0 +1,10 @@ +--- +version: 1 +rules: + - name: "Only save first n packets of SSH" + when: "fieldSet" + fields: + protocols: + - ssh + ops: + _maxPacketsToSave: 20 diff --git a/arkime/rules/tls_trunate.yml b/arkime/rules/tls_trunate.yml new file mode 100644 index 000000000..0e51a8e3d --- /dev/null +++ b/arkime/rules/tls_trunate.yml @@ -0,0 +1,10 @@ +--- +version: 1 +rules: + - name: "Only save first n packets of TLS" + when: "fieldSet" + fields: + protocols: + - tls + ops: + _maxPacketsToSave: 15 diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 2a8c5e751..1bcd41415 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -9,6 +9,7 @@ function urlencodeall() { } ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} +ARKIME_RULES_DIR=${ARKIME_RULES_DIR:-"/opt/arkime/rules"} ARKIME_CONFIG_FILE="${ARKIME_DIR}"/etc/config.ini ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} @@ -115,6 +116,12 @@ if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true fi + # rules files + if [[ -d "${ARKIME_RULES_DIR}" ]]; then + RULES_FILES="$(find "${ARKIME_RULES_DIR}" -mindepth 1 -maxdepth 1 -type f -size +0c \( -name '*.yml' -o -name '*.yaml' \) | tr '\n' ';' | sed 's/;$//' )" + sed -r -i "s|(rulesFiles)\s*=\s*.*|\1=$RULES_FILES|" "${ARKIME_CONFIG_FILE}" + fi + # comment-out features that are unused in hedgehog run profile mode and in live-capture mode if [[ "$MALCOLM_PROFILE" == "hedgehog" ]] || [[ "$LIVE_CAPTURE" == "true" ]]; then sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_CONFIG_FILE}" diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index dfbb1f9a5..aa443ec5f 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -219,6 +219,7 @@ services: volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap healthcheck: test: ["CMD", "curl", "--insecure", "--silent", "--fail", "https://localhost:8005/_ns_/nstest.html"] @@ -257,6 +258,7 @@ services: volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap zeek: image: ghcr.io/idaholab/malcolm/zeek:23.12.1 diff --git a/docker-compose.yml b/docker-compose.yml index 365f29cd4..a997afce8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -241,6 +241,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap + - ./arkime/rules:/opt/arkime/rules:ro - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro @@ -285,6 +286,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap + - ./arkime/rules:/opt/arkime/rules:ro - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index bbc9bf8dd..b045eb1d8 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -73,6 +73,8 @@ spec: name: arkime-var-local-catrust-volume - mountPath: /var/local/curlrc/secretmap name: arkime-opensearch-curlrc-secret-volume + - mountPath: "/opt/arkime/rules" + name: arkime-rules-volume - mountPath: "/data/pcap" name: arkime-pcap-volume initContainers: @@ -97,6 +99,9 @@ spec: - name: arkime-opensearch-curlrc-secret-volume secret: secretName: opensearch-curlrc + - name: arkime-rules-volume + configMap: + name: arkime-rules - name: arkime-pcap-volume persistentVolumeClaim: claimName: pcap-claim diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 5e3fff84d..8c9213e68 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -56,6 +56,8 @@ spec: name: arkime-live-var-local-catrust-volume - mountPath: /var/local/curlrc/secretmap name: arkime-live-opensearch-curlrc-secret-volume + - mountPath: "/opt/arkime/rules" + name: arkime-live-rules-volume - mountPath: "/data/pcap" name: arkime-live-pcap-volume initContainers: @@ -80,6 +82,9 @@ spec: - name: arkime-live-opensearch-curlrc-secret-volume secret: secretName: opensearch-curlrc + - name: arkime-live-rules-volume + configMap: + name: arkime-rules - name: arkime-live-pcap-volume persistentVolumeClaim: claimName: pcap-claim diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index cf5a39973..feb2a0f99 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -92,6 +92,7 @@ if [ -d "$WORKDIR" ]; then # grab things from the Malcolm parent directory into /etc/skel so the user's got it set up in their home/Malcolm dir pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 MALCOLM_DEST_DIR="$WORKDIR/work/$IMAGE_NAME-Live-Build/config/includes.chroot/etc/skel/Malcolm" + mkdir -p "$MALCOLM_DEST_DIR/arkime/rules/" mkdir -p "$MALCOLM_DEST_DIR/config/" mkdir -p "$MALCOLM_DEST_DIR/filebeat/certs/" mkdir -p "$MALCOLM_DEST_DIR/htadmin/" @@ -146,6 +147,7 @@ if [ -d "$WORKDIR" ]; then cp ./scripts/malcolm_utils.py "$MALCOLM_DEST_DIR/scripts/" cp ./kubernetes/*.* "$MALCOLM_DEST_DIR/kubernetes/" grep -v "^#" ./kubernetes/.gitignore | xargs -r -I XXX rm -f "$MALCOLM_DEST_DIR/kubernetes/XXX" + cp ./arkime/rules/*.yml "$MALCOLM_DEST_DIR/arkime/rules/" cp ./logstash/certs/*.conf "$MALCOLM_DEST_DIR/logstash/certs/" cp ./logstash/maps/malcolm_severity.yaml "$MALCOLM_DEST_DIR/logstash/maps/" cp -r ./netbox/config/ "$MALCOLM_DEST_DIR/netbox/" diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index a0360483f..d4fb0f795 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -61,6 +61,7 @@ if mkdir "$DESTDIR"; then # ensure that if we "grabbed a lock", we release it (works for clean exit, SIGTERM, and SIGINT/Ctrl-C) trap "cleanup" EXIT + mkdir $VERBOSE -p "$DESTDIR/arkime/rules/" mkdir $VERBOSE -p "$DESTDIR/filebeat/certs/" mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" @@ -102,6 +103,7 @@ if mkdir "$DESTDIR"; then cp $VERBOSE ./scripts/malcolm_kubernetes.py "$DESTDIR/scripts/" cp $VERBOSE ./scripts/malcolm_utils.py "$DESTDIR/scripts/" cp $VERBOSE ./README.md "$DESTDIR/" + cp $VERBOSE ./arkime/rules/*.yml "$DESTDIR/arkime/rules/" cp $VERBOSE ./logstash/certs/*.conf "$DESTDIR/logstash/certs/" cp $VERBOSE ./logstash/maps/malcolm_severity.yaml "$DESTDIR/logstash/maps/" cp $VERBOSE -r ./netbox/config/ "$DESTDIR/netbox/" diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index aa6ac8557..bc29f4b88 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -107,6 +107,12 @@ 'path': os.path.join(MalcolmPath, os.path.join('logstash', 'logstash.keystore')), }, ], + 'arkime-rules': [ + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('arkime', 'rules')), + }, + ], 'yara-rules': [ { 'secret': False, From 87ef4ed62c66a691ed12c6d43f4847d61391ddbf Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 11:34:29 -0700 Subject: [PATCH 41/78] documentation updates --- docs/README.md | 1 + docs/live-analysis.md | 15 +++++++++++---- docs/ubuntu-install-example.md | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index a1f428dc0..560f87caa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation * [Live analysis](live-analysis.md#LiveAnalysis) - [Using a network sensor appliance](live-analysis.md#Hedgehog) - [Monitoring local network interfaces](live-analysis.md#LocalPCAP) + + ["Hedgehog" run profile](live-analysis.md#Profiles) - [Manually forwarding logs from an external source](live-analysis.md#ExternalForward) * [Arkime](arkime.md#Arkime) - [Zeek log integration](arkime.md#ArkimeZeek) diff --git a/docs/live-analysis.md b/docs/live-analysis.md index 2095b8649..1389384db 100644 --- a/docs/live-analysis.md +++ b/docs/live-analysis.md @@ -3,6 +3,7 @@ * [Live analysis](#LiveAnalysis) - [Using a network sensor appliance](#Hedgehog) - [Monitoring local network interfaces](#LocalPCAP) + + ["Hedgehog" run profile](#Profiles) - [Manually forwarding logs from an external source](#ExternalForward) ## Using a network sensor appliance @@ -18,16 +19,22 @@ Please see the [Hedgehog Linux README](hedgehog.md) for more information. ## Monitoring local network interfaces -Malcolm's `pcap-capture`, `suricata-live` and `zeek-live` containers can monitor one or more local network interfaces, specified by the `PCAP_IFACE` environment variable in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). These containers are started with additional privileges (`IPC_LOCK`, `NET_ADMIN`, `NET_RAW`) to allow opening network interfaces in promiscuous mode for capture. +The options for monitoring traffic on local network interfaces can be [configured](malcolm-hedgehog-e2e-iso-install.md#MalcolmConfig) by running [`./scripts/configure`](malcolm-config.md#ConfigAndTuning). -The instances of Zeek and Suricata (in the `suricata-live` and `zeek-live` containers when the `SURICATA_LIVE_CAPTURE` and `ZEEK_LIVE_CAPTURE` [environment variables](malcolm-config.md#MalcolmConfigEnvVars) are set to `true`, respectively) analyze traffic on-the-fly and generate log files containing network session metadata. These log files are in turn scanned by [Filebeat](https://www.elastic.co/products/beats/filebeat) and forwarded to [Logstash](https://www.elastic.co/products/logstash) for enrichment and indexing into the [OpenSearch](https://opensearch.org/) document store. +Malcolm's `pcap-capture`, `suricata-live` and `zeek-live` containers can monitor one or more local network interfaces, specified by the `PCAP_IFACE` environment variable in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). These containers are started with additional privileges to allow opening network interfaces in promiscuous mode for capture. -In contrast, the `pcap-capture` container buffers traffic to PCAP files and periodically rotates these files for processing (by Arkime's `capture` utlity in the `arkime` container) according to the thresholds defined by the `PCAP_ROTATE_MEGABYTES` and `PCAP_ROTATE_MINUTES` environment variables in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). If for some reason (e.g., a low resources environment) you also want Zeek and Suricata to process these intermediate PCAP files rather than monitoring the network interfaces directly, you can set `SURICATA_ROTATED_PCAP`/`ZEEK_ROTATED_PCAP` to `true` and `SURICATA_LIVE_CAPTURE`/`ZEEK_LIVE_CAPTURE` to false. +The instances of Zeek and Suricata (in the `suricata-live` and `zeek-live` containers when the `SURICATA_LIVE_CAPTURE` and `ZEEK_LIVE_CAPTURE` [environment variables](malcolm-config.md#MalcolmConfigEnvVars) are set to `true`, respectively) analyze traffic on-the-fly and generate log files containing network session metadata. These log files are in turn scanned by [Filebeat](https://www.elastic.co/products/beats/filebeat) and forwarded to [Logstash](https://www.elastic.co/products/logstash) for enrichment and indexing into the [OpenSearch](https://opensearch.org/) document store. -These various options for monitoring traffic on local network interfaces can also be configured by running [`./scripts/configure`](malcolm-config.md#ConfigAndTuning). +In contrast, the `pcap-capture` container buffers traffic to PCAP files and periodically rotates these files for processing (by Arkime's `capture` utlity in the `arkime` container) according to the thresholds defined by the `PCAP_ROTATE_MEGABYTES` and `PCAP_ROTATE_MINUTES` environment variables in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). If for some reason (e.g., a low resources environment) you also want Zeek and Suricata to process these intermediate PCAP files rather than monitoring the network interfaces directly, you can set `SURICATA_ROTATED_PCAP`/`ZEEK_ROTATED_PCAP` to `true` and `SURICATA_LIVE_CAPTURE`/`ZEEK_LIVE_CAPTURE` to false. The only exception to this behavior (i.e., the creation of intermediate PCAP files by `netsniff-ng` or `tcpdump` in the `pcap-capture` which are periodically rolled over for processing by Arkime) is when running the ["Hedgehog" run profile](#Profiles) or when using [a remote OpenSearch or Elasticsearch instance](opensearch-instances.md#OpenSearchInstance). In either of these configurations, users may choose to have Arkime's `capture` tool monitor live traffic on the network interface without using the intermediate PCAP file. Note that Microsoft Windows and Apple macOS platforms currently run Docker inside of a virtualized environment. Live traffic capture and analysis on those platforms would require additional configuration of virtual interfaces and port forwarding in Docker, which is outside of the scope of this document. +### "Hedgehog" run profile + +Another configuration for monitoring local network interfaces is to use the `hedgehog` run profile. During [Malcolm configuration](malcolm-hedgehog-e2e-iso-install.md#MalcolmConfig) users are prompted "**Run with Malcolm (all containers) or Hedgehog (capture only) profile?**" Docker Compose can use [profiles](https://docs.docker.com/compose/profiles/) to selectively start services. While the `malcolm` run profile runs all of Malcolm's containers (OpenSearch, Dashboards, LogStash, etc.), the `hedgehog` profile runs *only* the containers necessary for traffic capture. + +When configuring the `hedgehog` profile, users must provide connection details for another Malcolm instance to which to forward its network traffic logs. + ## Manually forwarding logs from an external source Malcolm's Logstash instance can also be configured to accept logs from a [remote forwarder](https://www.elastic.co/products/beats/filebeat) by running [`./scripts/configure`](malcolm-config.md#ConfigAndTuning) and answering "yes" to "`Expose Logstash port to external hosts?`" Enabling encrypted transport of these log files is discussed in [Configure authentication](authsetup.md#AuthSetup) and the description of the `BEATS_SSL` environment variable in [`beats-common.env`](malcolm-config.md#MalcolmConfigEnvVars). diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index e47cc726f..389a0ad39 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -229,7 +229,7 @@ As an alternative to manually copying the files to the sensor, Malcolm can facil In this example, rather than [building Malcolm from scratch](development.md#Build), images may be pulled from [GitHub](https://github.com/orgs/idaholab/packages?repo_name=Malcolm): ``` -user@host:~/Malcolm$ docker compose --profile=malcolm pull +user@host:~/Malcolm$ docker compose --profile malcolm pull Pulling api ... done Pulling arkime ... done Pulling dashboards ... done From 89b904df95d13548edf686720d76563f41a3a014 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 13:09:53 -0700 Subject: [PATCH 42/78] for idaholab/Malcolm#282 (container capabilities), netsniff-ng needs SYS_ADMIN to set disk i/o scheduler policy --- docker-compose-standalone.yml | 2 ++ docker-compose.yml | 2 ++ kubernetes/20-pcap-capture.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index aa443ec5f..3fa863bab 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -423,6 +423,8 @@ services: - IPC_LOCK # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE + # SYS_ADMIN - for netsniff-ng to set the disc I/O scheduler policy + - SYS_ADMIN # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/docker-compose.yml b/docker-compose.yml index a997afce8..ba4a0fc63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -473,6 +473,8 @@ services: - IPC_LOCK # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE + # SYS_ADMIN - for netsniff-ng to set the disc I/O scheduler policy + - SYS_ADMIN # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 7eb5c83ec..610f0ade8 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -27,6 +27,8 @@ spec: - IPC_LOCK # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE + # SYS_ADMIN - for netsniff-ng to set the disc I/O scheduler policy + - SYS_ADMIN # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW From 08a76adfb0c342ea308bcedfcb4301c40066edf9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Dec 2023 13:12:24 -0700 Subject: [PATCH 43/78] for idaholab/Malcolm#282 (container capabilities), netsniff-ng needs SYS_ADMIN to set disk i/o scheduler policy --- Dockerfiles/pcap-capture.Dockerfile | 2 +- pcap-capture/scripts/supervisor.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index b13b7b009..c582e9e26 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -87,7 +87,7 @@ RUN apt-get -q update && \ chown root:${PGROUP} /usr/bin/tcpdump && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump && \ chown root:${PGROUP} /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng && \ chmod 755 /usr/local/bin/*.sh WORKDIR "$PCAP_PATH" diff --git a/pcap-capture/scripts/supervisor.sh b/pcap-capture/scripts/supervisor.sh index 07e1096e3..618e775ff 100755 --- a/pcap-capture/scripts/supervisor.sh +++ b/pcap-capture/scripts/supervisor.sh @@ -14,7 +14,7 @@ function join_by { local IFS="$1"; shift; echo "$*"; } function SetCaptureCapabilities() { setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump || true - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng || true } # Create config files for each capture interface for the various capture programs (tcpdump, netsniff) From cc53ae67167eb4293dffeed47218c2511373a53d Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 11 Dec 2023 08:24:29 -0700 Subject: [PATCH 44/78] idaholab/Malcolm#309, rsyslog no longer in bookworm --- docs/malcolm-hedgehog-e2e-iso-install.md | 6 +++--- logstash/pipelines/beats/11_beats_logs.conf | 5 +++++ .../0991-security-performance.hook.chroot | 8 ------- .../0991-security-performance.hook.chroot | 8 ------- .../interface/sensor_ctl/control_vars.conf | 2 +- .../supervisor.d/fluentbit-syslog.conf | 21 ------------------- .../supervisor.d/fluentbit-systemd.conf | 18 ++++++++++++++++ 7 files changed, 27 insertions(+), 41 deletions(-) delete mode 100644 sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf create mode 100644 sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index ba5511a5f..cc756826a 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -572,9 +572,9 @@ Despite configuring capture and/or forwarder services as described in previous s * **AUTOSTART_FILEBEAT** - [filebeat](#Hedgehogfilebeat) Zeek and Suricata log forwarder * **AUTOSTART_FLUENTBIT_AIDE** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/exec) [AIDE](https://aide.github.io/) file system integrity checks * **AUTOSTART_FLUENTBIT_AUDITLOG** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/tail) [auditd](https://man7.org/linux/man-pages/man8/auditd.8.html) logs -* *AUTOSTART_FLUENTBIT_KMSG* - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/kernel-logs) the Linux kernel log buffer (these are generally reflected in syslog as well, which may make this agent redundant) +* *AUTOSTART_FLUENTBIT_KMSG* - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/kernel-logs) the Linux kernel log buffer (these are generally reflected in the Systemd log as well, which may make this agent redundant) * **AUTOSTART_FLUENTBIT_METRICS** - [Fluent Bit](https://fluentbit.io/) agent for collecting [various](https://docs.fluentbit.io/manual/pipeline/inputs) system resource and performance metrics -* **AUTOSTART_FLUENTBIT_SYSLOG** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/syslog) Linux syslog messages +* **AUTOSTART_FLUENTBIT_SYSTEMD** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/systemd) log messages from the Linux Journald daemon * **AUTOSTART_FLUENTBIT_THERMAL** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/thermal) system temperatures (only applicable on actual hardware, not if Hedgehog is running on a virtual machine) * **AUTOSTART_MISCBEAT** - [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-tcp.html) forwarder which sends system metrics collected by [Fluent Bit](https://fluentbit.io/) to a remote Logstash instance (e.g., [Malcolm]({{ site.github.repository_url }})'s) * *AUTOSTART_NETSNIFF* - [netsniff-ng](http://netsniff-ng.org/) PCAP engine for saving packet capture (PCAP) files @@ -617,7 +617,7 @@ fluentbit-metrics:disk RUNNING pid 6468, uptime 0:03:17 fluentbit-metrics:mem RUNNING pid 6472, uptime 0:03:17 fluentbit-metrics:mem_p RUNNING pid 6473, uptime 0:03:17 fluentbit-metrics:netif RUNNING pid 6474, uptime 0:03:17 -fluentbit-syslog RUNNING pid 6478, uptime 0:03:17 +fluentbit-systemd RUNNING pid 6478, uptime 0:03:17 fluentbit-thermal RUNNING pid 6480, uptime 0:03:17 netsniff:netsniff-enp1s0 STOPPED Not started prune:prune-pcap RUNNING pid 6484, uptime 0:03:17 diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 17c7a3998..944d07445 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -545,6 +545,11 @@ filter { } # if ([miscbeat][syslog]) + # if ([miscbeat][systemd]) { + # #------------------------------------------------- + # # systemd - https://docs.fluentbit.io/manual/pipeline/inputs/systemd + # } # if ([miscbeat][systemd]) + if ([miscbeat][thermal]) { #------------------------------------------------- # thermal - https://docs.fluentbit.io/manual/pipeline/inputs/thermal diff --git a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot index 9d160f065..7e4f86f29 100755 --- a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -121,14 +121,6 @@ echo "umask 077" >> /etc/profile echo "export UMASK=077" >> /etc/profile echo "export PYTHONDONTWRITEBYTECODE=1" >> /etc/profile -# enable cron logging -sed -r -i "s@^#(cron\.\*\s+.*/var/log/cron\.log)@\1@" /etc/rsyslog.conf - -# enable rsyslog forwarding to localhost:9514 over UDP (for filebeat syslog input) -echo >> /etc/rsyslog.conf -echo '*.* @127.0.0.1:9514' >> /etc/rsyslog.conf -echo >> /etc/rsyslog.conf - # put sudoers log into its own logfile awk 'FNR==NR{ if (/^Defaults/) p=NR; next} 1; FNR==p{ print "Defaults\t!syslog\nDefaults\tlogfile=/var/log/sudo.log" }' /etc/sudoers /etc/sudoers > /tmp/newsudoers mv /tmp/newsudoers /etc/sudoers && chmod 440 /etc/sudoers diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot index 95349d90f..93afb4f60 100755 --- a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -96,14 +96,6 @@ echo "umask 077" >> /etc/profile echo "export UMASK=077" >> /etc/profile echo "export PYTHONDONTWRITEBYTECODE=1" >> /etc/profile -# enable cron logging -sed -r -i "s@^#(cron\.\*\s+.*/var/log/cron\.log)@\1@" /etc/rsyslog.conf - -# enable rsyslog forwarding to localhost:9514 over UDP (for filebeat syslog input) -echo >> /etc/rsyslog.conf -echo '*.* @127.0.0.1:9514' >> /etc/rsyslog.conf -echo >> /etc/rsyslog.conf - # put sudoers log into its own logfile awk 'FNR==NR{ if (/^Defaults/) p=NR; next} 1; FNR==p{ print "Defaults\t!syslog\nDefaults\tlogfile=/var/log/sudo.log" }' /etc/sudoers /etc/sudoers > /tmp/newsudoers mv /tmp/newsudoers /etc/sudoers && chmod 440 /etc/sudoers diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 98301dae5..67a48f3dc 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -140,7 +140,7 @@ export AUTOSTART_FLUENTBIT_AIDE=false export AUTOSTART_FLUENTBIT_AUDITLOG=false export AUTOSTART_FLUENTBIT_KMSG=false export AUTOSTART_FLUENTBIT_METRICS=false -export AUTOSTART_FLUENTBIT_SYSLOG=false +export AUTOSTART_FLUENTBIT_SYSTEMD=false export AUTOSTART_FLUENTBIT_THERMAL=false export AUTOSTART_MISCBEAT=false export AUTOSTART_NETSNIFF=false diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf deleted file mode 100644 index 0aaef35eb..000000000 --- a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf +++ /dev/null @@ -1,21 +0,0 @@ -[program:fluentbit-syslog] -command=/opt/fluent-bit/bin/fluent-bit - -R /etc/fluent-bit/parsers.conf - -i syslog - -p Mode=udp - -p Listen=127.0.0.1 - -p Port=9514 - -p Parser=syslog-rfc3164 - -p Buffer_Chunk_Size=32000 - -p Buffer_Max_Size=64000 - -o tcp://localhost:%(ENV_MISCBEAT_PORT)s - -p format=json_lines - -F nest -p Operation=nest -p Nested_under=syslog -p WildCard='*' -m '*' - -F record_modifier -p "Record=module syslog" -m '*' - -f 1 -startsecs=20 -startretries=3 -stopasgroup=true -killasgroup=true -autostart=%(ENV_AUTOSTART_FLUENTBIT_SYSLOG)s -autorestart=%(ENV_AUTOSTART_FLUENTBIT_SYSLOG)s diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf new file mode 100644 index 000000000..d62057e30 --- /dev/null +++ b/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf @@ -0,0 +1,18 @@ +[program:fluentbit-systemd] +command=/opt/fluent-bit/bin/fluent-bit + -R /etc/fluent-bit/parsers.conf + -i systemd + -p Read_From_Tail=On + -p Lowercase=On + -p Strip_Underscores=On + -o tcp://localhost:%(ENV_MISCBEAT_PORT)s + -p format=json_lines + -F nest -p Operation=nest -p Nested_under=systemd -p WildCard='*' -m '*' + -F record_modifier -p "Record=module systemd" -m '*' + -f 1 +startsecs=20 +startretries=3 +stopasgroup=true +killasgroup=true +autostart=%(ENV_AUTOSTART_FLUENTBIT_SYSTEMD)s +autorestart=%(ENV_AUTOSTART_FLUENTBIT_SYSTEMD)s From ad74561429432d3e565fdd46e53f430b681c3554 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 11 Dec 2023 15:35:13 -0700 Subject: [PATCH 45/78] set defaults for ZEEK_INTEL_PATH and ZEEK_CUSTOM_PATH --- Dockerfiles/zeek.Dockerfile | 4 ++++ config/zeek-live.env.example | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 534fb5021..0c3d43172 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -266,6 +266,8 @@ ARG ZEEK_INTEL_REFRESH_THREADS=2 ARG ZEEK_INTEL_FEED_SINCE= ARG ZEEK_EXTRACTOR_MODE=none ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files +ARG ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel +ARG ZEEK_CUSTOM_PATH=/opt/zeek/share/zeek/site/custom ARG PCAP_PIPELINE_VERBOSITY="" ARG PCAP_MONITOR_HOST=pcap-monitor ARG ZEEK_LIVE_CAPTURE=false @@ -287,6 +289,8 @@ ENV ZEEK_INTEL_REFRESH_THREADS $ZEEK_INTEL_REFRESH_THREADS ENV ZEEK_INTEL_FEED_SINCE $ZEEK_INTEL_FEED_SINCE ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH +ENV ZEEK_INTEL_PATH $ZEEK_INTEL_PATH +ENV ZEEK_CUSTOM_PATH $ZEEK_CUSTOM_PATH ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST ENV ZEEK_LIVE_CAPTURE $ZEEK_LIVE_CAPTURE diff --git a/config/zeek-live.env.example b/config/zeek-live.env.example index 3659dacac..4cf6fc0cb 100644 --- a/config/zeek-live.env.example +++ b/config/zeek-live.env.example @@ -5,6 +5,4 @@ ZEEK_LIVE_CAPTURE=false ZEEK_PCAP_PROCESSOR=false ZEEK_CRON=true ZEEK_LOG_PATH=/zeek/live -ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel -ZEEK_CUSTOM_PATH=/opt/zeek/share/zeek/site/custom EXTRACT_FILES_PATH=/zeek/extract_files \ No newline at end of file From e0b9287cae71b9b382997c3999cf57869e70355b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 12 Dec 2023 06:56:55 -0700 Subject: [PATCH 46/78] beats and logstash to to v8.11.3 --- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/logstash.Dockerfile | 2 +- sensor-iso/build.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 37226950b..a5897b36d 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.11.2 +FROM docker.elastic.co/beats/filebeat-oss:8.11.3 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index f4424705d..e5e792f2e 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.11.2 +FROM docker.elastic.co/logstash/logstash-oss:8.11.3 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index b5841ce94..29bd10f03 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.11.2" +BEATS_VER="8.11.3" BEATS_OSS="-oss" BUILD_ERROR_CODE=1 From 53eefa6ed89d6fff7a9b64568e8f0b6f1fabc3ac Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 12 Dec 2023 08:05:36 -0700 Subject: [PATCH 47/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../composable/component/miscbeat.json | 52 +++++++++++++++++++ logstash/pipelines/beats/11_beats_logs.conf | 47 +++++++++++++++-- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index 62f272927..d6610251e 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -93,6 +93,58 @@ "properties": { } }, + "systemd": { + "properties": { + "audit_loginuid": { "type": "integer" }, + "audit_session": { "type": "integer" }, + "boot_id": { "type": "keyword" }, + "cap_effective": { "type": "keyword" }, + "cmdline": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "code_file": { "type": "keyword", "ignore_above": 512, "fields": { "text": { "type": "text" } } }, + "code_func": { "type": "keyword", "ignore_above": 512, "fields": { "text": { "type": "text" } } }, + "code_line": { "type": "integer" }, + "comm": { "type": "keyword" }, + "exe": { "type": "keyword", "ignore_above": 512, "fields": { "text": { "type": "text" } } }, + "gid": { "type": "integer" }, + "hostname": { "type": "keyword" }, + "invocation_id": { "type": "keyword" }, + "job_id": { "type": "integer" }, + "job_result": { "type": "keyword" }, + "job_type": { "type": "keyword" }, + "machine_id": { "type": "keyword" }, + "message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "message_id": { "type": "keyword" }, + "nm_device": { "type": "keyword" }, + "nm_log_domains": { "type": "keyword" }, + "nm_log_level": { "type": "keyword" }, + "pid": { "type": "integer" }, + "priority": { "type": "integer" }, + "runtime_scope": { "type": "keyword" }, + "selinux_context": { "type": "keyword" }, + "source_monotonic_timestamp": { "type": "long" }, + "source_realtime_timestamp" : { "type": "date" }, + "stream_id": { "type": "keyword" }, + "syslog_facility": { "type": "integer" }, + "syslog_identifier": { "type": "keyword" }, + "syslog_pid": { "type": "integer" }, + "syslog_raw": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "syslog_timestamp": { "type": "keyword" }, + "systemd_cgroup": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "systemd_invocation_id": { "type": "keyword" }, + "systemd_owner_uid": { "type": "integer" }, + "systemd_session": { "type": "keyword" }, + "systemd_slice": { "type": "keyword" }, + "systemd_unit": { "type": "keyword" }, + "systemd_user_slice": { "type": "keyword" }, + "systemd_user_unit": { "type": "keyword" }, + "tid": { "type": "integer" }, + "timestamp_boottime": { "type": "float" }, + "timestamp_monotonic": { "type": "float" }, + "transport": { "type": "keyword" }, + "uid": { "type": "integer" }, + "unit": { "type": "keyword" } + } + }, "thermal": { "properties": { "name": { "type": "keyword"}, diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 944d07445..9a0f16b0d 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -545,10 +545,49 @@ filter { } # if ([miscbeat][syslog]) - # if ([miscbeat][systemd]) { - # #------------------------------------------------- - # # systemd - https://docs.fluentbit.io/manual/pipeline/inputs/systemd - # } # if ([miscbeat][systemd]) + if ([miscbeat][systemd]) { + #------------------------------------------------- + # systemd - https://docs.fluentbit.io/manual/pipeline/inputs/systemd + + if ([miscbeat][systemd][source_realtime_timestamp]) { + # microseconds to milliseconds + ruby { + id => "ruby_miscbeat_systemd_source_realtime_timestamp_calc" + code => "event.set('[miscbeat][systemd][source_realtime_timestamp]', (event.get('[miscbeat][systemd][source_realtime_timestamp]').to_i / 1000)" + } + # milliseconds to date + date { + id => "date_beats_miscbeat_systemd_source_realtime_timestamp" + match => [ "[miscbeat][systemd][source_realtime_timestamp]", "UNIX_MS" ] + target => "[miscbeat][systemd][source_realtime_timestamp]" + } + } + + if (![event][hash]) { + fingerprint { + id => "fingerprint_malcolm_miscbeat_systemd" + source => [ "[host][name]", + "[event][module]", + "[miscbeat][systemd][pid]", + "[miscbeat][systemd][uid]", + "[miscbeat][systemd][gid]", + "[miscbeat][systemd][unit]", + "[miscbeat][systemd][comm]", + "[miscbeat][systemd][exe]", + "[miscbeat][systemd][cmdline]", + "[miscbeat][systemd][machine_id]", + "[miscbeat][systemd][message]", + "[miscbeat][systemd][message_id]", + "[@timestamp]" ] + concatenate_sources => true + # uses event.hash + ecs_compatibility => "v8" + method => "MURMUR3_128" + base64encode => true + } + } + + } # if ([miscbeat][systemd]) if ([miscbeat][thermal]) { #------------------------------------------------- From 82021da6654a9ae0709e7f5e66f374987e506ed0 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 12 Dec 2023 10:03:13 -0700 Subject: [PATCH 48/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- logstash/pipelines/beats/11_beats_logs.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 9a0f16b0d..83e38450e 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -553,7 +553,10 @@ filter { # microseconds to milliseconds ruby { id => "ruby_miscbeat_systemd_source_realtime_timestamp_calc" - code => "event.set('[miscbeat][systemd][source_realtime_timestamp]', (event.get('[miscbeat][systemd][source_realtime_timestamp]').to_i / 1000)" + code => " + event.set('[miscbeat][systemd][source_realtime_timestamp]', + (event.get('[miscbeat][systemd][source_realtime_timestamp]').to_i / 1000)) + " } # milliseconds to date date { From 8715fafe702975cd403d4601742ae50987adb650 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 12 Dec 2023 15:00:19 -0700 Subject: [PATCH 49/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../composable/component/miscbeat.json | 18 ------- logstash/pipelines/beats/11_beats_logs.conf | 49 +++++++++++++++++-- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index d6610251e..28647d642 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -95,17 +95,7 @@ }, "systemd": { "properties": { - "audit_loginuid": { "type": "integer" }, - "audit_session": { "type": "integer" }, "boot_id": { "type": "keyword" }, - "cap_effective": { "type": "keyword" }, - "cmdline": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "code_file": { "type": "keyword", "ignore_above": 512, "fields": { "text": { "type": "text" } } }, - "code_func": { "type": "keyword", "ignore_above": 512, "fields": { "text": { "type": "text" } } }, - "code_line": { "type": "integer" }, - "comm": { "type": "keyword" }, - "exe": { "type": "keyword", "ignore_above": 512, "fields": { "text": { "type": "text" } } }, - "gid": { "type": "integer" }, "hostname": { "type": "keyword" }, "invocation_id": { "type": "keyword" }, "job_id": { "type": "integer" }, @@ -117,31 +107,23 @@ "nm_device": { "type": "keyword" }, "nm_log_domains": { "type": "keyword" }, "nm_log_level": { "type": "keyword" }, - "pid": { "type": "integer" }, "priority": { "type": "integer" }, "runtime_scope": { "type": "keyword" }, "selinux_context": { "type": "keyword" }, "source_monotonic_timestamp": { "type": "long" }, "source_realtime_timestamp" : { "type": "date" }, "stream_id": { "type": "keyword" }, - "syslog_facility": { "type": "integer" }, - "syslog_identifier": { "type": "keyword" }, - "syslog_pid": { "type": "integer" }, "syslog_raw": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "syslog_timestamp": { "type": "keyword" }, "systemd_cgroup": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "systemd_invocation_id": { "type": "keyword" }, - "systemd_owner_uid": { "type": "integer" }, "systemd_session": { "type": "keyword" }, "systemd_slice": { "type": "keyword" }, "systemd_unit": { "type": "keyword" }, "systemd_user_slice": { "type": "keyword" }, "systemd_user_unit": { "type": "keyword" }, - "tid": { "type": "integer" }, "timestamp_boottime": { "type": "float" }, "timestamp_monotonic": { "type": "float" }, - "transport": { "type": "keyword" }, - "uid": { "type": "integer" }, "unit": { "type": "keyword" } } }, diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 83e38450e..5c1f6ed4b 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -566,20 +566,61 @@ filter { } } + # rename fields according to ECS + # https://www.elastic.co/guide/en/ecs/current/ecs-process.html + # https://www.elastic.co/guide/en/ecs/current/ecs-log.html + mutate { id => "mutate_rename_miscbeat_systemd_log_fields" + # todo: translate this to CAP_BPF, CAP_SYS_ADMIN, etc. + rename => { "[miscbeat][systemd][cap_effective]" => "[process][thread][capabilities][effective]" } + rename => { "[miscbeat][systemd][cmdline]" => "[process][command_line]" } + rename => { "[miscbeat][systemd][code_file]" => "[log][origin][file][name]" } + rename => { "[miscbeat][systemd][code_func]" => "[log][origin][function]" } + rename => { "[miscbeat][systemd][code_line]" => "[log][origin][file][line]" } + rename => { "[miscbeat][systemd][comm]" => "[process][name]" } + rename => { "[miscbeat][systemd][exe]" => "[process][executable]" } + rename => { "[miscbeat][systemd][gid]" => "[process][group_leader][pid]" } + rename => { "[miscbeat][systemd][pid]" => "[process][pid]" } + rename => { "[miscbeat][systemd][syslog_facility]" => "[log][syslog][facility][code]" } + rename => { "[miscbeat][systemd][syslog_identifier]" => "[log][syslog][appname]" } + rename => { "[miscbeat][systemd][syslog_pid]" => "[log][syslog][procid]" } + rename => { "[miscbeat][systemd][systemd_owner_uid]" => "[process][real_user][id]" } + rename => { "[miscbeat][systemd][tid]" => "[process][thread][id]" } + rename => { "[miscbeat][systemd][uid]" => "[process][user][id]" } + rename => { "[miscbeat][systemd][transport]" => "[log][logger]" } + } + if ([log][syslog][facility][code]) { + translate { + id => "translate_sensor_systemd_syslog_facility_code" + source => "[log][syslog][facility][code]" + target => "[log][syslog][facility][name]" + dictionary_path => "/etc/syslog_facility_codes.yaml" + } + } + + # store raw message text as event.original + if ([miscbeat][systemd][syslog_raw]) { + mutate { id => "mutate_rename_miscbeat_systemd_syslog_raw" + rename => { "[miscbeat][systemd][syslog_raw]" => "[event][original]" } } + } else if ([miscbeat][systemd][message]) { + mutate { id => "mutate_rename_miscbeat_systemd_message" + rename => { "[miscbeat][systemd][message]" => "[event][original]" } } + } + if (![event][hash]) { fingerprint { id => "fingerprint_malcolm_miscbeat_systemd" source => [ "[host][name]", "[event][module]", - "[miscbeat][systemd][pid]", - "[miscbeat][systemd][uid]", - "[miscbeat][systemd][gid]", + "[process][pid]", + "[process][user][id]", + "[process][thread][id]", + "[process][group_leader][pid]", "[miscbeat][systemd][unit]", "[miscbeat][systemd][comm]", "[miscbeat][systemd][exe]", "[miscbeat][systemd][cmdline]", "[miscbeat][systemd][machine_id]", - "[miscbeat][systemd][message]", + "[event][original]", "[miscbeat][systemd][message_id]", "[@timestamp]" ] concatenate_sources => true From a8f1bedf5a02265a96a601d179ed2d1000b62022 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 12 Dec 2023 15:51:00 -0700 Subject: [PATCH 50/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../composable/component/miscbeat.json | 20 ++++++++++++++++++- logstash/pipelines/beats/11_beats_logs.conf | 7 +++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index 28647d642..d3072b3c4 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -95,12 +95,21 @@ }, "systemd": { "properties": { + "audit_loginuid": { "type": "integer" }, + "audit_session": { "type": "integer" }, "boot_id": { "type": "keyword" }, + "cpu_usage_nsec": { "type": "long" }, + "device": { "type": "keyword" }, + "glib_domain": { "type": "keyword" }, + "glib_old_log_api": { "type": "keyword" }, "hostname": { "type": "keyword" }, "invocation_id": { "type": "keyword" }, "job_id": { "type": "integer" }, "job_result": { "type": "keyword" }, "job_type": { "type": "keyword" }, + "kernel_device": { "type": "keyword" }, + "kernel_subsystem": { "type": "keyword" }, + "leader": { "type": "keyword" }, "machine_id": { "type": "keyword" }, "message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "message_id": { "type": "keyword" }, @@ -110,6 +119,7 @@ "priority": { "type": "integer" }, "runtime_scope": { "type": "keyword" }, "selinux_context": { "type": "keyword" }, + "session_id": { "type": "keyword" }, "source_monotonic_timestamp": { "type": "long" }, "source_realtime_timestamp" : { "type": "date" }, "stream_id": { "type": "keyword" }, @@ -124,7 +134,15 @@ "systemd_user_unit": { "type": "keyword" }, "timestamp_boottime": { "type": "float" }, "timestamp_monotonic": { "type": "float" }, - "unit": { "type": "keyword" } + "udev_devnode": { "type": "keyword" }, + "udev_sysname": { "type": "keyword" }, + "unit": { "type": "keyword" }, + "user_id": { "type": "keyword" }, + "user_invocation_id": { "type": "keyword" }, + "user_unit": { "type": "keyword" }, + "userspace_usec": { "type": "integer" }, + "wp_object": { "type": "keyword" }, + "wp_object_type": { "type": "keyword" } } }, "thermal": { diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 5c1f6ed4b..e991be9f8 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -548,6 +548,7 @@ filter { if ([miscbeat][systemd]) { #------------------------------------------------- # systemd - https://docs.fluentbit.io/manual/pipeline/inputs/systemd + # see also - https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html if ([miscbeat][systemd][source_realtime_timestamp]) { # microseconds to milliseconds @@ -606,6 +607,12 @@ filter { rename => { "[miscbeat][systemd][message]" => "[event][original]" } } } + # + if ([miscbeat][systemd][errno]) { + mutate { id => "mutate_rename_miscbeat_systemd_errno" + rename => { "[miscbeat][systemd][errno]" => "[event][result]" } } + } + if (![event][hash]) { fingerprint { id => "fingerprint_malcolm_miscbeat_systemd" From bea5499591c955f7dbd069eba1966b883e038dec Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 12 Dec 2023 15:56:52 -0700 Subject: [PATCH 51/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../db0312a0-e842-11ec-88e0-67bf497b7120.json | 212 ------------------ 1 file changed, 212 deletions(-) delete mode 100644 dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json diff --git a/dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json b/dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json deleted file mode 100644 index 06ff9ba5b..000000000 --- a/dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "version": "2.0.0", - "objects": [ - { - "id": "db0312a0-e842-11ec-88e0-67bf497b7120", - "type": "dashboard", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T23:00:39.461Z", - "version": "WzkyMywxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog", - "hits": 0, - "description": "Syslog logs from Malcolm sensors", - "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":9,\"h\":19,\"i\":\"e1d6d345-c417-4f95-8907-17806a08cc34\"},\"panelIndex\":\"e1d6d345-c417-4f95-8907-17806a08cc34\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":9,\"y\":0,\"w\":9,\"h\":19,\"i\":\"ec19724f-f84a-4dde-acff-f539df5afef4\"},\"panelIndex\":\"ec19724f-f84a-4dde-acff-f539df5afef4\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":15,\"h\":19,\"i\":\"ff1d5bec-24fd-43b1-8a7b-e1b6942aa306\"},\"panelIndex\":\"ff1d5bec-24fd-43b1-8a7b-e1b6942aa306\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":33,\"y\":0,\"w\":15,\"h\":19,\"i\":\"6874104c-2a3e-432f-9cf1-3324dce134c9\"},\"panelIndex\":\"6874104c-2a3e-432f-9cf1-3324dce134c9\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":19,\"w\":48,\"h\":19,\"i\":\"f8d075cc-f2fc-4da7-8c0b-c1e691386eaf\"},\"panelIndex\":\"f8d075cc-f2fc-4da7-8c0b-c1e691386eaf\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]", - "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", - "version": 1, - "timeRestore": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" - } - }, - "references": [ - { - "name": "panel_0", - "type": "visualization", - "id": "6ccdb970-e842-11ec-88e0-67bf497b7120" - }, - { - "name": "panel_1", - "type": "visualization", - "id": "a6549790-e842-11ec-88e0-67bf497b7120" - }, - { - "name": "panel_2", - "type": "visualization", - "id": "42bd0a50-e847-11ec-8b84-87a2d75f23b1" - }, - { - "name": "panel_3", - "type": "visualization", - "id": "8e4a7890-e847-11ec-8b84-87a2d75f23b1" - }, - { - "name": "panel_4", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "dashboard": "7.9.3" - } - }, - { - "id": "6ccdb970-e842-11ec-88e0-67bf497b7120", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:51:27.963Z", - "version": "WzkxMywxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Host", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Host\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "a6549790-e842-11ec-88e0-67bf497b7120", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:51:27.963Z", - "version": "WzkxNCwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Program", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Program\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.appname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":200,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Program\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "42bd0a50-e847-11ec-8b84-87a2d75f23b1", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:55:30.164Z", - "version": "WzkyMSwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Severity", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Severity\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.severity.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":8,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "8e4a7890-e847-11ec-8b84-87a2d75f23b1", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:57:36.921Z", - "version": "WzkyMiwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Facility", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Facility\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.facility.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Facility\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.facility.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Facility\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "37139ac0-e842-11ec-88e0-67bf497b7120", - "type": "search", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:51:27.963Z", - "version": "WzkxNSwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Logs", - "description": "", - "hits": 0, - "columns": [ - "log.syslog.hostname", - "log.syslog.severity.name", - "log.syslog.facility.name", - "log.syslog.appname", - "event.original" - ], - "sort": [], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"log.syslog:*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "malcolm_beats_*" - } - ], - "migrationVersion": { - "search": "7.9.3" - } - } - ] -} \ No newline at end of file From 8c7e04dd7c054c719cd4ea339d9b1002cebb66bd Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 12 Dec 2023 16:02:14 -0700 Subject: [PATCH 52/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../systemd/user/systemd-localhost-malcolm.service | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service new file mode 100644 index 000000000..ed79496b0 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i systemd -p Read_From_Tail=On -p Lowercase=On -p Strip_Underscores=On -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=systemd -p WildCard='*' -m '*' -F record_modifier -p 'Record=module systemd' -m '*' -F modify -p 'Add=host.name %H' -m '*' -F nest -p 'Operation=nest' -p 'Wildcard=host.*' -p 'Nest_under=host' -p 'Remove_prefix=host.' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target From 7f06a8be6560ad4568ee25a697431729a892fe4a Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 12 Dec 2023 16:14:47 -0700 Subject: [PATCH 53/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../f6600310-9943-11ee-a029-e973f4774355.json | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json new file mode 100644 index 000000000..5136f7308 --- /dev/null +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -0,0 +1,76 @@ +{ + "version": "2.11.1", + "objects": [ + { + "id": "f6600310-9943-11ee-a029-e973f4774355", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-12T23:13:35.905Z", + "version": "Wzk2NCwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald Logs", + "hits": 0, + "description": "", + "panelsJSON": "[{\"version\":\"2.11.1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":32,\"i\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\"},\"panelIndex\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"}]", + "optionsJSON": "{\"useMargins\":true,\"hidePanelTitles\":false}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "c79c1e60-9943-11ee-a029-e973f4774355", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-12T23:11:28.581Z", + "version": "Wzk2MiwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Logs", + "description": "", + "hits": 0, + "columns": [ + "@timestamp", + "miscbeat.systemd.hostname", + "process.name", + "process.pid", + "process.user.id", + "event.original", + "miscbeat.systemd.systemd_unit", + "miscbeat.systemd.systemd_user_unit", + "log.logger" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.systemd:*\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1w\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file From 5d7503b442f2899b4876c8e4453593f07b701f8d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 13 Dec 2023 10:41:42 -0700 Subject: [PATCH 54/78] only overwrite Arkime's config.ini with config.orig.ini config.ini doesn't already exist idaholab/Malcolm#311 --- arkime/scripts/docker_entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 1bcd41415..5b8abbd58 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -50,7 +50,8 @@ if ( [[ "$OPENSEARCH_PRIMARY" == "opensearch-remote" ]] || [[ "$OPENSEARCH_PRIMA OPENSEARCH_URL_FINAL="${PROTOCOL}${NEW_USER}:${NEW_PASSWORD}@${HOSTPORT}" fi -if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then +# iff config.ini does not exist but config.orig.ini does, use it as a basis and modify based on env. vars +if [[ ! -f "${ARKIME_CONFIG_FILE}" ]] && [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_CONFIG_FILE}" sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_CONFIG_FILE}" From 517c2b4c3a7661456d9acb5698c931e5d750d93c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 13 Dec 2023 12:28:51 -0700 Subject: [PATCH 55/78] work on idaholab/Malcolm#309, rsyslog no longer in bookworm --- .../f6600310-9943-11ee-a029-e973f4774355.json | 189 +++++++++++++++++- 1 file changed, 182 insertions(+), 7 deletions(-) diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json index 5136f7308..e623a6bb7 100644 --- a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -7,23 +7,48 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-12T23:13:35.905Z", - "version": "Wzk2NCwxXQ==", + "updated_at": "2023-12-13T18:46:18.238Z", + "version": "Wzk2MSwxXQ==", "attributes": { "title": "Malcolm Sensor Journald Logs", "hits": 0, "description": "", - "panelsJSON": "[{\"version\":\"2.11.1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":32,\"i\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\"},\"panelIndex\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"}]", - "optionsJSON": "{\"useMargins\":true,\"hidePanelTitles\":false}", + "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"gridData\":{\"h\":19,\"i\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"w\":11,\"x\":14,\"y\":0},\"panelIndex\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"w\":23,\"x\":25,\"y\":0},\"panelIndex\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"w\":24,\"x\":0,\"y\":19},\"panelIndex\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"w\":24,\"x\":24,\"y\":19},\"panelIndex\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":32,\"i\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"w\":48,\"x\":0,\"y\":37},\"panelIndex\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_5\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" } }, "references": [ { "name": "panel_0", + "type": "visualization", + "id": "82e154f0-99e5-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "5a3cb5c0-99e6-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "02c189b0-99e6-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_3", + "type": "visualization", + "id": "81e6b660-99e7-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_4", + "type": "visualization", + "id": "9601eb20-99e6-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_5", "type": "search", "id": "c79c1e60-9943-11ee-a029-e973f4774355" } @@ -32,14 +57,164 @@ "dashboard": "7.9.3" } }, + { + "id": "82e154f0-99e5-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-13T18:29:11.742Z", + "version": "Wzk1MSwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Logger", + "visState": "{\"title\":\"Malcolm Sensor Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "5a3cb5c0-99e6-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-13T18:35:13.052Z", + "version": "Wzk1NCwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Process UID", + "visState": "{\"title\":\"Malcolm Sensor Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "02c189b0-99e6-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-13T18:32:46.283Z", + "version": "Wzk1MywxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Logs by Host Over Time", + "visState": "{\"title\":\"Malcolm Sensor Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "81e6b660-99e7-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-13T18:46:02.857Z", + "version": "Wzk2MCwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Systemd Unit", + "visState": "{\"title\":\"Malcolm Sensor Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "9601eb20-99e6-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-13T18:44:25.664Z", + "version": "Wzk1OSwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Process Name", + "visState": "{\"title\":\"Malcolm Sensor Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, { "id": "c79c1e60-9943-11ee-a029-e973f4774355", "type": "search", "namespaces": [ "default" ], - "updated_at": "2023-12-12T23:11:28.581Z", - "version": "Wzk2MiwxXQ==", + "updated_at": "2023-12-13T18:11:33.018Z", + "version": "Wzk0MCwxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Logs", "description": "", From e5f63bf35c996c178f28628a29e13578a7e171ec Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 13 Dec 2023 12:33:49 -0700 Subject: [PATCH 56/78] idaholab/Malcolm#251; include CVE-2023-28771 rule based on Zyxel SektorCERT Report --- .../OT/malcolm/CVE-2023-28771_Zyxel.rules | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules diff --git a/suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules b/suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules new file mode 100644 index 000000000..0d633a768 --- /dev/null +++ b/suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules @@ -0,0 +1,17 @@ +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show users"; nocase; sid:1000001; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show sessions"; nocase; sid:1000002; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show configuration"; nocase; sid:1000003; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show running-config"; nocase; sid:1000004; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show firewall rule"; nocase; sid:1000005; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"export config"; nocase; sid:1000006; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/mipskiller"; sid:1000007; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/mipskiller"; sid:1000008; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1000009; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1000009; rev:1;) +alert tcp any any -> any 8081 (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1000010; rev:1;) +alert tcp any any -> any 82 (msg: "Potential Zyxel Payload connection"; content:"/fuckjewishpeople.mips"; sid:1000011; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mips"; sid:1000012; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mpsl"; sid:1000013; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/paraiso.mips"; sid:1000014; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/libcurl1337.mips"; sid:1000015; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy1"; sid:1000016; rev:1;) \ No newline at end of file From ca41a4e7fd8dd9048d282ea21af8055c8c22ab8f Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 13 Dec 2023 15:13:51 -0700 Subject: [PATCH 57/78] improvements to suricata custom rule handling, and added SURICATA_DISABLE_ICS_ALL variable --- ...sensor-iso-build-docker-wrap-push-ghcr.yml | 4 +- Dockerfiles/suricata.Dockerfile | 9 +- config/suricata.env.example | 1 + docs/malcolm-config.md | 1 + ...ilebeat-clean-zeeklogs-processed-folder.py | 1 - scripts/install.py | 16 ++-- sensor-iso/build.sh | 3 +- sensor-iso/build_via_vagrant.sh | 5 +- .../interface/sensor_ctl/control_vars.conf | 1 + .../suricata_config_populate.sh | 2 + sensor-iso/interface/sensor_ctl/supervisor.sh | 6 +- shared/bin/sensor-init.sh | 5 +- shared/bin/suricata_config_populate.py | 87 ++++++++++++++----- shared/bin/suricata_update_config_populate.py | 4 +- .../OT/malcolm/CVE-2023-28771_Zyxel.rules | 17 ---- .../IT/.gitignore | 0 .../OT/.gitignore | 0 .../OT/malcolm/CVE-2023-28771_Zyxel.rules | 14 +++ .../AllenBradley_RockwellAutomation.rules | 0 .../SchweitzerEngineeringLaboratories.rules | 0 .../OT/nsacyber/ELITEWOLF/Siemens.rules | 0 21 files changed, 120 insertions(+), 56 deletions(-) delete mode 100644 suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules rename suricata/{default-rules => rules-default}/IT/.gitignore (100%) rename suricata/{default-rules => rules-default}/OT/.gitignore (100%) create mode 100644 suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules rename suricata/{default-rules => rules-default}/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules (100%) rename suricata/{default-rules => rules-default}/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules (100%) rename suricata/{default-rules => rules-default}/OT/nsacyber/ELITEWOLF/Siemens.rules (100%) diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml index db374d6f2..d5ce7f2b2 100644 --- a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml @@ -97,6 +97,8 @@ jobs: cp ./scripts/malcolm_utils.py ./sensor-iso/shared/bin/ cp ./scripts/documentation_build.sh ./sensor-iso/docs/ cp -r ./arkime/patch ./sensor-iso/shared/arkime_patch + mkdir -p ./sensor-iso/suricata + cp -r ./suricata/rules-default ./sensor-iso/suricata/ pushd ./sensor-iso echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt @@ -104,7 +106,7 @@ jobs: echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot echo "BUILD_JOBS=2" > ./shared/environment.chroot sudo /usr/bin/env bash ./build.sh - rm -rf ./shared/ ./docs/ ./_config.yml ./_includes ./_layouts /Gemfile ./README.md + rm -rf ./shared/ ./docs/ ./_config.yml ./_includes ./_layouts /Gemfile ./README.md ./suricata sudo chmod 644 ./hedgehog-*.* popd - diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index fdf5a5d68..c79a5a9d4 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -42,6 +42,7 @@ ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_ ENV SURICATA_CONFIG_DIR /etc/suricata ENV SURICATA_CONFIG_FILE "$SURICATA_CONFIG_DIR"/suricata.yaml ENV SURICATA_CUSTOM_RULES_DIR /opt/suricata/rules +ENV SURICATA_DEFAULT_RULES_DIR /opt/suricata/rules-default ENV SURICATA_CUSTOM_CONFIG_DIR /opt/suricata/include-configs ENV SURICATA_LOG_DIR /var/log/suricata ENV SURICATA_MANAGED_DIR /var/lib/suricata @@ -52,8 +53,6 @@ ENV SURICATA_UPDATE_DIR "$SURICATA_MANAGED_DIR/update" ENV SURICATA_UPDATE_SOURCES_DIR "$SURICATA_UPDATE_DIR/sources" ENV SURICATA_UPDATE_CACHE_DIR "$SURICATA_UPDATE_DIR/cache" -COPY --chmod=644 suricata/default-rules/ /tmp/default-rules/ - RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sources && \ apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ @@ -116,11 +115,10 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour usermod -a -G tty ${PUSER} && \ ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_suricata_processor.py && \ (echo "*/5 * * * * /usr/local/bin/eve-clean-logs.sh\n0 */6 * * * /bin/bash /usr/local/bin/suricata-update-rules.sh\n" > ${SUPERCRONIC_CRONTAB}) && \ - mkdir -p "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ - chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ + mkdir -p "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_DEFAULT_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ + chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_DEFAULT_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ cp "$(dpkg -L suricata-update | grep 'update\.yaml$' | head -n 1)" \ "$SURICATA_UPDATE_CONFIG_FILE" && \ - find /tmp/default-rules/ -not -path '*/.gitignore' -type f -exec cp "{}" "$SURICATA_CONFIG_DIR"/rules/ \; && \ suricata-update update-sources --verbose --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ suricata-update update --fail --verbose --etopen --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ cp /usr/bin/suricata /usr/bin/suricata-offline && \ @@ -140,6 +138,7 @@ COPY --chmod=755 shared/bin/suricata_config_populate.py /usr/local/bin/ COPY --chmod=755 suricata/scripts/docker_entrypoint.sh /usr/local/bin/ COPY --chmod=755 suricata/scripts/eve-clean-logs.sh /usr/local/bin/ COPY --chmod=755 suricata/scripts/suricata-update-rules.sh /usr/local/bin/ +COPY --chmod=u=rwX,go=rX suricata/rules-default/ "$SURICATA_DEFAULT_RULES_DIR"/ ARG PCAP_PIPELINE_VERBOSITY="" ARG PCAP_MONITOR_HOST=pcap-monitor diff --git a/config/suricata.env.example b/config/suricata.env.example index aada5d39a..d652854ae 100644 --- a/config/suricata.env.example +++ b/config/suricata.env.example @@ -3,6 +3,7 @@ SURICATA_CUSTOM_RULES_ONLY=false SURICATA_UPDATE_RULES=false SURICATA_UPDATE_DEBUG=false SURICATA_UPDATE_ETOPEN=true +SURICATA_DISABLE_ICS_ALL=false # suricata_config_populate.py can use MANY more environment variables to tweak # suricata.yaml (see https://github.com/OISF/suricata/blob/master/suricata.yaml.in and # https://suricata.readthedocs.io/en/latest/configuration/suricata-yaml.html). diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 2df1292a4..94ee44959 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -69,6 +69,7 @@ Although the configuration script automates many of the following configuration - `SURICATA_UPDATE_RULES` – if set to `true`, Suricata signatures will periodically be updated (default `false`) - `SURICATA_LIVE_CAPTURE` - if set to `true`, Suricata will monitor live traffic on the local interface(s) defined by `PCAP_FILTER` - `SURICATA_ROTATED_PCAP` - if set to `true`, Suricata can analyze PCAP files captured by `netsniff-ng` or `tcpdump` (see `PCAP_ENABLE_NETSNIFF` and `PCAP_ENABLE_TCPDUMP`, as well as `SURICATA_AUTO_ANALYZE_PCAP_FILES`); if `SURICATA_LIVE_CAPTURE` is `true`, this should be `false`; otherwise Suricata will see duplicate traffic + - `SURICATA_DISABLE_ICS_ALL` - if set to `true`, this variable can be used to disable Malcolm's [built-in Suricata rules for Operational Technology/Industrial Control Systems (OT/ICS) vulnerabilities and exploits]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/suricata/rules-default/OT) - `SURICATA_…` - the [`suricata` container entrypoint script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/suricata_config_populate.py) can use **many** more environment variables to tweak [suricata.yaml](https://github.com/OISF/suricata/blob/master/suricata.yaml.in); in that script, `DEFAULT_VARS` defines those variables (albeit without the `SURICATA_` prefix you must add to each for use) Note that for some variables (e.g., something with a sequence like `HOME_NET`) Suricata wants values to be quoted. To accomplish that in the `suricata.env` file, use outer single quotes with inner double quotes, like this: + `SURICATA_HOME_NET='"[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"'` * **`upload-common.env`** - settings for dealing with PCAP files [uploaded](upload.md#Upload) to Malcolm for analysis diff --git a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py index 3c2a38f9e..2484f9c39 100755 --- a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py +++ b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py @@ -9,7 +9,6 @@ import errno import time import fcntl -import fnmatch import magic import json import pprint diff --git a/scripts/install.py b/scripts/install.py index 6a6e37d08..030bcfea4 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1070,14 +1070,14 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) autoZeek = InstallerYesOrNo('Automatically analyze all PCAP files with Zeek?', default=args.autoZeek) - zeekIcs = InstallerYesOrNo( + malcolmIcs = InstallerYesOrNo( 'Is Malcolm being used to monitor an Operational Technology/Industrial Control Systems (OT/ICS) network?', - default=args.zeekIcs, + default=args.malcolmIcs, ) zeekICSBestGuess = ( autoZeek - and zeekIcs + and malcolmIcs and InstallerYesOrNo( 'Should Malcolm use "best guess" to identify potential OT/ICS traffic with Zeek?', default=args.zeekICSBestGuess, @@ -1717,6 +1717,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'SURICATA_UPDATE_RULES', TrueOrFalseNoQuote(suricataRuleUpdate), ), + # disable/enable ICS analyzers + EnvValue( + os.path.join(args.configDir, 'suricata.env'), + 'SURICATA_DISABLE_ICS_ALL', + TrueOrFalseNoQuote(not malcolmIcs), + ), # live traffic analysis with Suricata EnvValue( os.path.join(args.configDir, 'suricata-live.env'), @@ -1805,7 +1811,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): EnvValue( os.path.join(args.configDir, 'zeek.env'), 'ZEEK_DISABLE_ICS_ALL', - '' if zeekIcs else TrueOrFalseNoQuote(not zeekIcs), + '' if malcolmIcs else TrueOrFalseNoQuote(not malcolmIcs), ), # disable/enable ICS best guess EnvValue( @@ -3610,7 +3616,7 @@ def main(): ) analysisArgGroup.add_argument( '--zeek-ics', - dest='zeekIcs', + dest='malcolmIcs', type=str2bool, metavar="true|false", nargs='?', diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index 29bd10f03..85630c9ab 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -106,12 +106,13 @@ if [ -d "$WORKDIR" ]; then # replace capture interface for now, it'll need to be automatically detected/configured on boot sed -i "s/CAPTURE_INTERFACE=.*/CAPTURE_INTERFACE=xxxx/g" ./config/includes.chroot/opt/sensor/sensor_ctl/control_vars.conf - # copy shared scripts + # copy shared scripts and files rsync -a "$SCRIPT_PATH/shared/bin/" ./config/includes.chroot/usr/local/bin/ mkdir -p ./config/includes.chroot/opt/zeek/bin/ mv ./config/includes.chroot/usr/local/bin/zeekdeploy.sh ./config/includes.chroot/opt/zeek/bin/ ln -s -r ./config/includes.chroot/usr/local/bin/malcolm_utils.py ./config/includes.chroot/opt/zeek/bin/ chown -R root:root ./config/includes.chroot/usr/local/bin/ ./config/includes.chroot/opt/zeek/bin/ + rsync -a "$SCRIPT_PATH/suricata/" ./config/includes.chroot/opt/sensor/sensor_ctl/suricata/ # write out some version stuff specific to this installation version echo "BUILD_ID=\"$(date +'%Y-%m-%d')-${IMAGE_VERSION}\"" > ./config/includes.chroot/opt/sensor/.os-info diff --git a/sensor-iso/build_via_vagrant.sh b/sensor-iso/build_via_vagrant.sh index 7fd7fd43d..fa9a2f84a 100755 --- a/sensor-iso/build_via_vagrant.sh +++ b/sensor-iso/build_via_vagrant.sh @@ -28,7 +28,8 @@ function cleanup_shared_and_docs { "$SCRIPT_PATH"/_includes \ "$SCRIPT_PATH"/_layouts \ "$SCRIPT_PATH"/Gemfile \ - "$SCRIPT_PATH"/README.md + "$SCRIPT_PATH"/README.md \ + "$SCRIPT_PATH"/suricata } unset FORCE_PROVISION @@ -87,6 +88,8 @@ cp -r "$SCRIPT_PATH"/../shared \ "$SCRIPT_PATH"/../README.md "$SCRIPT_PATH"/ cp "$SCRIPT_PATH"/../scripts/documentation_build.sh "$SCRIPT_PATH"/docs/ cp "$SCRIPT_PATH"/../scripts/malcolm_utils.py "$SCRIPT_PATH"/shared/bin/ +mkdir "$SCRIPT_PATH"/suricata +cp -r "$SCRIPT_PATH"/../suricata/rules-default "$SCRIPT_PATH"/suricata/ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose-standalone.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 67a48f3dc..d821c64d8 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -89,6 +89,7 @@ export ZEEK_DISABLE_BEST_GUESS_ICS=true # Suricata export SURICATA_CUSTOM_RULES_ONLY=false +export SURICATA_DISABLE_ICS_ALL=false export SURICATA_RUNMODE=workers export SURICATA_AF_PACKET_BLOCK_SIZE=32768 export SURICATA_AF_PACKET_BLOCK_TIMEOUT=10 diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh index 96473ab4b..63f020636 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh @@ -12,8 +12,10 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r /usr/local/bin/suricata_config_populate.py [[ ! -f "$SUPERVISOR_PATH"/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" "$SUPERVISOR_PATH"/suricata/update.yaml # specify the custom rules and configuration directories relative to the supervisor path + SURICATA_DEFAULT_RULES_DIR="$SUPERVISOR_PATH"/suricata/rules-default SURICATA_CUSTOM_RULES_DIR="$SUPERVISOR_PATH"/suricata/rules SURICATA_CUSTOM_CONFIG_DIR="$SUPERVISOR_PATH"/suricata/include-configs + [[ -d "$SURICATA_DEFAULT_RULES_DIR" ]] && export SURICATA_DEFAULT_RULES_DIR [[ -d "$SURICATA_CUSTOM_RULES_DIR" ]] && export SURICATA_CUSTOM_RULES_DIR [[ -d "$SURICATA_CUSTOM_CONFIG_DIR" ]] && export SURICATA_CUSTOM_CONFIG_DIR diff --git a/sensor-iso/interface/sensor_ctl/supervisor.sh b/sensor-iso/interface/sensor_ctl/supervisor.sh index cfe2562c5..933baf5cf 100755 --- a/sensor-iso/interface/sensor_ctl/supervisor.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.sh @@ -81,7 +81,11 @@ mkdir -p "$SUPERVISOR_PATH/"{log,run} rm -f "$SUPERVISOR_PATH/"/log/* rm -rf /opt/sensor/sensor_ctl/zeek/intel/lock || true -mkdir -p "$SUPERVISOR_PATH"/suricata/rules "$SUPERVISOR_PATH"/suricata/include-configs "$ZEEK_LOG_PATH"/suricata 2>/dev/null || true +mkdir -p "$SUPERVISOR_PATH"/suricata/rules \ + "$SUPERVISOR_PATH"/suricata/rules-default/OT \ + "$SUPERVISOR_PATH"/suricata/rules-default/IT \ + "$SUPERVISOR_PATH"/suricata/include-configs \ + "$ZEEK_LOG_PATH"/suricata 2>/dev/null || true mkdir -p "$PCAP_PATH"/ 2>/dev/null || true mkdir -p "$SUPERVISOR_PATH"/supercronic 2>/dev/null && touch "$SUPERVISOR_PATH"/supercronic/crontab || true diff --git a/shared/bin/sensor-init.sh b/shared/bin/sensor-init.sh index 4deda1256..0f0ddaaa0 100755 --- a/shared/bin/sensor-init.sh +++ b/shared/bin/sensor-init.sh @@ -66,7 +66,10 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then if dpkg -s suricata >/dev/null 2>&1 ; then mkdir -p /etc/suricata/rules /var/log/suricata /var/lib/suricata/rules if [[ -d /opt/sensor/sensor_ctl ]]; then - mkdir -p /opt/sensor/sensor_ctl/suricata/rules /opt/sensor/sensor_ctl/suricata/include-configs + mkdir -p /opt/sensor/sensor_ctl/suricata/rules \ + /opt/sensor/sensor_ctl/suricata/rules-default/OT \ + /opt/sensor/sensor_ctl/suricata/rules-default/IT \ + /opt/sensor/sensor_ctl/suricata/include-configs [[ ! -f /opt/sensor/sensor_ctl/suricata/suricata.yaml ]] && cp /etc/suricata/suricata.yaml /opt/sensor/sensor_ctl/suricata/suricata.yaml [[ ! -f /opt/sensor/sensor_ctl/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" /opt/sensor/sensor_ctl/suricata/update.yaml fi diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index eeb870077..8fa6ca445 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -13,7 +13,7 @@ import argparse import contextlib -import fnmatch +import glob import logging import os import sys @@ -503,39 +503,83 @@ def __call__(self, repr, data): ################################################################################################### -def GetRuleSources(requireRulesExist=False): +def GetRuleFiles(): global DEFAULT_VARS - ruleSources = [] + ruleFiles = [] if not val2bool(DEFAULT_VARS['CUSTOM_RULES_ONLY']): - ruleSources.append('suricata.rules') + # built-in suricata rules + ruleFiles.append('suricata.rules') + + # Malcolm's default IT rules + ruleFiles.extend( + sorted( + list( + glob.iglob( + os.path.join( + os.path.join(os.path.join(DEFAULT_VARS['DEFAULT_RULES_DIR'], 'IT'), '**'), '*.rules' + ), + recursive=True, + ) + ) + ) + if os.path.isdir(str(DEFAULT_VARS['DEFAULT_RULES_DIR'])) + else [] + ) + + # Malcolm's default OT rules + ruleFiles.extend( + sorted( + list( + glob.iglob( + os.path.join( + os.path.join(os.path.join(DEFAULT_VARS['DEFAULT_RULES_DIR'], 'OT'), '**'), '*.rules' + ), + recursive=True, + ) + ) + ) + if ( + os.path.isdir(str(DEFAULT_VARS['DEFAULT_RULES_DIR'])) + and (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) + ) + else [] + ) - customRuleFiles = ( - fnmatch.filter(os.listdir(DEFAULT_VARS['CUSTOM_RULES_DIR']), '*.rules') - if DEFAULT_VARS['CUSTOM_RULES_DIR'] is not None + # User's custom rules + ruleFiles.extend( + sorted( + list( + glob.iglob( + os.path.join(os.path.join(DEFAULT_VARS['CUSTOM_RULES_DIR'], '**'), '*.rules'), + recursive=True, + ) + ) + ) + if os.path.isdir(str(DEFAULT_VARS['CUSTOM_RULES_DIR'])) else [] ) - if (DEFAULT_VARS['CUSTOM_RULES_DIR'] is not None) and ((not requireRulesExist) or (len(customRuleFiles) > 0)): - ruleSources.append(os.path.join(DEFAULT_VARS['CUSTOM_RULES_DIR'], '*.rules')) - - return ruleSources + return ruleFiles ################################################################################################### def GetIncludeConfigSources(): global DEFAULT_VARS - configSources = list( - [ - os.path.join(DEFAULT_VARS['CUSTOM_CONFIG_DIR'], x) - for x in fnmatch.filter(os.listdir(DEFAULT_VARS['CUSTOM_CONFIG_DIR']), '*.yaml') - ] - if DEFAULT_VARS['CUSTOM_CONFIG_DIR'] is not None + configSources = ( + sorted( + list( + glob.iglob( + os.path.join(os.path.join(DEFAULT_VARS['CUSTOM_CONFIG_DIR'], '**'), '*.yaml'), + recursive=True, + ) + ) + ) + if os.path.isdir(str(DEFAULT_VARS['CUSTOM_CONFIG_DIR'])) else [] ) - return configSources @@ -627,7 +671,8 @@ def main(): if os.path.isfile(args.output) and os.path.samefile(args.input, args.output): backupFile = inFileParts[0] + "_bak_" + str(int(round(time.time()))) + inFileParts[1] CopyFile(args.input, backupFile) - backupFiles = sorted(fnmatch.filter(os.listdir(os.path.dirname(backupFile)), '*_bak_*')) + backupFiles = sorted(list(glob.glob(os.path.join(os.path.dirname(backupFile), '*_bak_*')))) + while len(backupFiles) > BACKUP_FILES_MAX: toDeleteFileName = os.path.join(os.path.dirname(backupFile), backupFiles.pop(0)) logging.debug(f'Removing old backup file "{toDeleteFileName}"') @@ -1080,7 +1125,7 @@ def main(): deep_set(cfg, ['stats', 'enabled'], True) cfg.pop('rule-files', None) - deep_set(cfg, ['rule-files'], GetRuleSources(requireRulesExist=True)) + deep_set(cfg, ['rule-files'], GetRuleFiles()) # Hackety-hack, don't talk back! Despite the "Including multiple files" section of # https://docs.suricata.io/en/latest/configuration/includes.html#including-multiple-files @@ -1127,7 +1172,7 @@ def main(): # final tweaks deep_set(cfg, ['stats', 'enabled'], False) cfg.pop('rule-files', None) - deep_set(cfg, ['rule-files'], GetRuleSources(requireRulesExist=False)) + deep_set(cfg, ['rule-files'], GetRuleFiles()) # see note on 'include' above cfg.pop('include', None) diff --git a/shared/bin/suricata_update_config_populate.py b/shared/bin/suricata_update_config_populate.py index 529bfce14..52cf643f8 100755 --- a/shared/bin/suricata_update_config_populate.py +++ b/shared/bin/suricata_update_config_populate.py @@ -13,7 +13,7 @@ import argparse import contextlib -import fnmatch +import glob import logging import os import sys @@ -157,7 +157,7 @@ def main(): if os.path.isfile(args.output) and os.path.samefile(args.input, args.output): backupFile = inFileParts[0] + "_bak_" + str(int(round(time.time()))) + inFileParts[1] CopyFile(args.input, backupFile) - backupFiles = sorted(fnmatch.filter(os.listdir(os.path.dirname(backupFile)), '*_bak_*')) + backupFiles = sorted(list(glob.glob(os.path.join(os.path.dirname(backupFile), '*_bak_*')))) while len(backupFiles) > BACKUP_FILES_MAX: toDeleteFileName = os.path.join(os.path.dirname(backupFile), backupFiles.pop(0)) logging.debug(f'Removing old backup file "{toDeleteFileName}"') diff --git a/suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules b/suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules deleted file mode 100644 index 0d633a768..000000000 --- a/suricata/default-rules/OT/malcolm/CVE-2023-28771_Zyxel.rules +++ /dev/null @@ -1,17 +0,0 @@ -alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show users"; nocase; sid:1000001; rev:1;) -alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show sessions"; nocase; sid:1000002; rev:1;) -alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show configuration"; nocase; sid:1000003; rev:1;) -alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show running-config"; nocase; sid:1000004; rev:1;) -alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show firewall rule"; nocase; sid:1000005; rev:1;) -alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"export config"; nocase; sid:1000006; rev:1;) -alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/mipskiller"; sid:1000007; rev:1;) -alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/mipskiller"; sid:1000008; rev:1;) -alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1000009; rev:1;) -alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1000009; rev:1;) -alert tcp any any -> any 8081 (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1000010; rev:1;) -alert tcp any any -> any 82 (msg: "Potential Zyxel Payload connection"; content:"/fuckjewishpeople.mips"; sid:1000011; rev:1;) -alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mips"; sid:1000012; rev:1;) -alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mpsl"; sid:1000013; rev:1;) -alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/paraiso.mips"; sid:1000014; rev:1;) -alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/libcurl1337.mips"; sid:1000015; rev:1;) -alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy1"; sid:1000016; rev:1;) \ No newline at end of file diff --git a/suricata/default-rules/IT/.gitignore b/suricata/rules-default/IT/.gitignore similarity index 100% rename from suricata/default-rules/IT/.gitignore rename to suricata/rules-default/IT/.gitignore diff --git a/suricata/default-rules/OT/.gitignore b/suricata/rules-default/OT/.gitignore similarity index 100% rename from suricata/default-rules/OT/.gitignore rename to suricata/rules-default/OT/.gitignore diff --git a/suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules b/suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules new file mode 100644 index 000000000..4d1d2db9a --- /dev/null +++ b/suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules @@ -0,0 +1,14 @@ +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show users"; nocase; sid:1001001; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show sessions"; nocase; sid:1001002; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show configuration"; nocase; sid:1001003; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show running-config"; nocase; sid:1001004; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show firewall rule"; nocase; sid:1001005; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"export config"; nocase; sid:1001006; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/mipskiller"; sid:1001007; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1001008; rev:1;) +alert tcp any any -> any 82 (msg: "Potential Zyxel Payload connection"; content:"/fuckjewishpeople.mips"; sid:1001009; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mips"; sid:1001010; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mpsl"; sid:1001011; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/paraiso.mips"; sid:1001012; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/libcurl1337.mips"; sid:1001013; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy1"; sid:1001014; rev:1;) \ No newline at end of file diff --git a/suricata/default-rules/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules b/suricata/rules-default/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules similarity index 100% rename from suricata/default-rules/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules rename to suricata/rules-default/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules diff --git a/suricata/default-rules/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules b/suricata/rules-default/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules similarity index 100% rename from suricata/default-rules/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules rename to suricata/rules-default/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules diff --git a/suricata/default-rules/OT/nsacyber/ELITEWOLF/Siemens.rules b/suricata/rules-default/OT/nsacyber/ELITEWOLF/Siemens.rules similarity index 100% rename from suricata/default-rules/OT/nsacyber/ELITEWOLF/Siemens.rules rename to suricata/rules-default/OT/nsacyber/ELITEWOLF/Siemens.rules From b50b7ddb2a7153519669be2c054d3816476d3a9c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 13 Dec 2023 15:50:58 -0700 Subject: [PATCH 58/78] disable ICS based on variables --- shared/bin/suricata_config_populate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index 8fa6ca445..50c365304 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -271,8 +271,8 @@ def __call__(self, repr, data): ), 'dnp3': ProtocolConfig( [], - val2bool(DEFAULT_VARS['DNP3_ENABLED']), - val2bool(DEFAULT_VARS['DNP3_EVE_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['DNP3_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['DNP3_EVE_ENABLED']), True, DEFAULT_VARS['DNP3_PORTS'], None, @@ -287,8 +287,8 @@ def __call__(self, repr, data): ), 'enip': ProtocolConfig( [], - val2bool(DEFAULT_VARS['ENIP_ENABLED']), - val2bool(DEFAULT_VARS['ENIP_EVE_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['ENIP_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['ENIP_EVE_ENABLED']), False, DEFAULT_VARS['ENIP_PORTS'], DEFAULT_VARS['ENIP_PORTS'], @@ -359,8 +359,8 @@ def __call__(self, repr, data): ), 'modbus': ProtocolConfig( [], - val2bool(DEFAULT_VARS['MODBUS_ENABLED']), - val2bool(DEFAULT_VARS['MODBUS_EVE_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['MODBUS_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['MODBUS_EVE_ENABLED']), False, DEFAULT_VARS['MODBUS_PORTS'], None, From ab590232b25d23472e7caa55e004456619376800 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 14 Dec 2023 08:43:30 -0700 Subject: [PATCH 59/78] passthrough opensearch, netbox and logstash APIs vi malcolm API --- nginx/nginx.conf | 36 ++++++++++++++++++++++++++++-------- nginx/nginx_readonly.conf | 13 +++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 6741c5861..e6c109bcd 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -145,14 +145,6 @@ http { client_max_body_size 50G; } - # Logstash statistics - location ~* ^/logstash\b(.*) { - include /etc/nginx/nginx_auth_rt.conf; - proxy_pass http://logstash-stats/_node/stats$1; - proxy_redirect off; - proxy_set_header Host arkime.malcolm.local; - } - # Arkime -> Dashboards shortcut location ~* ^/idark2dash(.*) { include /etc/nginx/nginx_auth_rt.conf; @@ -253,6 +245,34 @@ http { proxy_set_header Authorization ""; } + # passthrough Logstash statistics from the Malcolm API + location /mapi/logstash/ { + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://logstash-stats/_node/stats/; + proxy_redirect off; + } + + # passthrough OpenSearch from the Malcolm API + location /mapi/opensearch/ { + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://opensearch/; + proxy_redirect off; + proxy_set_header Host os.malcolm.local; + client_max_body_size 50m; + } + + # passthrough NetBox from the Malcolm API + location /mapi/netbox/ { + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://netbox/netbox/api/; + proxy_redirect off; + proxy_set_header Host netbox.malcolm.local; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Remote-Auth $authenticated_user; + } + # Malcolm API location /mapi { include /etc/nginx/nginx_auth_rt.conf; diff --git a/nginx/nginx_readonly.conf b/nginx/nginx_readonly.conf index bbc611d02..4bb3705fc 100644 --- a/nginx/nginx_readonly.conf +++ b/nginx/nginx_readonly.conf @@ -177,6 +177,19 @@ http { proxy_set_header Authorization ""; } + # passthrough NetBox from the Malcolm API + location /mapi/netbox/ { + limit_except GET { deny all; } + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://netbox/netbox/api/; + proxy_redirect off; + proxy_set_header Host netbox.malcolm.local; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Remote-Auth $authenticated_user; + } + # Malcolm API location /mapi { proxy_pass http://api; From 6163aaede728dc1440b61f672173516c3e56ca52 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 14 Dec 2023 15:27:02 -0700 Subject: [PATCH 60/78] fix config location in docker-compose.yml --- docker-compose-standalone.yml | 16 ++++++++-------- docker-compose.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 3fa863bab..4dba66286 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -246,15 +246,15 @@ services: # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - - ./config.elastic/process.env - - ./config.elastic/ssl.env - - ./config.elastic/opensearch.env - - ./config.elastic/upload-common.env + - ./config/process.env + - ./config/ssl.env + - ./config/opensearch.env + - ./config/upload-common.env - ./config.seth/pcap-capture.env - - ./config.elastic/auth.env - - ./config.elastic/arkime.env - - ./config.elastic/arkime-live.env - - ./config.elastic/arkime-secret.env + - ./config/auth.env + - ./config/arkime.env + - ./config/arkime-live.env + - ./config/arkime-secret.env volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro diff --git a/docker-compose.yml b/docker-compose.yml index ba4a0fc63..4e478100a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -273,15 +273,15 @@ services: # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling - SYS_NICE env_file: - - ./config.elastic/process.env - - ./config.elastic/ssl.env - - ./config.elastic/opensearch.env - - ./config.elastic/upload-common.env + - ./config/process.env + - ./config/ssl.env + - ./config/opensearch.env + - ./config/upload-common.env - ./config.seth/pcap-capture.env - - ./config.elastic/auth.env - - ./config.elastic/arkime.env - - ./config.elastic/arkime-live.env - - ./config.elastic/arkime-secret.env + - ./config/auth.env + - ./config/arkime.env + - ./config/arkime-live.env + - ./config/arkime-secret.env volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro From 89dd9b00b706afe214dffdce9900f2fac0b33040 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 14 Dec 2023 15:29:14 -0700 Subject: [PATCH 61/78] fixes that shouldn't have been merged in --- docker-compose-standalone.yml | 2 +- docker-compose.yml | 2 +- kubernetes/03-opensearch.yml | 4 ++-- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 ++-- kubernetes/06-pcap-monitor.yml | 4 ++-- kubernetes/07-arkime.yml | 4 ++-- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 ++-- kubernetes/11-suricata.yml | 4 ++-- kubernetes/12-file-monitor.yml | 4 ++-- kubernetes/13-filebeat.yml | 4 ++-- kubernetes/14-logstash.yml | 4 ++-- kubernetes/15-netbox-redis.yml | 4 ++-- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 ++-- kubernetes/18-netbox.yml | 4 ++-- kubernetes/19-htadmin.yml | 4 ++-- kubernetes/20-pcap-capture.yml | 4 ++-- kubernetes/21-zeek-live.yml | 4 ++-- kubernetes/22-suricata-live.yml | 4 ++-- kubernetes/23-arkime-live.yml | 4 ++-- kubernetes/24-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 ++-- 25 files changed, 43 insertions(+), 43 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 4dba66286..697718674 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -250,7 +250,7 @@ services: - ./config/ssl.env - ./config/opensearch.env - ./config/upload-common.env - - ./config.seth/pcap-capture.env + - ./config/pcap-capture.env - ./config/auth.env - ./config/arkime.env - ./config/arkime-live.env diff --git a/docker-compose.yml b/docker-compose.yml index 4e478100a..3bc8b9014 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -277,7 +277,7 @@ services: - ./config/ssl.env - ./config/opensearch.env - ./config/upload-common.env - - ./config.seth/pcap-capture.env + - ./config/pcap-capture.env - ./config/auth.env - ./config/arkime.env - ./config/arkime-live.env diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 1d8370e8a..4a14aaeed 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/mmguero-dev/malcolm/opensearch:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/opensearch:development imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 6c1774a8b..cfbb8b422 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/mmguero-dev/malcolm/dashboards:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dashboards:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 2266cdba3..7631d405f 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/mmguero-dev/malcolm/file-upload:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/file-upload:development imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 283baeb62..70da6fc02 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/mmguero-dev/malcolm/pcap-monitor:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/pcap-monitor:development imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index b045eb1d8..965859fe8 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/mmguero-dev/malcolm/arkime:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/arkime:development imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index fee194155..dff8c4274 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/mmguero-dev/malcolm/api:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/api:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 8bdaf4c2d..3c1292517 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/mmguero-dev/malcolm/dashboards-helper:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dashboards-helper:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 93223ecf4..8148c47fa 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/mmguero-dev/malcolm/zeek:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/zeek:development imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index fad8c6ef4..5bdf9472f 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/mmguero-dev/malcolm/suricata:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/suricata:development imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index ee9879e08..9cf768a47 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/mmguero-dev/malcolm/file-monitor:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/file-monitor:development imagePullPolicy: Always stdin: false tty: true @@ -81,7 +81,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index fc6823a91..da45a94d1 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/mmguero-dev/malcolm/filebeat-oss:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/filebeat-oss:development imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 22f7fa97f..0cb84994b 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/mmguero-dev/malcolm/logstash-oss:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/logstash-oss:development imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index ebe65715e..922f54f1d 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/mmguero-dev/malcolm/redis:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/redis:development imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 171a64176..0fef1bbf0 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/mmguero-dev/malcolm/redis:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/redis:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 18a6076ea..55a066358 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/mmguero-dev/malcolm/postgresql:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/postgresql:development imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 19c901803..f81438018 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/mmguero-dev/malcolm/netbox:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/netbox:development imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index b3761a565..de5293761 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/mmguero-dev/malcolm/htadmin:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/htadmin:development imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 610f0ade8..2e0f46fca 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/mmguero-dev/malcolm/pcap-capture:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/pcap-capture:development imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index f145bca23..1573472de 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/mmguero-dev/malcolm/zeek:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/zeek:development imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 44df37747..3fbca4d03 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/mmguero-dev/malcolm/suricata:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/suricata:development imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 8c9213e68..2c31ddd6c 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/mmguero-dev/malcolm/arkime:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/arkime:development imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index 6c26785db..b9dc580df 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/mmguero-dev/malcolm/freq:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/freq:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index ec26aef8b..94e7861e2 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/mmguero-dev/malcolm/nginx-proxy:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/nginx-proxy:development imagePullPolicy: Always stdin: false tty: true @@ -95,7 +95,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:23.12.1 + image: ghcr.io/mmguero-dev/malcolm/dirinit:development imagePullPolicy: Always stdin: false tty: true From 60a8619982d153deecdae0121404715c7308f6fa Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 14 Dec 2023 15:45:33 -0700 Subject: [PATCH 62/78] fix arkime deploy --- .../sensor_ctl/supervisor.init/arkime_config_populate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index 2bbe965d5..fd7c32ce2 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -40,7 +40,7 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t if [[ -n $ARKIME_VIEWER_PORT ]]; then sed -r -i "s/(viewPort)\s*=\s*.*/\1=$ARKIME_VIEWER_PORT/" "$ARKIME_CONFIG_FILE" - f + fi if [[ -n $ARKIME_PASSWORD_SECRET ]]; then # place the Arkime viewer cluster password hash in the config file From 391ab2675c2ea344432821c4c704911ffd15ae8d Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 14 Dec 2023 15:52:37 -0700 Subject: [PATCH 63/78] systemd dashboard --- .../f6600310-9943-11ee-a029-e973f4774355.json | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json index e623a6bb7..6a4e2047e 100644 --- a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -7,8 +7,8 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:46:18.238Z", - "version": "Wzk2MSwxXQ==", + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "WzkzOCwxXQ==", "attributes": { "title": "Malcolm Sensor Journald Logs", "hits": 0, @@ -63,8 +63,8 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:29:11.742Z", - "version": "Wzk1MSwxXQ==", + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "WzkzOSwxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Logger", "visState": "{\"title\":\"Malcolm Sensor Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", @@ -93,11 +93,11 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:35:13.052Z", - "version": "Wzk1NCwxXQ==", + "updated_at": "2023-12-14T22:50:36.981Z", + "version": "Wzk1NSwxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Process UID", - "visState": "{\"title\":\"Malcolm Sensor Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "visState": "{\"title\":\"Malcolm Sensor Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Host\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", "description": "", "version": 1, @@ -123,8 +123,8 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:32:46.283Z", - "version": "Wzk1MywxXQ==", + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0MSwxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Logs by Host Over Time", "visState": "{\"title\":\"Malcolm Sensor Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", @@ -153,8 +153,8 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:46:02.857Z", - "version": "Wzk2MCwxXQ==", + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0MiwxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Systemd Unit", "visState": "{\"title\":\"Malcolm Sensor Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -183,8 +183,8 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:44:25.664Z", - "version": "Wzk1OSwxXQ==", + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0MywxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Process Name", "visState": "{\"title\":\"Malcolm Sensor Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", @@ -213,8 +213,8 @@ "namespaces": [ "default" ], - "updated_at": "2023-12-13T18:11:33.018Z", - "version": "Wzk0MCwxXQ==", + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0NCwxXQ==", "attributes": { "title": "Malcolm Sensor Journald - Logs", "description": "", From 835dd2c3a6b0ab6d32e772fc2dab28111b4c74f6 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 07:38:25 -0700 Subject: [PATCH 64/78] gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 11b451b19..c47bd8a97 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ config.*/ .vagrant malcolm_*images.tar.gz malcolm_*images.tar.xz -malcolm_netbox_backup_*.psql.gz +malcolm_netbox_backup_*.gz *.iso *-build.log Gemfile.lock From 673738980bbadf85cbf7b9e80d325165edce9933 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 07:52:22 -0700 Subject: [PATCH 65/78] bump supercronic to v0.2.29 and netbox to v3.6.7 --- Dockerfiles/dashboards-helper.Dockerfile | 4 ++-- Dockerfiles/file-monitor.Dockerfile | 4 ++-- Dockerfiles/file-upload.Dockerfile | 4 ++-- Dockerfiles/filebeat.Dockerfile | 4 ++-- Dockerfiles/netbox.Dockerfile | 6 +++--- Dockerfiles/suricata.Dockerfile | 4 ++-- Dockerfiles/zeek.Dockerfile | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 72d86bf73..217c139fe 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -47,10 +47,10 @@ ENV DASHBOARDS_URL $DASHBOARDS_URL ENV DASHBOARDS_DARKMODE $DASHBOARDS_DARKMODE ENV PATH="/data:${PATH}" -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV ECS_RELEASES_URL "https://api.github.com/repos/elastic/ecs/releases/latest" diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 34b429a91..6ed4d539e 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -93,10 +93,10 @@ ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/ diff --git a/Dockerfiles/file-upload.Dockerfile b/Dockerfiles/file-upload.Dockerfile index cb194a588..fd582306a 100644 --- a/Dockerfiles/file-upload.Dockerfile +++ b/Dockerfiles/file-upload.Dockerfile @@ -49,10 +49,10 @@ ENV FILEPOND_SERVER_BRANCH $FILEPOND_SERVER_BRANCH ARG STALE_UPLOAD_DELETE_MIN=360 ENV STALE_UPLOAD_DELETE_MIN $STALE_UPLOAD_DELETE_MIN -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" COPY --from=npmget /usr/local/lib/node_modules/filepond /var/www/upload/filepond diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index a5897b36d..f3ee8e66c 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -62,10 +62,10 @@ ARG FILEBEAT_TCP_PARSE_DROP_FIELD="" ARG FILEBEAT_TCP_TAG="_malcolm_beats" ARG PCAP_NODE_NAME=malcolm -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV TINI_VERSION v0.19.0 diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 4d18a2339..aa8724f36 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -1,4 +1,4 @@ -FROM netboxcommunity/netbox:v3.6.6 +FROM netboxcommunity/netbox:v3.6.7 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -24,10 +24,10 @@ ENV PUSER "ubuntu" ENV PGROUP "ubuntu" ENV PUSER_PRIV_DROP true -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.33.3" diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index c79a5a9d4..62f026d4b 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -30,10 +30,10 @@ ENV PGROUP "suricata" ENV PUSER_PRIV_DROP false ENV PUSER_RLIMIT_UNLOCK true -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.33.3" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 0c3d43172..b71ae6801 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -93,10 +93,10 @@ ENV PGROUP "zeeker" # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" # for download and install From 8b69bf8ef58a61eb9a73f23585a0d0db298357f6 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 08:28:15 -0700 Subject: [PATCH 66/78] Reduce verbosity --- scripts/malcolm_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index b4e0794fa..d8e95ded3 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -705,6 +705,7 @@ def DownloadToFile(url, local_filename, debug=False): | GET\s+/(_cat/health|api/status|sessions2-|arkime_\w+).+HTTP/[\d\.].+\b200\b | GET\s+/\s+.+\b200\b.+ELB-HealthChecker | (GET|POST|PATCH)\s+/netbox/.+HTTP/[\d\.].+\b20[01]\b + | (GET|POST)\s+/(fields|get|valueActions|views|fieldActions)\b.+bytes\s+[\d\.]+\s+ms | loaded\s+config\s+'/etc/netbox/config/ | LOG:\s+checkpoint\s+(complete|starting) | "netbox"\s+application\s+started From 7f457afdb20c12dc0648e3f4f30823b0f7c4491e Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 11:03:07 -0700 Subject: [PATCH 67/78] in docker container startup, allow files with _MALDIR_ in the name to be expanded out to a subdirectory --- Dockerfiles/dirinit.Dockerfile | 2 +- shared/bin/docker-uid-gid-setup.sh | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/dirinit.Dockerfile b/Dockerfiles/dirinit.Dockerfile index a17ecafc2..e13689ee5 100644 --- a/Dockerfiles/dirinit.Dockerfile +++ b/Dockerfiles/dirinit.Dockerfile @@ -24,7 +24,7 @@ COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ RUN apk update --no-cache && \ apk upgrade --no-cache && \ - apk --no-cache add bash psmisc shadow tini && \ + apk --no-cache add bash psmisc rsync shadow tini && \ addgroup -g ${DEFAULT_GID} ${PGROUP} ; \ adduser -D -H -u ${DEFAULT_UID} -h /nonexistant -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \ addgroup ${PUSER} tty ; \ diff --git a/shared/bin/docker-uid-gid-setup.sh b/shared/bin/docker-uid-gid-setup.sh index 0022249ac..15a990edb 100755 --- a/shared/bin/docker-uid-gid-setup.sh +++ b/shared/bin/docker-uid-gid-setup.sh @@ -40,11 +40,24 @@ if [[ -n ${CONFIG_MAP_DIR} ]] && command -v rsync >/dev/null 2>&1; then awk '{print gsub("/","/"), $0}' | sort -n | cut -d' ' -f2- | \ while read CMDIR; do + DSTDIR="$(realpath "${CMDIR}"/../)" rsync --recursive --copy-links \ "--usermap=*:${PUID:-${DEFAULT_UID}}" \ "--groupmap=*:${PGID:-${DEFAULT_GID}}" \ --exclude='..*' --exclude="${MAP_DIR}"/ --exclude=.dockerignore --exclude=.gitignore \ - "${CMDIR}"/ "${CMDIR}"/../ + "${CMDIR}"/ "${DSTDIR}"/ + + # Additionally, files in these directories with _MALDIR_ in the name will be expanded out, + # creating the intermediate paths. For example: + # ./acid_MALDIR_ACID_MALDIR_s7comm_MALDIR_detect_MALDIR_copy.zeek + # will be renamed to + # ./acid/ACID/s7comm/detect/copy.zeek + find "${DSTDIR}" -type f -name '*_MALDIR_*' -print -o -path "${CMDIR}" -prune 2>/dev/null | \ + while read FLATTENED_FILE; do + EXPANDED_FILE="$(echo "${FLATTENED_FILE}" | sed 's@_MALDIR_@/@g')" + mkdir -p "$(dirname "${EXPANDED_FILE}")" && \ + mv "${FLATTENED_FILE}" "${EXPANDED_FILE}" + done # loop over flattened filenames # TODO - regarding ownership and permissions: # From 4a3842ef393322a4ee29c706eba19d1bde3983c9 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 11:04:00 -0700 Subject: [PATCH 68/78] Adjust kubernetes mountPaths for a few configMaps --- kubernetes/07-arkime.yml | 2 +- kubernetes/10-zeek.yml | 4 ++-- kubernetes/21-zeek-live.yml | 4 ++-- kubernetes/23-arkime-live.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 965859fe8..25a3d8cbe 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -74,7 +74,7 @@ spec: - mountPath: /var/local/curlrc/secretmap name: arkime-opensearch-curlrc-secret-volume - mountPath: "/opt/arkime/rules" - name: arkime-rules-volume + name: arkime-rules-volume/configmap - mountPath: "/data/pcap" name: arkime-pcap-volume initContainers: diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 8148c47fa..daa925943 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -55,9 +55,9 @@ spec: - mountPath: "/zeek/upload" name: zeek-offline-zeek-volume subPath: "upload" - - mountPath: "/opt/zeek/share/zeek/site/custom" + - mountPath: "/opt/zeek/share/zeek/site/custom/configmap" name: zeek-offline-custom-volume - - mountPath: "/opt/zeek/share/zeek/site/intel-preseed" + - mountPath: "/opt/zeek/share/zeek/site/intel-preseed/configmap" name: zeek-offline-intel-preseed-volume - mountPath: "/opt/zeek/share/zeek/site/intel" name: zeek-offline-intel-volume diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 1573472de..2575cfdc3 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -52,9 +52,9 @@ spec: - mountPath: "/zeek/upload" name: zeek-live-zeek-volume subPath: "upload" - - mountPath: "/opt/zeek/share/zeek/site/custom" + - mountPath: "/opt/zeek/share/zeek/site/custom/configmap" name: zeek-live-custom-volume - - mountPath: "/opt/zeek/share/zeek/site/intel-preseed" + - mountPath: "/opt/zeek/share/zeek/site/intel-preseed/configmap" name: zeek-live-intel-preseed-volume - mountPath: "/opt/zeek/share/zeek/site/intel" name: zeek-live-intel-volume diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 2c31ddd6c..79b4bc4d3 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -56,7 +56,7 @@ spec: name: arkime-live-var-local-catrust-volume - mountPath: /var/local/curlrc/secretmap name: arkime-live-opensearch-curlrc-secret-volume - - mountPath: "/opt/arkime/rules" + - mountPath: "/opt/arkime/rules/configmap" name: arkime-live-rules-volume - mountPath: "/data/pcap" name: arkime-live-pcap-volume From a111b20517a8f3bee7279a60c5e161205376d12b Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 11:23:14 -0700 Subject: [PATCH 69/78] don't fail if we can't create expanded dirs, leave the error up to the container --- shared/bin/docker-uid-gid-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/bin/docker-uid-gid-setup.sh b/shared/bin/docker-uid-gid-setup.sh index 15a990edb..eb5189c2d 100755 --- a/shared/bin/docker-uid-gid-setup.sh +++ b/shared/bin/docker-uid-gid-setup.sh @@ -56,7 +56,8 @@ if [[ -n ${CONFIG_MAP_DIR} ]] && command -v rsync >/dev/null 2>&1; then while read FLATTENED_FILE; do EXPANDED_FILE="$(echo "${FLATTENED_FILE}" | sed 's@_MALDIR_@/@g')" mkdir -p "$(dirname "${EXPANDED_FILE}")" && \ - mv "${FLATTENED_FILE}" "${EXPANDED_FILE}" + mv "${FLATTENED_FILE}" "${EXPANDED_FILE}" || \ + true done # loop over flattened filenames # TODO - regarding ownership and permissions: From 58450879ae0f6a869de4c8d55521ef3a32dd9789 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 12:38:34 -0700 Subject: [PATCH 70/78] fix kubernetes typo --- kubernetes/07-arkime.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 25a3d8cbe..e050e6036 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -73,8 +73,8 @@ spec: name: arkime-var-local-catrust-volume - mountPath: /var/local/curlrc/secretmap name: arkime-opensearch-curlrc-secret-volume - - mountPath: "/opt/arkime/rules" - name: arkime-rules-volume/configmap + - mountPath: "/opt/arkime/rules/configmap" + name: arkime-rules-volume - mountPath: "/data/pcap" name: arkime-pcap-volume initContainers: From 1d106e4a24eba59466cd20bd72ca3c484a9c59aa Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 12:54:21 -0700 Subject: [PATCH 71/78] idaholab/Malcolm#308, create configmaps with flattened path names for expansion on startup --- scripts/malcolm_kubernetes.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index bc29f4b88..7643f58d5 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -11,6 +11,7 @@ from collections import defaultdict from itertools import chain from io import StringIO +from pathlib import Path from malcolm_common import ( DotEnvDynamic, @@ -23,7 +24,6 @@ from malcolm_utils import ( deep_get, dictsearch, - eprint, get_iterable, file_contents, remove_suffix, @@ -37,6 +37,7 @@ MALCOLM_IMAGE_PREFIX = 'ghcr.io/idaholab/malcolm/' MALCOLM_DOTFILE_SECRET_KEY = 'K8S_SECRET' +MALCOLM_CONFIGMAP_DIR_REPLACER = '_MALDIR_' MALCOLM_CONFIGMAPS = { 'etc-nginx': [ @@ -710,6 +711,8 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): results_dict['create_namespace']['error'] = str(x) # create configmaps from files + # files in nested directories will be created with a name like foo_MALDIR_bar_MALDIR_baz.txt + # and then renamed to foo/bar/baz.txt during container start up by docker-uid-gid-setup.sh results_dict['create_namespaced_config_map']['result'] = dict() results_dict['create_namespaced_secret']['result'] = dict() for configMapName, configMapFiles in MALCOLM_CONFIGMAPS.items(): @@ -731,20 +734,21 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): else: dataMap[os.path.basename(fname)] = contents elif os.path.isdir(fname): - for subfname in glob.iglob( - os.path.join(os.path.join(fname, '**'), '*'), recursive=True - ): - if os.path.isfile(subfname): - contents = file_contents( - subfname, - binary_fallback=True, + for root, dirNames, fileNames in os.walk(fname): + for f in fileNames: + subfname = os.path.join(root, f) + relfname = str(Path(os.path.join(root, f)).relative_to(fname)).replace( + os.sep, MALCOLM_CONFIGMAP_DIR_REPLACER ) - if hasattr(contents, 'decode'): - binaryDataMap[os.path.basename(subfname)] = base64.b64encode( - contents - ).decode('utf-8') - else: - dataMap[os.path.basename(subfname)] = contents + if os.path.isfile(subfname): + contents = file_contents( + subfname, + binary_fallback=True, + ) + if hasattr(contents, 'decode'): + binaryDataMap[relfname] = base64.b64encode(contents).decode('utf-8') + else: + dataMap[relfname] = contents metadata = kubeImported.client.V1ObjectMeta( name=configMapName, namespace=namespace, From 0398befc4f99b55ac3ed47e66ed6985b5993479f Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 18 Dec 2023 13:03:45 -0700 Subject: [PATCH 72/78] rename kubernetes image references prior to a v23.12.1 release --- kubernetes/03-opensearch.yml | 4 ++-- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 ++-- kubernetes/06-pcap-monitor.yml | 4 ++-- kubernetes/07-arkime.yml | 4 ++-- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 ++-- kubernetes/11-suricata.yml | 4 ++-- kubernetes/12-file-monitor.yml | 4 ++-- kubernetes/13-filebeat.yml | 4 ++-- kubernetes/14-logstash.yml | 4 ++-- kubernetes/15-netbox-redis.yml | 4 ++-- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 ++-- kubernetes/18-netbox.yml | 4 ++-- kubernetes/19-htadmin.yml | 4 ++-- kubernetes/20-pcap-capture.yml | 4 ++-- kubernetes/21-zeek-live.yml | 4 ++-- kubernetes/22-suricata-live.yml | 4 ++-- kubernetes/23-arkime-live.yml | 4 ++-- kubernetes/24-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 ++-- 23 files changed, 41 insertions(+), 41 deletions(-) diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 4a14aaeed..97ef53dfe 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/mmguero-dev/malcolm/opensearch:development + image: ghcr.io/idaholab/malcolm/opensearch:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index cfbb8b422..8481d5d6c 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/mmguero-dev/malcolm/dashboards:development + image: ghcr.io/idaholab/malcolm/dashboards:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 7631d405f..44b281c0a 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/mmguero-dev/malcolm/file-upload:development + image: ghcr.io/idaholab/malcolm/file-upload:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 70da6fc02..fb7a356c5 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/mmguero-dev/malcolm/pcap-monitor:development + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index e050e6036..70cdf1e56 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/mmguero-dev/malcolm/arkime:development + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index dff8c4274..34b6a62a3 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/mmguero-dev/malcolm/api:development + image: ghcr.io/idaholab/malcolm/api:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 3c1292517..35ecffa08 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/mmguero-dev/malcolm/dashboards-helper:development + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index daa925943..60041dd52 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/mmguero-dev/malcolm/zeek:development + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index 5bdf9472f..41c126f2e 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/mmguero-dev/malcolm/suricata:development + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index 9cf768a47..8f8140a47 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/mmguero-dev/malcolm/file-monitor:development + image: ghcr.io/idaholab/malcolm/file-monitor:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -81,7 +81,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index da45a94d1..9681fbb7b 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/mmguero-dev/malcolm/filebeat-oss:development + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 0cb84994b..6f6ee2b2a 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/mmguero-dev/malcolm/logstash-oss:development + image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 922f54f1d..6062e0098 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/mmguero-dev/malcolm/redis:development + image: ghcr.io/idaholab/malcolm/redis:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 0fef1bbf0..091e49ada 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/mmguero-dev/malcolm/redis:development + image: ghcr.io/idaholab/malcolm/redis:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 55a066358..2d04687ba 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/mmguero-dev/malcolm/postgresql:development + image: ghcr.io/idaholab/malcolm/postgresql:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index f81438018..83f9db69b 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/mmguero-dev/malcolm/netbox:development + image: ghcr.io/idaholab/malcolm/netbox:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index de5293761..c4a9b3ebb 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/mmguero-dev/malcolm/htadmin:development + image: ghcr.io/idaholab/malcolm/htadmin:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 2e0f46fca..66014d9c8 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/mmguero-dev/malcolm/pcap-capture:development + image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 2575cfdc3..c63a9acb8 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/mmguero-dev/malcolm/zeek:development + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 3fbca4d03..1b373130f 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/mmguero-dev/malcolm/suricata:development + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 79b4bc4d3..41f508378 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/mmguero-dev/malcolm/arkime:development + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index b9dc580df..32e834237 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/mmguero-dev/malcolm/freq:development + image: ghcr.io/idaholab/malcolm/freq:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 94e7861e2..c8e208b56 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/mmguero-dev/malcolm/nginx-proxy:development + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -95,7 +95,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/mmguero-dev/malcolm/dirinit:development + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true From b7ab36b2499f380452eb075588d96ea8c09ef5e1 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 19 Dec 2023 11:00:02 -0700 Subject: [PATCH 73/78] for idaholab/Malcolm#281, make sure PCAP_TWEAK_IFACE gets set correctly for Arkime capture --- Dockerfiles/arkime.Dockerfile | 1 + arkime/scripts/docker_entrypoint.sh | 14 +++++++++++++- scripts/install.py | 25 +++++++++++++++++-------- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 26007ec75..4fc7c1e4d 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -23,6 +23,7 @@ RUN apt-get -q update && \ bison \ cmake \ curl \ + ethtool \ file \ flex \ g++ \ diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index 5b8abbd58..1a6b4b031 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -115,7 +115,19 @@ if [[ ! -f "${ARKIME_CONFIG_FILE}" ]] && [[ -r "${ARKIME_DIR}"/etc/config.orig.i # ensure capabilities for capture setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true - fi + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true + + # disable NIC hardware offloading features and adjust ring buffer sizes for each interface + for IFACE in ${CAPTURE_INTERFACE//,/ }; do + + [[ "${PCAP_IFACE_TWEAK:-false}" == "true" ]] && \ + [[ "$IFACE" != "lo" ]] && \ + [[ -x /usr/local/bin/nic-capture-setup.sh ]] && \ + /usr/local/bin/nic-capture-setup.sh "$IFACE" >/dev/null 2>&1 || true + + done # loop over capture interfaces + + fi # capture interface(s) defined and live capture enabled # rules files if [[ -d "${ARKIME_RULES_DIR}" ]]; then diff --git a/scripts/install.py b/scripts/install.py index 030bcfea4..0b341bcae 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1273,7 +1273,8 @@ def tweak_malcolm_runtime(self, malcolm_install_path): default=args.netboxLogstashAutoPopulate, ) and ( - InstallerYesOrNo( + args.acceptDefaultsNonInteractive + or InstallerYesOrNo( "Autopopulating NetBox's inventory is not recommended. Are you sure?", default=args.netboxLogstashAutoPopulate, ) @@ -1324,10 +1325,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): choices=[(x, '', x == captureOptions[0]) for x in captureOptions], )[0] if captureSelection == 'y': - liveArkime = malcolmProfile == PROFILE_HEDGEHOG + liveArkime = (malcolmProfile == PROFILE_HEDGEHOG) or (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) pcapNetSniff = not liveArkime liveSuricata = True liveZeek = True + tweakIface = True elif captureSelection == 'c': if InstallerYesOrNo( 'Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?', @@ -1352,12 +1354,13 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'Should Malcolm analyze live network traffic with Suricata?', default=args.liveSuricata ) liveZeek = InstallerYesOrNo('Should Malcolm analyze live network traffic with Zeek?', default=args.liveZeek) - if pcapNetSniff or pcapTcpDump or liveZeek or liveSuricata: + if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: pcapFilter = InstallerAskForString( 'Capture filter (tcpdump-like filter expression; leave blank to capture all traffic)', default=args.pcapFilter, ) - tweakIface = InstallerYesOrNo( + # Arkime requires disabling NIC offloading: https://arkime.com/faq#arkime_requires_full_packet_captures_error + tweakIface = liveArkime or InstallerYesOrNo( 'Disable capture interface hardware offloading and adjust ring buffer sizes?', default=args.tweakIface, ) @@ -1375,9 +1378,15 @@ def tweak_malcolm_runtime(self, malcolm_install_path): f"Enter this node's hostname or IP to associate with network traffic metadata", default=args.liveArkimeNodeHost, ) - if not liveArkimeNodeHost and not InstallerYesOrNo( - f'With live Arkime capture node hostname or IP is required for viewer session retrieval. Are you sure?', - default=False, + if ( + (not liveArkimeNodeHost) + and (not args.acceptDefaultsNonInteractive) + and ( + not InstallerYesOrNo( + f'With live Arkime capture node hostname or IP is required for viewer session retrieval. Are you sure?', + default=False, + ) + ) ): liveArkimeNodeHost = InstallerAskForString( f"Enter this node's hostname or IP to associate with network traffic metadata", @@ -3830,7 +3839,7 @@ def main(): metavar="true|false", nargs='?', const=True, - default=False, + default=True, help="Disable capture interface hardware offloading and adjust ring buffer sizes", ) captureArgGroup.add_argument( From 1d5b57e3adf098fdb5010e6494d26c5eb2e5be4c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 19 Dec 2023 11:33:05 -0700 Subject: [PATCH 74/78] for idaholab/Malcolm#281, make sure PCAP_TWEAK_IFACE gets set correctly for Arkime capture --- Dockerfiles/arkime.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 4fc7c1e4d..e1fbcc82e 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -23,7 +23,6 @@ RUN apt-get -q update && \ bison \ cmake \ curl \ - ethtool \ file \ flex \ g++ \ @@ -171,6 +170,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour apt-get install -q -y --no-install-recommends \ bc \ curl \ + ethtool \ file \ geoip-bin \ gettext \ @@ -253,6 +253,8 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ cp $ARKIME_DIR/bin/capture $ARKIME_DIR/bin/capture-offline && \ chown root:${PGROUP} $ARKIME_DIR/bin/capture && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ + chown root:${PGROUP} /sbin/ethtool && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ mkdir -p /var/run/arkime && \ chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/rules $ARKIME_DIR/logs /var/run/arkime #Update Path From 65046c65c4e5f3ea596b3f18ed2839a3a47f6504 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 19 Dec 2023 12:26:06 -0700 Subject: [PATCH 75/78] allow ufw_allow_viewer.sh by %netdev group --- .../config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer diff --git a/malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer b/malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer new file mode 100644 index 000000000..55f79d5e2 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer @@ -0,0 +1,2 @@ +# allow unprivileged mgmt of UFW access for the local Arkime viewer instance +%netdev ALL=(root) NOPASSWD: /usr/local/bin/ufw_allow_viewer.sh From 92c8f78c1a81ed418c558d6ff9f0d9bd18a2d023 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 19 Dec 2023 12:59:05 -0700 Subject: [PATCH 76/78] Revert "allow ufw_allow_viewer.sh by %netdev group" This reverts commit 65046c65c4e5f3ea596b3f18ed2839a3a47f6504. --- .../config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer diff --git a/malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer b/malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer deleted file mode 100644 index 55f79d5e2..000000000 --- a/malcolm-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer +++ /dev/null @@ -1,2 +0,0 @@ -# allow unprivileged mgmt of UFW access for the local Arkime viewer instance -%netdev ALL=(root) NOPASSWD: /usr/local/bin/ufw_allow_viewer.sh From a4479bab447b542f42eb3b58d4a61b456d21a329 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 19 Dec 2023 13:01:06 -0700 Subject: [PATCH 77/78] for idaholab/Malcolm#281, allow docker to talk to host for retrieving packet payloads --- .../config/hooks/normal/0991-security-performance.hook.chroot | 1 + 1 file changed, 1 insertion(+) diff --git a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot index 7e4f86f29..9c0131952 100755 --- a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -31,6 +31,7 @@ UFW_ALLOW_RULES=( for i in ${UFW_ALLOW_RULES[@]}; do ufw allow "$i" done +ufw allow proto tcp from 172.29.0.0/16 to any port 8005 # docker (disallow overriding firewall) mkdir -p /etc/docker/ From d5e2cef2ea71fc0adf47786926830ee8d863fedf Mon Sep 17 00:00:00 2001 From: SG Date: Wed, 20 Dec 2023 11:01:58 -0700 Subject: [PATCH 78/78] update documents for v23.12.1 release --- docs/download.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/download.md b/docs/download.md index 190220c55..588fc90ef 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.12.1.iso](/iso/malcolm-23.12.1.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-23.12.1.iso.sha256.txt) | +| [malcolm-23.12.1.iso](/iso/malcolm-23.12.1.iso) (5.2GiB) | [`a3dd7dbcaa78322f1ae62b93efd4d95e3644a1b52b65ba24dd1bccf4ac6b173a`](/iso/malcolm-23.12.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.12.1.iso](/iso/hedgehog-23.12.1.iso) (2.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-23.12.1.iso.sha256.txt) | +| [hedgehog-23.12.1.iso](/iso/hedgehog-23.12.1.iso) (2.4GiB) | [`009e67d61ae6e8ffa53e8e134091263b91c0f7a442a0717594434761db60b1b5`](/iso/hedgehog-23.12.1.iso.sha256.txt) | ## Warning