Merge branch 'main' into dependabot/go_modules/go_modules-126a66c1b2

installer/Installer.md

@@ -1,7 +1,13 @@
+## RITA/Zeek Installer
+
+#### Generated installer directory
 ```
 rita-<version>.tar.gz
 │   install_rita.yml
 │   install_rita.sh
+|   install_zeek.yml
+|   install_pre.yml
+|   install_post.yml
 │
 └───/scripts
 │    │   ansible-installer.sh
@@ -19,6 +25,7 @@ rita-<version>.tar.gz
 │   │   │   README
 │   │   │   LICENSE
 │   │   │   rita.sh
+|   |   |   zeek
 │   │
 │   └───/etc
 │       │   config.hjson
@@ -29,4 +36,54 @@ rita-<version>.tar.gz
 │       │   timezone.xml
 │       └───/threat_intel_feeds
 
-```
+```
+
+
+### Generating an installer
+
+Note: generating the installer on a branch that has no tag when running `git describe --always --abbrev=0 --tags` will generate a broken installer.
+
+Run:
+`./installer/generate_installer.sh`
+
+The script will generate an installer tar file in the `installer` folder, named `rita-v<version number>-installer.tar.gz`.
+
+Verify that all files in the above directory tree exist in the generated tar file.
+
+Verify that all occurences of "REPLACE_ME" within scripts and/or playbooks got updated with the proper version number that is expected.
+The version for RITA that gets replaced should match the current tag.
+
+The version for Zeek that gets replaced should be the desired version of docker-zeek to be used in this release.
+
+The docker-zeek repo pushes a built multi-architecture image of zeek to DockerHub using Github Actions. The generate_installer script should specify which tag version on [Dockerhub](https://hub.docker.com/r/activecm/zeek/tags) you wish to include with this release. Multi-architecture tags require all architectures to finish building before being merged into one tag, so if the build actions are in progress, please be patient and wait for them to finish before attempting to install it. 
+
+### Running the installer
+To install RITA on the current system, run:
+`./rita-v<version>-installer/install_rita.sh localhost`.
+
+To install RITA on a remote system, run:
+`./rita-v<version>-installer/install_rita.sh [email protected]`.
+
+### Updating the installer
+Each file that is expected to be in the installer must be explicitly copied to the installer within the `./installer/generate_installer.sh` script. 
+
+If any new Ansible playbook or script that uses the "REPLACE_ME" string to insert a version is added, the generate_installer script must be updated to replace that string with the proper version.
+
+Any versions for RITA should NOT be hard-coded. The version should be retrieved by the generate_installer script automatically. The only hard-coded versions in the generator should be for external projects.
+
+
+### "One-line installer"
+To make installing both RITA and Zeek easier, a one-line installer is created and uploaded to the release artifacts on Github. This installer is generated with the generate_installer.sh script as well, but is uploaded to the release within the Generate Installer Github Action.
+This one line installer is a single script (not a tar file). It installs RITA & Zeek on the local system and does NOT require passing any arguments to it.
+
+
+### Zeek
+There are multiple moving parts in order to build Zeek and include it in a RITA install bundle.
+
+The main Zeek repo is [docker-zeek](https://github.com/activecm/docker-zeek). This repository contains the Dockerfile definition needed to build the docker image of Zeek that includes custom modifications like timeouts and the [zeek-open-connections](https://github.com/activecm/zeek-open-connections) plugin.
+
+The docker-zeek repo is responsible for building the multi-arch image for Zeek in Github Actions. The actions automatically upload the image to Dockerhub. In order to test changes locally without uploading them to Dockerhub, the docker-zeek image must be built on your local system and tagged with a name that is NOT similar to `activecm/zeek:<any version>`. To test the zeek script with this custom-built image, the `zeek` script in the docker-zeek repo must be updated to use your custom tag instead of whatever is listed in the `IMAGE_NAME` variable.
+
+The zeek-open-connections plugin must have an updated tag in order to be recognized by the Zeek package manager (zkg). Follow the instructions in that repo's README for more details.
+
+The RITA installer includes an Ansible playbook that pulls the desired version of `activecm/zeek` from Dockerhub and creates the necessary directories needed to run Zeek. The installer generator also pulls the `zeek` script from the `docker-zeek` repo and includes it in the installer, along with listing the proper image version in the `IMAGE_NAME` variable. Aside from these two items, Zeek and RITA are independent of each other. 

installer/generate_installer.sh

@@ -22,15 +22,12 @@ rm -rf "$BASE_DIR"
 # mkdir ./stage
 
 # create ansible subfolders
-# ANSIBLE_FILES=./stage/.ansible/files
 SCRIPTS="$BASE_DIR/scripts"
 ANSIBLE_FILES="$BASE_DIR/files"
-ANSIBLE_PLAYBOOKS="$BASE_DIR/.ansible/playbooks"
 
 mkdir "$BASE_DIR"
 mkdir -p "$ANSIBLE_FILES"
 mkdir -p "$SCRIPTS"
-mkdir -p "$ANSIBLE_PLAYBOOKS"
 
 # create subfolders (for files that installed RITA will contain)
 INSTALL_OPT="$ANSIBLE_FILES"/opt
@@ -42,6 +39,9 @@ mkdir "$ANSIBLE_FILES"/etc
 # copy files in base dir
 cp ./install_scripts/install_zeek.yml "$BASE_DIR"
 cp ./install_scripts/install_rita.yml "$BASE_DIR"
+cp ./install_scripts/install_pre.yml "$BASE_DIR"
+cp ./install_scripts/install_post.yml "$BASE_DIR"
+
 cp ./install_scripts/install_rita.sh "$BASE_DIR" # entrypoint
 
 # copy files to helper script folder

installer/install_scripts/install_post.yml

@@ -0,0 +1,63 @@
+---
+# ansible install playbook for rita V2.
+# Version: 202408061413
+# sample runs:
+# 	Optional: Add the following block, without #'s  to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports).
+# The hosts must each be on their own line.  These can be full or short hostnames or a name following "Host" in ~/.ssh/config .
+#
+# [allritas]
+# ro810
+# ub2404
+#
+# 	Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list:
+#
+# 	ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: '	#-C (no changes) means do a dry run
+# 	ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: '
+
+# Many thanks to but-i-am-dominator for his help with this playbook.
+
+
+- name: "RITA Post: Reboot checks."
+  hosts: "{{  install_hosts  }}"
+  become: true
+
+  vars:
+    ansible_python_interpreter: /bin/python3							# Centos 7 defaults to using python2, so we force python 3.  This change does not break any other distros
+
+
+#Late tasks, including rebooting
+  post_tasks:
+    - name: "RITA Post: Check if reboot required on rpm-based systems."
+      command: needs-restarting -r
+      register: reboot_result
+      ignore_errors: true
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+    - name: "RITA Post: Check if reboot required on Debian/Ubuntu-based systems."
+      register: reboot_required_file
+      stat:
+        path: /var/run/reboot-required
+        get_checksum: no
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' )
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+    - name: "RITA Post: Rebooting system if needed."
+      reboot:
+        reboot_timeout: 120
+      when: ( ansible_connection != 'local' and ( ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) and ( reboot_required_file.stat.exists ) ) or ( ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and ( reboot_result.rc == 1 ) ) )
+      register: reboot_status
+      async: 1
+      poll: 0
+      ignore_errors: True		#If unable to reboot (as ansible refuses to do if installing to localhost) we leave the error at the end of the output but don't treat it as a failure.
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+        - linuxrpm