Implement RPM and GEM for efficient installation and setup (#10)

* First stem to gem pdnssoc * Installation script to gem * Improve automatic installation * Adjustments to allow install rake and let it running * Restrucure of the project and rpm working * New version fetching the gem file directly from rubygems.org * Few corrections for smooth rpm install * Few corrections for smooth rpm install * Remove Docker config and install.sh as it is not updated * td-agent.conf supporting opensearch * Update README.md * Corrections on the spec file and new version of gem to use td-agent.conf as template * Update README.md * Update README.md * Timers services restart and delete after the uninstall * Update README.md * Update README.md * Update README.md --------- Co-authored-by: root <[email protected]> Co-authored-by: Developer <[email protected]>
CERN-CERT · Aug 21, 2023 · 642363a · 642363a
1 parent c595ef5
commit 642363a
Show file tree

Hide file tree

Showing 29 changed files with 368 additions and 208 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,15 @@
+## 0.1.3 - 2023-08-07
+
+### Added
+- Integrated OpenSearch support to enable powerful search capabilities.
+- Implemented OpenSearch template with specific mappings to align with new data structure.
+- Added nested field support in OpenSearch for enhanced querying within the `misp` field.
+- RPM install capabilities. 
+- Gem packaging of the code. 
+
+### Changed
+- Modified Fluentd configuration to forward logs to OpenSearch.
+- Adjusted log format to match the new OpenSearch template (if applicable).
+
+### Fixed
+- 
diff --git a/Gemfile b/Gemfile
@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out_filter_list.gemspec
+gemspec
+
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    pdnssoc (0.1.0)
+      fluent-plugin-filter-list
+      misp
+      parseconfig
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.2.2)
+    cool.io (1.7.1)
+    fluent-plugin-filter-list (0.7.5)
+      fluentd (>= 0.14.0, < 2.0.0)
+    fluentd (1.16.2)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.3.2, < 3.0.0)
+      sigdump (~> 0.2.5)
+      strptime (>= 0.2.4, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (~> 1.4)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.8.0)
+    misp (0.1.4)
+    msgpack (1.7.2)
+    parseconfig (1.1.2)
+    rake (13.0.6)
+    serverengine (2.3.2)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.5)
+    strptime (0.2.5)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2023.3)
+      tzinfo (>= 1.0.0)
+    webrick (1.8.1)
+    yajl-ruby (1.4.3)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  fluent-plugin-filter-list
+  misp
+  parseconfig
+  pdnssoc!
+  rake
+
+BUNDLED WITH
+   2.4.17
diff --git a/README.md b/README.md
@@ -70,103 +70,72 @@ pDNSSOC operators are typically security teams at NREN CERTs, e-infratructure se
 ### VM deployment
 
 1. Get a fresh CENTOS 9 or AlmaLinux 9 (or any binary-compatible system with Red Hat Enterprise Linux 9) VM
-2. Execute the install script, for example:
+2. Install the needed libraries:
 ```
- curl -L https://raw.githubusercontent.com/CERN-CERT/pDNSSOC/main/files/install.sh  | sh
+yum install gem jq rubygems-devel
 ```
-3. Populate the configuration in `/etc/pdnssoc/pdnssoc.conf` with the MISP server(s) details and the alert emails details
-4. Do an initial load of malicious domains from MISP:
+3. Download and install td-agent (fluentd):
 ```
-/etc/pdnssoc/pdnssoc.cron 
-systemctl restart td-agent
+curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent4.sh | sh
 ```
-5. That's it!
-
-Directory structure:
+4. Download the gem file:
 ```
-/etc/pdnssoc/pdnssoc.conf
-/etc/pdnssoc/notification_email.html
-/etc/td-agent/misp_domains.txt
-/etc/td-agent/td-agent.conf
-/var/log/td-agent/alerts.log
-/var/log/td-agent/pdnssoc_sys.log
-/var/log/td-agent/alerts.log
-/var/log/td-agent/buffer
-/var/log/td-agent/pdnssoc-alerts
-/var/log/td-agent/pdnssoc-alerts/pdnssoc-buffer
-/var/log/td-agent/pdnssoc-alerts/results
-/var/log/td-agent/pdnssoc_sys.log
-/var/log/td-agent/queries
+curl -o /path/to/download/pdnssoc-VERSION.gem https://rubygems.org/downloads/pdnssoc-VERSION.gem
 ```
-
-
-### Container deployment
-
-1. You need to have `docker`, `docker-compose` and `git` installed. An example for AlmaLinux 9 / Centos 9:
+5. Install the rpm:
 ```
-sudo yum update -y
-sudo yum install -y yum-utils
-sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-sudo yum update -y
-sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-sudo systemctl start docker
-sudo yum install -y git
+rpm -i pdnssoc-VERSION-RELEASE.noarch.rpm
 ```
-
-2. Clone the repo:
-
+6. Populate the configuration in `/etc/pdnssoc/pdnssoc.conf` with the MISP server(s) details and the alert emails details
+7. Use the fluentd config template in `/etc/pdnssoc/td-agent.conf.template` to overwrite or adapt `/etc/td-agent/td-agent.conf`
+8. Include the appropriate firewall rules in order to accept incoming traffic:
 ```
-git clone https://github.com/CERN-CERT/pDNSSOC.git
+firewall-cmd --zone=public --add-port=5140-5143/tcp --permanent
+firewall-cmd --zone=public --add-port=5555/tcp --permanent
+firewall-cmd --reload
 ```
-
-3. Go to the `files` directory inside the repo.
-4. Populate the configuration `pdnssoc.conf` with the MISP server(s) details and the alert emails details.
-5. Make any changes (if needed) on the fluentd configuration `td-agent.conf`.
-6. Build the docker container:
-
+9. Do an initial load of malicious domains from MISP:
 ```
-docker build -t pdnssoc .
+/bin/bash /usr/local/bin/pdnssoc/misp_refresh.sh
+systemctl restart td-agent
 ```
-
-7. Start the docker container:
-
+10. That's it! You can do the following to check that the system is well configured:
 ```
-docker compose up
+systemctl list-timers
+netstat -putan | grep -E ':(514[0-3]|5555) '
 ```
 
-8. That's it!
+Directory structure:
+```
+/etc/pdnssoc/
+├── notification_email.html
+├── pdnssoc.conf
+└── td-agent.conf.template
 
+/etc/td-agent/
+├── misp_domains.txt
+├── misp_ips.txt
+├── plugin
+└── td-agent.conf -> /etc/pdnssoc/td-agent.conf
 
-Directory structure:
+/var/log/td-agent/
+...
+├── queries/
+├── pdnssoc-alerts
+│   └──pdnssoc-buffer/
+├── alerts.log
+├── pdnssoc_sys.log
+└── td-agent.log
+
+/usr/lib/systemd/system/
+...
+├── lookingback.service
+├── lookingback.timer
+├── misp_refresh.service
+├── misp_refresh.timer
+├── pdnssoc.service
+└── pdnssoc.timer
 
-```
-./pDNSSOC
-├── docs
-├── files
-│   ├── code
-│   │   ├── alerts.rb
-│   │   ├── configalerts.rb
-│   │   ├── constants.rb
-│   │   ├── email.rb
-│   │   ├── pdnssoc.rb
-│   │   └── trigger.rb
-│   ├── crontab
-│   ├── docker-compose.yml
-│   ├── Dockerfile
-│   ├── install.sh
-│   ├── logs
-│   │   ├── alerts.log
-│   │   ├── buffer
-│   │   ├── pdnssoc-alerts
-│   │   │   ├── pdnssoc-buffer
-│   │   ├── pdnssoc_sys.log
-│   │   ├── queries
-│   ├── notification_email.html
-│   ├── pdnssoc.conf
-│   ├── pdnssoc.cron
-│   └── td-agent.conf
-├── images
-└── README.md
 ```
 
 ### Configuration file

diff --git a/files/notification_email.html → config/notification_email.html b/files/notification_email.html → config/notification_email.html
diff --git a/files/pdnssoc.conf → config/pdnssoc.conf b/files/pdnssoc.conf → config/pdnssoc.conf
diff --git a/files/td-agent.conf → config/td-agent.conf.template b/files/td-agent.conf → config/td-agent.conf.template
@@ -66,12 +66,26 @@
   @type forward
   port 5555
   tag pdnssocdata
+  # tls true
+  # tls_cert_path /path/to/fluentd.crt
+  # tls_private_key_path /path/to/fluentd.key
   <parse>
         @type regexp
     expression /{"date":"(?<date>.*)","client":"(?<client>.*)","query":"(?<query>.*)"}/
   </parse>
 </source>
 
+## OpenSearch
+# Getting logs from pdnssoc to send to OpenSearch
+# <source>
+#   @type tail
+#   path /var/log/td-agent/alerts.log
+#   tag pdnssoc_alerts
+#   read_from_head true
+#   <parse>
+#     @type json
+#   </parse>
+# </source>
 
 ## DATA ROUTING
 # Copying our pdnssocdata into multiple streams
@@ -101,6 +115,8 @@
         # <server>
         #   host upstream-pdnssoc.domain.edu
         #   port 5555
+        #   tls true
+        #   tls_cert_path /path/to/fluentd.pem
         # </server>
         #</store>
         #######################
@@ -130,7 +146,7 @@
   @type filter_list
   filter AC
   key_to_filter query
-  pattern_file_paths ["/etc/td-agent/misp_domains.txt"]
+  pattern_file_paths ["/etc/pdnssoc/misp_domains.txt"]
   filter_empty true
   action whitelist
 </filter>
@@ -146,7 +162,7 @@
   @type filter_list
   filter AC
   key_to_filter answer
-  pattern_file_paths ["/etc/td-agent/misp_domains.txt", "/etc/td-agent/misp_ips.txt"]
+  pattern_file_paths ["/etc/pdnssoc/misp_domains.txt", "/etc/pdnssoc/misp_ips.txt"]
   filter_empty true
   action whitelist
 </filter>
@@ -171,3 +187,13 @@
   </buffer>
 </match>
 </label>
+
+# Forward logs to OpenSearch
+# <match pdnssoc_alerts>
+#   @type opensearch
+#   hosts https://opensearch.domain.org
+#   logstash_format true
+#   logstash_prefix pdnssoc
+#   user %{USER}
+#   password %{PASSWORD}
+# </match>
diff --git a/files/Dockerfile b/files/Dockerfile
diff --git a/files/crontab b/files/crontab
diff --git a/files/docker-compose.yml b/files/docker-compose.yml