Updated rollover minimum size to 10 and allowed for override

rollover.py

@@ -88,19 +88,21 @@ def apply_rollover_policy_to_alias(client_config, alias, index_rollover_policies
             if index_size_in_gb >= size_check:
                 rollover_reason = 'Size Policy'
                 rollover = True
+
             # If the # of days exceeds the policy's day check and the index size is at
-            # least 1 GB, set rollover
-            if days_ago >= index_rollover_policies[policy]["days"] and index_size_in_gb >= 1:
+            # least 10 GB or settings['rollover']['shard_minimum_size'], set rollover
+            if 'shard_minimum_size' in settings['rollover']:
+                minimum_size = settings['rollover']['shard_minimum_size']
+            else:
+                minimum_size = 10
+            if days_ago >= index_rollover_policies[policy]["days"] and index_size_in_gb >= minimum_size:
                 rollover_reason = 'Days Policy'
                 rollover = True
             else:
                 if days_ago >= index_rollover_policies[policy]["days"]:
                     print(f"Index {index['index']} meets required days to rollover " + \
-                        f"but is not larger than 1 gb. Skipping")
-            # if alias['index'] == 'logstash-justin-test-000003':
-            #     rollover = True
-            # print(f"Processing index {index['index']} with size of {index_size_in_gb} and
-            # age of {days_ago}")
+                        f"but is not larger than {minimum_size} gb. Skipping")
+
             # If index is rollover ready, append to list
             if rollover:
                 print(

settings.toml.example

@@ -55,6 +55,7 @@ jira = false
 [rollover]
 enabled = false
 minutes_between_run = 10
+shard_minimum_size = 10
 health_check_level = 'yellow'
 
 # Which notifications to use on failure

soc_report.py

@@ -1,9 +1,127 @@
 """Generates a SOC report based on a given time period"""
+import sys
+import os
+import ssl
+import json
 from datetime import datetime, timedelta
+import urllib3
 from opensearch_dsl import Search, A
-import es
-from config import load_configs, load_settings
+from opensearchpy import OpenSearch
 
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+def build_es_connection(client_config, timeout=10):
+    """Builds OpenSearch connection
+
+    Args:
+        client_config (dict): Dictionary of client configuration
+        timeout (int, optional): OS Timeout. Defaults to 10.
+
+    Raises:
+        e: OpenSearch Error
+
+    Returns:
+        Object: OpenSearch connection class
+    """
+    es_config = {}
+    try:
+        # Check to see if SSL is enabled
+        ssl_enabled = False
+        if "ssl_enabled" in client_config:
+            if client_config['ssl_enabled']:
+                ssl_enabled = True
+
+        # Get the SSL settings for the connection if SSL is enabled
+        if ssl_enabled:
+            # Support older variable implementations of grabbing the ca.crt file
+            ca_file = ""
+            if "ca_file" in client_config:
+                if os.path.exists(client_config['ca_file']):
+                    ca_file = client_config['ca_file']
+                else:
+                    exit("CA file referenced does not exist")
+            elif "client_file_location" in client_config:
+                if os.path.exists(client_config['client_file_location'] + "/ca/ca.crt"):
+                    ca_file = client_config['client_file_location'] + \
+                        "/ca/ca.crt"
+
+            if ca_file != "":
+                context = ssl.create_default_context(
+                    cafile=ca_file)
+            else:
+                context = ssl.create_default_context()
+
+            if "check_hostname" in client_config:
+                check_hostname = client_config['check_hostname']
+            if check_hostname:
+                context.check_hostname = True
+            else:
+                context.check_hostname = False
+
+            if "ssl_certificate" in client_config:
+                ssl_certificate = client_config['ssl_certificate']
+            if ssl_certificate == "required":
+                context.verify_mode = ssl.CERT_REQUIRED
+            elif ssl_certificate == "optional":
+                context.verify_mode = ssl.CERT_OPTIONAL
+            else:
+                context.verify_mode = ssl.CERT_NONE
+
+            es_config = {
+                "scheme": "https",
+                "ssl_context": context,
+            }
+
+        # Enable authentication if there is a passwod section in the client JSON
+        password_authentication = False
+        if 'password_authentication' in client_config:
+            if client_config['password_authentication']:
+                password_authentication = True
+        elif 'admin_password' in client_config['password']:
+            password_authentication = True
+        if password_authentication:
+            user = ''
+            password = ''
+            if 'es_password' in client_config:
+                password = client_config['es_password']
+            elif 'admin_password' in client_config['password']:
+                password = client_config['password']['admin_password']
+            if 'es_user' in client_config:
+                user = client_config['es_user']
+            elif client_config['platform'] == "elastic":
+                user = 'elastic'
+            else:
+                user = 'admin'
+            es_config['http_auth'] = (
+                user, password)
+
+        # Get the Elasticsearch port to connect to
+        if 'es_port' in client_config:
+            es_port = client_config['es_port']
+        elif client_config['client_number'] == 0:
+            es_port = "9200"
+        else:
+            es_port = str(client_config['client_number']) + "03"
+
+        # Get the Elasticsearch host to connect to
+        if 'es_host' in client_config:
+            es_host = client_config['es_host']
+        else:
+            es_host = client_config['client_name'] + "_client"
+
+        es_config['retry_on_timeout'] = True
+        es_config['max_retries'] = 10
+        es_config['timeout'] = timeout
+        if os.getenv('DEBUGON') == "1":
+            print(es_config)
+            print(es_host)
+            print(es_port)
+        return OpenSearch(
+            [{'host': es_host, 'port': es_port}], **es_config)
+    except ConnectionError as error:
+        print(sys.exc_info())
+        print("Connection attempt to Elasticsearch Failed")
+        raise error
 
 def get_organization_uuid(client_name):
     """Converts a client name to organization uuid
@@ -74,32 +192,48 @@ def generate_soc_report():
     print(f"Number of alerts: {get_number_of_alerts()}")
     print(get_closer_codes())
 
+def load_configs():
+    """Load all JSON configuration files
 
-def soc_report(client):
+    Returns:
+        dict: All JSON configuration files
+    """
+    configs = {}
+    for file in os.listdir(config_folder):
+        if file.endswith(".json"):
+            with open(config_folder + '/' + file, encoding="UTF-8") as file:
+                client = json.load(file)
+            if 'client_name' in client:
+                client_name = client['client_name']
+            else:
+                print("File name " + file + " does not contain valid client information")
+                sys.exit(1)
+            configs[client_name] = client
+    return configs
+
+def soc_report():
     """Run SOC report
 
     Args:
         manual_client (str): Name of client. Empty means all
     """
-    settings = load_settings()
-    if "soc_report" in settings:
-        if settings['soc_report']['enabled'] is False:
-            return
-    else:
-        return
 
     # Add all clients initially to retry_list for first run
     for client, _ in clients.items():
-        tenant = clients[client]['reflex_tenant']
-        # This is intended so that ORGANIZATION does not have to be passed from
-        # module to module
-        global ORGANIZATION
-        ORGANIZATION = get_organization_uuid(tenant)
-        print(f"Processing and sending SOC report for client {client} with "
-              f"UTC date range of:\n{BEGIN_DATE} to {END_DATE}\n")
-        # If client set at command line only run it otherwise
+        if 'reflex_tenant' in clients[client]:
+            tenant = clients[client]['reflex_tenant']
+        else:
+            continue
+
         # execute for all clients
         if manual_client == "" or clients[client]['client_name'] == manual_client:
+            # This is intended so that ORGANIZATION does not have to be passed from
+            # module to module
+            global ORGANIZATION
+            ORGANIZATION = get_organization_uuid(tenant)
+            print(f"Processing and sending SOC report for client {client} with "
+                f"UTC date range of:\n{BEGIN_DATE} to {END_DATE}\n")
+            # If client set at command line only run it otherwise
             generate_soc_report()
 
 
@@ -112,6 +246,8 @@ def soc_report(client):
         formatter_class=RawTextHelpFormatter)
     parser.add_argument("--client", default="", type=str,
                         help="Set to a specific client name to limit the mssp script to one client")
+    parser.add_argument("--config-folder", default="/opt/maintenance", type=str,
+                        help="Set to the folder containing all your JSON configuration files")
     parser.add_argument("--notification", default="True",
                         type=str, help="Set to False to disable notifications")
     parser.add_argument("--till-now", default="False",
@@ -130,10 +266,12 @@ def soc_report(client):
     weeks_ago = args.weeks_ago
     months_ago = args.months_ago
 
+    config_folder = args.config_folder
+
     clients = load_configs()
     reflex_config = clients['ha']
 
-    os_connection = es.build_es_connection(reflex_config, timeout=30)
+    os_connection = build_es_connection(reflex_config, timeout=30)
 
     DATE_COUNT = 0
     if days_ago > 0:
@@ -177,6 +315,6 @@ def soc_report(client):
 
     ORGANIZATION = None
 
-    soc_report(manual_client)
+    soc_report()
 
     os_connection.close()