From cc80b553562baf206597b1573cb8e01c708d1bb8 Mon Sep 17 00:00:00 2001 From: Justin Henderson Date: Tue, 25 Oct 2022 22:02:16 +0000 Subject: [PATCH] Updated rollover minimum size to 10 and allowed for override --- __pycache__/es.cpython-38.pyc | Bin 25884 -> 25884 bytes rollover.py | 16 ++-- settings.toml.example | 1 + soc_report.py | 176 ++++++++++++++++++++++++++++++---- 4 files changed, 167 insertions(+), 26 deletions(-) diff --git a/__pycache__/es.cpython-38.pyc b/__pycache__/es.cpython-38.pyc index 2b2d58b4f1f5e925b5eecee8b59ab1164a89090a..d3d7d7610b9dde45a511d2b6affbf90436bc09c1 100644 GIT binary patch delta 1854 zcmZvcc}&z-5XYT$Sy)z5Dl4^I0anljT3OM6vXp}-{?3eX10y)L$qT6NtB92N$YUY#h+xx(|$gwcr~*# z>2bE!5HSk&^*{qq25jYzZ6$^S5D)Sl+fu{VQmh@$1DT_F)r5J^d?lCCwFw5xfI(A8 z{exS6$SU+9j+_vu6FqFnvQUp^&pJhPN{gOk)loMWWV=o9cocwlXZO;1tuAL1xqd|; zljz3oTR;mi8#o2vzd=K-@Oc70u^yTsf#lK}CfcD@NdzDr0#v#uInEx>t+}mqNjA>R z4|6}F%e<<{tQ~x=hUf~%=K1NGwjr;C=q#U^l5Px!1bm(bzqrlD{6y;KwEV;LfH&kN zacDtsLGX$UtH3|XckSoJkC3$q8TS}Z1#^074rklL3w0!IZX*Sy$J{VkuR6SH7hkD;D zZ6+f(*w@o=zHJ{9c^Bb_Tn0`r%b{@gmZeButTJ;)*-E;=3627JkZwl|-Q|}Z7P_l- zIR4O?RCC`%h||n398LcT;*Szelh5yN_W1p6azLG-a?$Gd3Gp5Fe*nJMUMtrVjpWrY z+P&)kkx)NQRraYJiUc{G>I|Jm8KkzUabeve5eemKxg89Znt*S4K#YVc3V2!~s8_#3 zCL#vPSmLVOXsu|*A~HSxhv?@|XL`t{Jyl^1q48WeH+DGb@t}j||!>n<-bjQkhK@$BC}wkO?S^?XD?L+E5aj zX4R`M1$nb8nI>|NYmROrvF-UN9+6?>jH(j4!J?{>^5s!())f;cWk&HE?&(y>XWi3i z8V^*P<=V&v)w9x*5!C{W0qhb%y?ipf@>wjC3ukD2k*! zD*nfLaEO-5G=m}aO&+fwad?fVh$^_>lTQaYv}O{ea$Zf0LAeLnRTFPecPc|`uZgCW z+t8Zrzz$$1unX7+907WP7t|DgweT&}!9)uWOcNVOK?Y-d)5tty1-a znzlx#v9`6e7$1$X7QCWTA6TtzqGBJ3)~wM+O-xg5i?)i@MtlBejiE{JhhOH*nVB;) z=g!%7hPIs{%bbXa!2`ti;AB>Zww)p0VSf&-|P_*qYuv@;IV(10>*5t%CM}>OTG=J&yKskL0cN zwuEQd+70S(*w+BHKnd^>f9`M^J0TwA2aaXNZ-iJikbBcd@>?16#(yoA!nGa-%Ym>d zq|@Q80Av~ZFh^#L(y52omT99yT5;xiqH|hwmR&~&I6vECfydJTyeIoOUD2v1Z6^0m z2xL(YVE0|15hw%B0XR2gtWy2npkG}NO}{{5X&kDx09 zkVr_sw7$AdRk7{_4g#k+)0s;TxzVYRfzLUQ&|O~Vnm+cb*etUT4hNSZWe;V@{@(w% z8pz1C#p{W90DAZ05G@eW%j1xV zIs(d(#N|2BTH%bvWO;UmXlCb3FQsWCX4wst$^~;{X*|2va=}+hRv;R%XW^6I*tYf0Ot< zOJR5d_zF15jY^z&i8d++E%GIjjqK)`3loe_4Vo7=P`38V!fc}F+3HR-Oc2E=TR9R(d7{*qz~h_J+IJY zzT|nCoZMRxCDvxnub7?k0-{=h1YoK_NH3lYpLiAv#X>RFzUEm}UE^1a_;`hbN`$v> zf=le!P3r$RhX-kyNHb)psrUKa)rwxV0Xup~I ikV7HIG>sf$odIPQFcHWG96%0`2RMNeU>du87ykvj{|iw7 diff --git a/rollover.py b/rollover.py index cb7d0a8..ecdf52a 100644 --- a/rollover.py +++ b/rollover.py @@ -88,19 +88,21 @@ def apply_rollover_policy_to_alias(client_config, alias, index_rollover_policies if index_size_in_gb >= size_check: rollover_reason = 'Size Policy' rollover = True + # If the # of days exceeds the policy's day check and the index size is at - # least 1 GB, set rollover - if days_ago >= index_rollover_policies[policy]["days"] and index_size_in_gb >= 1: + # least 10 GB or settings['rollover']['shard_minimum_size'], set rollover + if 'shard_minimum_size' in settings['rollover']: + minimum_size = settings['rollover']['shard_minimum_size'] + else: + minimum_size = 10 + if days_ago >= index_rollover_policies[policy]["days"] and index_size_in_gb >= minimum_size: rollover_reason = 'Days Policy' rollover = True else: if days_ago >= index_rollover_policies[policy]["days"]: print(f"Index {index['index']} meets required days to rollover " + \ - f"but is not larger than 1 gb. Skipping") - # if alias['index'] == 'logstash-justin-test-000003': - # rollover = True - # print(f"Processing index {index['index']} with size of {index_size_in_gb} and - # age of {days_ago}") + f"but is not larger than {minimum_size} gb. Skipping") + # If index is rollover ready, append to list if rollover: print( diff --git a/settings.toml.example b/settings.toml.example index 34f56a4..091579a 100644 --- a/settings.toml.example +++ b/settings.toml.example @@ -55,6 +55,7 @@ jira = false [rollover] enabled = false minutes_between_run = 10 +shard_minimum_size = 10 health_check_level = 'yellow' # Which notifications to use on failure diff --git a/soc_report.py b/soc_report.py index c90a564..78cf9d3 100644 --- a/soc_report.py +++ b/soc_report.py @@ -1,9 +1,127 @@ """Generates a SOC report based on a given time period""" +import sys +import os +import ssl +import json from datetime import datetime, timedelta +import urllib3 from opensearch_dsl import Search, A -import es -from config import load_configs, load_settings +from opensearchpy import OpenSearch +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +def build_es_connection(client_config, timeout=10): + """Builds OpenSearch connection + + Args: + client_config (dict): Dictionary of client configuration + timeout (int, optional): OS Timeout. Defaults to 10. + + Raises: + e: OpenSearch Error + + Returns: + Object: OpenSearch connection class + """ + es_config = {} + try: + # Check to see if SSL is enabled + ssl_enabled = False + if "ssl_enabled" in client_config: + if client_config['ssl_enabled']: + ssl_enabled = True + + # Get the SSL settings for the connection if SSL is enabled + if ssl_enabled: + # Support older variable implementations of grabbing the ca.crt file + ca_file = "" + if "ca_file" in client_config: + if os.path.exists(client_config['ca_file']): + ca_file = client_config['ca_file'] + else: + exit("CA file referenced does not exist") + elif "client_file_location" in client_config: + if os.path.exists(client_config['client_file_location'] + "/ca/ca.crt"): + ca_file = client_config['client_file_location'] + \ + "/ca/ca.crt" + + if ca_file != "": + context = ssl.create_default_context( + cafile=ca_file) + else: + context = ssl.create_default_context() + + if "check_hostname" in client_config: + check_hostname = client_config['check_hostname'] + if check_hostname: + context.check_hostname = True + else: + context.check_hostname = False + + if "ssl_certificate" in client_config: + ssl_certificate = client_config['ssl_certificate'] + if ssl_certificate == "required": + context.verify_mode = ssl.CERT_REQUIRED + elif ssl_certificate == "optional": + context.verify_mode = ssl.CERT_OPTIONAL + else: + context.verify_mode = ssl.CERT_NONE + + es_config = { + "scheme": "https", + "ssl_context": context, + } + + # Enable authentication if there is a passwod section in the client JSON + password_authentication = False + if 'password_authentication' in client_config: + if client_config['password_authentication']: + password_authentication = True + elif 'admin_password' in client_config['password']: + password_authentication = True + if password_authentication: + user = '' + password = '' + if 'es_password' in client_config: + password = client_config['es_password'] + elif 'admin_password' in client_config['password']: + password = client_config['password']['admin_password'] + if 'es_user' in client_config: + user = client_config['es_user'] + elif client_config['platform'] == "elastic": + user = 'elastic' + else: + user = 'admin' + es_config['http_auth'] = ( + user, password) + + # Get the Elasticsearch port to connect to + if 'es_port' in client_config: + es_port = client_config['es_port'] + elif client_config['client_number'] == 0: + es_port = "9200" + else: + es_port = str(client_config['client_number']) + "03" + + # Get the Elasticsearch host to connect to + if 'es_host' in client_config: + es_host = client_config['es_host'] + else: + es_host = client_config['client_name'] + "_client" + + es_config['retry_on_timeout'] = True + es_config['max_retries'] = 10 + es_config['timeout'] = timeout + if os.getenv('DEBUGON') == "1": + print(es_config) + print(es_host) + print(es_port) + return OpenSearch( + [{'host': es_host, 'port': es_port}], **es_config) + except ConnectionError as error: + print(sys.exc_info()) + print("Connection attempt to Elasticsearch Failed") + raise error def get_organization_uuid(client_name): """Converts a client name to organization uuid @@ -74,32 +192,48 @@ def generate_soc_report(): print(f"Number of alerts: {get_number_of_alerts()}") print(get_closer_codes()) +def load_configs(): + """Load all JSON configuration files -def soc_report(client): + Returns: + dict: All JSON configuration files + """ + configs = {} + for file in os.listdir(config_folder): + if file.endswith(".json"): + with open(config_folder + '/' + file, encoding="UTF-8") as file: + client = json.load(file) + if 'client_name' in client: + client_name = client['client_name'] + else: + print("File name " + file + " does not contain valid client information") + sys.exit(1) + configs[client_name] = client + return configs + +def soc_report(): """Run SOC report Args: manual_client (str): Name of client. Empty means all """ - settings = load_settings() - if "soc_report" in settings: - if settings['soc_report']['enabled'] is False: - return - else: - return # Add all clients initially to retry_list for first run for client, _ in clients.items(): - tenant = clients[client]['reflex_tenant'] - # This is intended so that ORGANIZATION does not have to be passed from - # module to module - global ORGANIZATION - ORGANIZATION = get_organization_uuid(tenant) - print(f"Processing and sending SOC report for client {client} with " - f"UTC date range of:\n{BEGIN_DATE} to {END_DATE}\n") - # If client set at command line only run it otherwise + if 'reflex_tenant' in clients[client]: + tenant = clients[client]['reflex_tenant'] + else: + continue + # execute for all clients if manual_client == "" or clients[client]['client_name'] == manual_client: + # This is intended so that ORGANIZATION does not have to be passed from + # module to module + global ORGANIZATION + ORGANIZATION = get_organization_uuid(tenant) + print(f"Processing and sending SOC report for client {client} with " + f"UTC date range of:\n{BEGIN_DATE} to {END_DATE}\n") + # If client set at command line only run it otherwise generate_soc_report() @@ -112,6 +246,8 @@ def soc_report(client): formatter_class=RawTextHelpFormatter) parser.add_argument("--client", default="", type=str, help="Set to a specific client name to limit the mssp script to one client") + parser.add_argument("--config-folder", default="/opt/maintenance", type=str, + help="Set to the folder containing all your JSON configuration files") parser.add_argument("--notification", default="True", type=str, help="Set to False to disable notifications") parser.add_argument("--till-now", default="False", @@ -130,10 +266,12 @@ def soc_report(client): weeks_ago = args.weeks_ago months_ago = args.months_ago + config_folder = args.config_folder + clients = load_configs() reflex_config = clients['ha'] - os_connection = es.build_es_connection(reflex_config, timeout=30) + os_connection = build_es_connection(reflex_config, timeout=30) DATE_COUNT = 0 if days_ago > 0: @@ -177,6 +315,6 @@ def soc_report(client): ORGANIZATION = None - soc_report(manual_client) + soc_report() os_connection.close()