go

accounting.py

@@ -256,7 +256,7 @@ def calculate_accounting(client_config, client_name):
             print("Accounting operation failed for " + client_name + ". Cluster health does not meet level:  " + settings['accounting']['health_check_level'])
             return False
 
-def run_accounting(manual_client):
+def run_accounting(manual_client=""):
     settings = load_settings()
     if settings['accounting']['enabled']:
         retry_count = settings['accounting']['retry_attempts']

backup.py

@@ -363,7 +363,7 @@ def take_snapshot_per_policies(client_config, job, backup_policy, repository, in
       print(message)
       send_notification(client_config, "backup", "Failed", message, teams=settings['backup']['ms-teams'], jira=settings['backup']['jira'])
 
-def run_backup(manual_client):
+def run_backup(manual_client=""):
   """[summary]
   Runs backup job for specific client configuration
 

config.py

@@ -7,7 +7,13 @@
 import time
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
-settings_file = base_dir + "/settings.toml"
+if os.path.exists("/etc/elastic-ilm/settings.toml"):
+    settings_file = "/etc/elastic-ilm/settings.toml"
+else:
+    settings_file = base_dir + "/settings.toml"
+if not os.path.exists(settings_file):
+    print("settings.toml not found - exiting")
+    exit()
 
 def retry(ExceptionToCheck, tries=5, delay=1, backoff=1, logger=None):
     """Retry calling the decorated function using an exponential backoff.
@@ -51,16 +57,16 @@ def f_retry(*args, **kwargs):
 
     return deco_retry
 
-def load_settings():
-    if os.path.exists(settings_file):
+def load_settings(format='toml'):
+    if format == 'toml':
         settings = toml.load(settings_file)
         if settings['settings']['client_json_folder'] == "":
             settings['settings']['client_json_folder'] = base_dir
-    else:
-        print("No settings.toml file found. Please clone and then edit settings.toml.example")
-        print("Then retry.")
-        exit()
-    return settings
+        return settings
+    if format == 'bytes':
+        with open(settings_file, "rb") as file:
+            read_bytes = file.read()  # read entire file as bytes
+        return read_bytes
 
 def load_configs(client_value=""):
     settings = load_settings()

ilm.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
+import hashlib
+import argparse
+import time
+from argparse import RawTextHelpFormatter
 from apscheduler.schedulers.background import BackgroundScheduler
 from config import load_settings
 from accounting import run_accounting
 #from custom_checks import run_custom_checks
 from retention import apply_retention_policies
 from rollover import apply_rollover_policies
 from backup import run_backup
-import argparse
-from argparse import RawTextHelpFormatter
 parser = argparse.ArgumentParser(description='Used to manually run script (Example: ilm.py --manual 1)', formatter_class=RawTextHelpFormatter)
 parser.add_argument("--manual", default=0, type=int, help="Set to 1 to manually run script")
 parser.add_argument("--client", default="", type=str, help="Set to a specific client name to limit calls to one client")
@@ -27,52 +29,43 @@
 else:
     sched = BackgroundScheduler(daemon=False)
 
-
-if __name__ == "__main__":
+def start_jobs():
     settings = load_settings()
 
     if "accounting" in settings:
         if settings['accounting']['enabled']:
-            sched.add_job(run_accounting,'interval',minutes=settings['accounting']['minutes_between_run'])
+            sched.add_job(run_accounting, 'interval', minutes=settings['accounting']['minutes_between_run'], args=[manual_client])
 
     if 'backup' in settings:
         if settings['backup']['enabled']:
-            sched.add_job(run_backup,'interval',minutes=settings['backup']['minutes_between_run'])
+            sched.add_job(run_backup, 'interval', minutes=settings['backup']['minutes_between_run'])
 
     if 'retention' in settings:
         if settings['retention']['enabled']:
-            sched.add_job(apply_retention_policies, 'interval', minutes=settings['retention']['minutes_between_run'], args=[settings['retention']['health_check_level'],manual_client])
+            sched.add_job(apply_retention_policies, 'interval', minutes=settings['retention']['minutes_between_run'], args=[settings['retention']['health_check_level']])
 
     if 'rollover' in settings:
         if settings['rollover']['enabled']:
-            sched.add_job(apply_rollover_policies, 'interval', minutes=settings['rollover']['minutes_between_run'], args=[manual_client])
+            sched.add_job(apply_rollover_policies, 'interval', minutes=settings['rollover']['minutes_between_run'])
 
     sched.start()
 
-    # # On service startup, immediately run retention, rollover, backup, and accounting
-    # run_backup(manual_client)
-    # apply_retention_policies(settings['retention']['health_check_level'], manual_client)
-    # apply_rollover_policies(manual_client)
-    # run_accounting(manual_client)
-
+if __name__ == "__main__":
+    settings_as_bytes = load_settings(format='bytes')
+    config_hash = hashlib.sha256(settings_as_bytes).hexdigest()
 
-    # if 'backup' in settings:
-    #     if settings['backup']['enabled']:
-    #         schedule.every(settings['backup']['minutes_between_run']).minutes.do(run_threaded, run_backup, "")
-    # #if settings['custom_checks']['enabled']:
-    # #    schedule.every(settings['custom_checks']['minutes_between_run']).minutes.do(run_threaded, run_custom_checks, "")
-    # # Example client info entry
-    # # "custom_checks": {
-    # #     "0": {
-    # #     "check": "/bin/bash /opt/elastic_stack/scripts/verify_acuity_custom_service.sh",
-    # #     "remediate": "/bin/bash /opt/elastic_stack/scripts/restart_acuity_custom_service.sh",
-    # #     "schedule": "15"
-    # #     }
-    # # },
-    # if 'retention' in settings:
-    #     if settings['retention']['enabled']:
-    #         schedule.every(settings['retention']['minutes_between_run']).minutes.do(run_threaded, apply_retention_policies, settings['retention']['health_check_level'], manual_client)
-    # if 'rollover' in settings:
-    #     if settings['rollover']['enabled']:
-    #         schedule.every(settings['rollover']['minutes_between_run']).minutes.do(run_threaded, apply_rollover_policies, manual_client)
+    start_jobs()
 
+    while True:
+        time.sleep(5)
+        settings_as_bytes = load_settings(format='bytes')
+        current_hash = hashlib.sha256(settings_as_bytes).hexdigest()
+        if current_hash != config_hash:
+            print("Configuration changed. Reloading jobs")
+            config_hash = current_hash
+            sched.shutdown()
+            if manual == 0:
+                sched = BackgroundScheduler(daemon=True)
+            else:
+                sched = BackgroundScheduler(daemon=False)
+            start_jobs()

retention.py

@@ -49,7 +49,7 @@ def apply_retention_to_old_indices(indices, index_retention_policies, client_con
     elastic_connection.close()
     return old_indices
 
-def apply_retention_policies(health_check_level, manual_client):
+def apply_retention_policies(health_check_level, manual_client=""):
     settings = load_settings()
     retry_count = 60
     sleep_time = 60
@@ -88,7 +88,7 @@ def apply_retention_policies(health_check_level, manual_client):
                             retry_count = retry_count - 1
                             print("Retry attempts left for retention operation set to " + str(retry_count) + " sleeping for " + str(sleep_time) + " seconds")
                             time.sleep(sleep_time)
-settings = load_settings()
+                    
 if __name__ == "__main__":
     import argparse
     from argparse import RawTextHelpFormatter

rollover.py

@@ -93,7 +93,7 @@ def rollover_client_indicies(client_config):
             print("Retry attempts left for rollover operation set to " + str(retry_count) + " sleeping for " + str(sleep_time) + " seconds")
             time.sleep(sleep_time)
 
-def apply_rollover_policies(manual_client):
+def apply_rollover_policies(manual_client=""):
     settings = load_settings()
     if settings['rollover']['enabled']:
         # Load all client configurations from /opt/maintenance/*.json