Merge pull request #1 from 4dn-dcic/core2

Core2

docs/source/index.rst

@@ -17,6 +17,15 @@ A shared component for Foursight and Foursight-CGAP.
    :target: https://foursight-core.readthedocs.io/en/latest/?badge=latest
    :alt: Documentation Status
 
+
+Dependency structure
+--------------------
+
+.. image: images/foursight-core-dependency-diagram-20201130.png
+   :height: 500
+   :target: images/foursight-core-dependency-diagram-20201130.png
+
+
 Beta version
 ------------
 

foursight_core/buckets.py

@@ -1,11 +1,10 @@
 import boto3
 import json
-from .chalicelib.vars import FOURSIGHT_PREFIX as PlaceholderPrefix
 
 class Buckets(object):
     """create and configure buckets for foursight"""
 
-    prefix = PlaceholderPrefix
+    prefix = 'placeholder_prefix'  # replace w/ e.g. 'foursight-cgap'
     envs = ['placeholder_env1', 'placeholder_env2']  # replace w/ e.g.  ['cgap', 'cgapdev', 'cgaptest', 'cgapwolf']
     default_acl = 'private'
     region = 'us-east-1'

foursight_core/check_utils.py

@@ -5,49 +5,44 @@
 import datetime
 import copy
 import json
-from .decorators import Decorators
 from .check_schema import CheckSchema
 from .exceptions import BadCheckSetup
-from .environment import Environment as PlaceholderEnvironment
-from .run_result import (
-    CheckResult as PlaceholderCheckResult,
-    ActionResult as PlaceholderActionResult
-)
+from .environment import Environment
+from .decorators import Decorators
+
 
 class CheckHandler(object):
     """
     Class CheckHandler is a collection of utils related to checks
     """
+    def __init__(self, foursight_prefix, check_package_name='foursight_core',
+                 check_setup_dir=dirname(__file__)):
+        self.prefix = foursight_prefix
+        self.check_package_name = check_package_name
+        self.decorators = Decorators(foursight_prefix)
+        self.CheckResult = self.decorators.CheckResult
+        self.ActionResult = self.decorators.ActionResult
+        self.CHECK_DECO = self.decorators.CHECK_DECO
+        self.ACTION_DECO = self.decorators.ACTION_DECO
+        self.environment = Environment(self.prefix)
 
-    # These must be overwritten for inherited classes
-    setup_dir = dirname(__file__)
-    CheckResult = PlaceholderCheckResult
-    ActionResult = PlaceholderActionResult
-    Environment = PlaceholderEnvironment
-    check_package_name = 'foursight_core'
-
-    @classmethod
-    def get_module_names(cls):
-        from .checks import __all__ as check_modules
-        return check_modules
-
-    # Methods below can be used as they are in inherited classes
-    def __init__(self):
         # read in the check_setup.json and parse it
-        setup_paths = glob.glob(self.setup_dir + "/check_setup.json")
+        setup_paths = glob.glob(check_setup_dir + "/check_setup.json")
         if not len(setup_paths) == 1:
             raise BadCheckSetup('Exactly one check_setup.json must be present in chalicelib!')
         with open(setup_paths[0], 'r') as jfile:
             self.CHECK_SETUP = json.load(jfile)
         # Validate and finalize CHECK_SETUP
         self.CHECK_SETUP = self.validate_check_setup(self.CHECK_SETUP)
 
-    @classmethod
-    def import_check_module(cls, module_name):
-        return importlib.import_module('.checks.' + module_name, cls.check_package_name)
+    def get_module_names(self):
+        check_modules = importlib.import_module('.checks', self.check_package_name)
+        return check_modules.__dict__["__all__"]
 
-    @classmethod
-    def get_check_strings(cls, specific_check=None):
+    def import_check_module(self, module_name):
+        return importlib.import_module('.checks.' + module_name, self.check_package_name)
+
+    def get_check_strings(self, specific_check=None):
         """
         Return a list of all formatted check strings (<module>/<check_name>) in system.
         By default runs on all checks (specific_check == None), but can be used
@@ -56,9 +51,9 @@ def get_check_strings(cls, specific_check=None):
         IMPORTANT: any checks in test_checks module are excluded.
         """
         all_checks = []
-        for mod_name in cls.get_module_names():
-            mod = cls.import_check_module(mod_name)
-            methods = cls.get_methods_by_deco(mod, Decorators.CHECK_DECO)
+        for mod_name in self.get_module_names():
+            mod = self.import_check_module(mod_name)
+            methods = self.get_methods_by_deco(mod, self.CHECK_DECO)
             for method in methods:
                 check_str = '/'.join([mod_name, method.__name__])
                 if specific_check and specific_check == method.__name__:
@@ -94,7 +89,7 @@ def validate_check_setup(self, check_setup):
         """
         found_checks = {}
         all_check_strings = self.get_check_strings()
-        all_environments = self.Environment().list_valid_schedule_environment_names()
+        all_environments = self.environment.list_valid_schedule_environment_names()
         # validate all checks
         for check_string in all_check_strings:
             mod_name, check_name = check_string.split('/')
@@ -150,15 +145,14 @@ def validate_check_setup(self, check_setup):
             check_setup[check_name]['module'] = found_checks[check_name]
         return check_setup
 
-    @classmethod
-    def get_action_strings(cls, specific_action=None):
+    def get_action_strings(self, specific_action=None):
         """
         Basically the same thing as get_check_strings, but for actions...
         """
         all_actions = []
-        for mod_name in cls.get_module_names():
-            mod = cls.import_check_module(mod_name)
-            methods = cls.get_methods_by_deco(mod, Decorators.ACTION_DECO)
+        for mod_name in self.get_module_names():
+            mod = self.import_check_module(mod_name)
+            methods = self.get_methods_by_deco(mod, self.ACTION_DECO)
             for method in methods:
                 act_str = '/'.join([mod_name, method.__name__])
                 if specific_action and specific_action == method.__name__:
@@ -291,8 +285,7 @@ def get_grouped_check_results(self, connection):
         grouped_list = [group for group in grouped_results.values()]
         return sorted(grouped_list, key=lambda v: v['_name'])
 
-    @classmethod
-    def run_check_or_action(cls, connection, check_str, check_kwargs):
+    def run_check_or_action(self, connection, check_str, check_kwargs):
         """
         Does validation of provided check_str, it's module, and kwargs.
         Determines by decorator whether the method is a check or action, then runs
@@ -315,35 +308,34 @@ def run_check_or_action(cls, connection, check_str, check_kwargs):
         if not isinstance(check_kwargs, dict):
             return ' '.join(['ERROR. Check kwargs must be a dict.', error_str])
         try:
-            check_mod = cls.import_check_module(mod_name)
+            check_mod = self.import_check_module(mod_name)
         except ModuleNotFoundError:
             return ' '.join(['ERROR. Check module is not valid.', error_str])
         except Exception as e:
             raise(e)
         check_method = check_mod.__dict__.get(check_name)
         if not check_method:
             return ' '.join(['ERROR. Check name is not valid.', error_str])
-        if not cls.check_method_deco(check_method, Decorators.CHECK_DECO) and \
-           not cls.check_method_deco(check_method, Decorators.ACTION_DECO):
+        if not self.check_method_deco(check_method, self.CHECK_DECO) and \
+           not self.check_method_deco(check_method, self.ACTION_DECO):
             return ' '.join(['ERROR. Check or action must use a decorator.', error_str])
         return check_method(connection, **check_kwargs)
 
-    @classmethod
-    def init_check_or_action_res(cls, connection, check):
+    def init_check_or_action_res(self, connection, check):
         """
         Use in cases where a string is provided that could be a check or an action
         Returns None if neither are valid. Tries checks first then actions.
         If successful, returns a CheckResult or ActionResult
         """
         is_action = False
         # determine whether it is a check or action
-        check_str = cls.get_check_strings(check)
+        check_str = self.get_check_strings(check)
         if not check_str:
-            check_str = cls.get_action_strings(check)
+            check_str = self.get_action_strings(check)
             is_action = True
         if not check_str: # not a check or an action. abort
             return None
-        return cls.ActionResult(connection, check) if is_action else cls.CheckResult(connection, check)
+        return self.ActionResult(connection, check) if is_action else self.CheckResult(connection, check)
 
     @classmethod
     def get_methods_by_deco(cls, mod, decorator):

foursight_core/checks/test_checks.py

@@ -1,9 +1,11 @@
-from ..run_result import CheckResult, ActionResult
 import random
 import time
-from ..decorators import Decorators
-check_function = Decorators().check_function
-action_function = Decorators().action_function
+
+# Use confchecks to import decorators object and its methods for each check module
+# rather than importing check_function, action_function, CheckResult, ActionResult
+# individually - they're now part of class Decorators in foursight-core::decorators
+# that requires initialization with foursight prefix.
+from .helpers.confchecks import *
 
 
 def test_function_unused():

foursight_core/decorators.py

@@ -1,31 +1,40 @@
 import traceback
 import signal
 import time
+import sys
 import os
 from functools import wraps
 from .check_schema import CheckSchema
 from .run_result import (
-    CheckResult as PlaceholderCheckResult,
-    ActionResult as PlaceholderActionResult
+    CheckResult as CheckResultBase,
+    ActionResult as ActionResultBase
 )
 from .exceptions import BadCheckOrAction
-from .sqs_utils import SQS as PlaceholderSQS
+from .sqs_utils import SQS
 
 
 class Decorators(object):
 
-    CheckResult = PlaceholderCheckResult
-    ActionResult = PlaceholderActionResult
-    SQS = PlaceholderSQS
-
     CHECK_DECO = 'check_function'
     ACTION_DECO = 'action_function'
     POLL_INTERVAL = 10  # check child process every 10 seconds
+    CHECK_TIMEOUT = 870  # in seconds. set to less than lambda limit (900 s)
 
-    def __init__(self):
-        self.CHECK_TIMEOUT = 870  # in seconds. set to less than lambda limit (900 s)
+    def __init__(self, foursight_prefix):
         if os.environ.get('CHECK_TIMEOUT'):
             self.set_timeout(os.environ.get('CHECK_TIMEOUT')) 
+        self.prefix = foursight_prefix
+        self.sqs = SQS(self.prefix)
+
+    def CheckResult(self, *args, **kwargs):
+        check = CheckResultBase(*args, **kwargs)
+        check.set_prefix(self.prefix)
+        return check
+
+    def ActionResult(self, *args, **kwargs):
+        action = ActionResultBase(*args, **kwargs)
+        action.set_prefix(self.prefix)
+        return action
 
     def set_timeout(self, timeout):
         try:
@@ -148,10 +157,10 @@ def timeout_handler(self, partials, signum=None, frame=None):
         or action with the appropriate information and then exits using sys.exit
         """
         if partials['is_check']:
-            result = CheckResult(partials['connection'], partials['name'])
+            result = self.CheckResult(partials['connection'], partials['name'])
             result.status = 'ERROR'
         else:
-            result = ActionResult(partials['connection'], partials['name'])
+            result = self.ActionResult(partials['connection'], partials['name'])
             result.status = 'FAIL'
         result.description = 'AWS lambda execution reached the time limit. Please see check/action code.'
         kwargs = partials['kwargs']
@@ -161,7 +170,7 @@ def timeout_handler(self, partials, signum=None, frame=None):
         # need to delete the sqs message and propogate if this is using the queue
         if kwargs.get('_run_info') and {'receipt', 'sqs_url'} <= set(kwargs['_run_info'].keys()):
             runner_input = {'sqs_url': kwargs['_run_info']['sqs_url']}
-            SQS.delete_message_and_propogate(runner_input, kwargs['_run_info']['receipt'])
+            self.sqs.delete_message_and_propogate(runner_input, kwargs['_run_info']['receipt'])
         sys.exit('-RUN-> TIMEOUT for execution of %s. Elapsed time is %s seconds; keep under %s.'
               % (partials['name'], kwargs['runtime_seconds'], self.CHECK_TIMEOUT))
 

foursight_core/environment.py

@@ -1,22 +1,16 @@
 import os
 import json
 from .s3_connection import S3Connection
-from .vars import (
-    FOURSIGHT_PREFIX as PlaceholderPrefix
-)
 
 
 class Environment(object):
 
-    # overwrite below in inherited classes
-    prefix = PlaceholderPrefix
-
-    def __init__(self):
+    def __init__(self, foursight_prefix):
+        self.prefix = foursight_prefix
         self.s3_connection = S3Connection(self.get_env_bucket_name())
 
-    @classmethod
-    def get_env_bucket_name(cls):
-        return cls.prefix + '-envs'
+    def get_env_bucket_name(self):
+        return self.prefix + '-envs'
 
     def list_environment_names(self):
         """

foursight_core/exceptions.py

@@ -20,3 +20,17 @@ def __init__(self, message=None):
         if message is None:
             message = "Malformed check setup found."
         super().__init__(message)
+
+
+class MissingFoursightPrefixException(Exception):
+    """
+    Generic exception for an issue with foursight prefix
+    not defined or initialized before using a method that
+    requires it.
+    __init__ takes some string error message
+    """
+    def __init__(self, message=None):
+        # default error message if none provided
+        if message is None:
+            message = "Foursight prefix is missing."
+        super().__init__(message)

foursight_core/run_result.py

@@ -3,17 +3,17 @@
 from abc import abstractmethod
 import json
 from .s3_connection import S3Connection
-from .vars import FOURSIGHT_PREFIX
-from .exceptions import BadCheckOrAction
+from .exceptions import (
+  BadCheckOrAction,
+  MissingFoursightPrefixException
+)
 
 
 class RunResult(object):
     """
     Generic class for CheckResult and ActionResult. Contains methods common
     to both.
     """
-    prefix = FOURSIGHT_PREFIX
-
     def __init__(self, connections, name):
         self.fs_conn = connections
         self.connections = connections.connections
@@ -24,6 +24,9 @@ def __init__(self, connections, name):
         self.extension = ".json"
         self.kwargs = {}
 
+    def set_prefix(self, foursight_prefix):
+        self.prefix = foursight_prefix
+
     def get_s3_object(self, key):
         """
         Returns None if not present, otherwise returns a JSON parsed res.
@@ -158,6 +161,8 @@ def record_run_info(self):
         Returns True on success, False otherwise
         """
         run_id = self.kwargs['_run_info']['run_id']
+        if not hasattr(self, 'prefix'):
+            raise MissingFoursightPrefixException("foursight prefix must be defined using set_prefix")
         s3_connection = S3Connection(self.prefix + '-runs')
         record_key = '/'.join([run_id, self.name])
         resp = s3_connection.put_object(record_key, json.dumps(self.status))