diff --git a/doc/getting_started.rst b/doc/getting_started.rst index c5106dc..8035f89 100644 --- a/doc/getting_started.rst +++ b/doc/getting_started.rst @@ -39,8 +39,9 @@ Boost 1.46, rapidjson 1.0, yaml-cpp, rabbitmq-c, and quill. Everything should be available from standard package managers, except Quill, which is not in apt (for Ubuntu/Debian). The Python wrapping of C++ code is done with `Pybind11 `_. -Version 2.6.0 or higher is required. -It can be installed from most package managers or by following their instructions. +Version 2.6.0 or higher is required, and it must be on the `smart_holder `_ branch. +The smart_holder branch closely tracks the master branch but includes the critical smart_holder feature. +See the `smart_holder docs `_ for more info. You will need `CMake `_ to build the C++ wrappers. Version 3.12 or higher is required. diff --git a/dripline/core/service.py b/dripline/core/service.py index 61ff88f..3a18522 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -55,11 +55,11 @@ class Service(_Service, ObjectCreator): using `process_message(message)`.) ''' - def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling=False, + def __init__(self, name, make_connection=True, endpoints=None, add_endpoints_now=True, enable_scheduling=False, broadcast_key='broadcast', loop_timeout_ms=1000, message_wait_ms=1000, heartbeat_interval_s=60, username=None, password=None, authentication_obj=None, - dripline_mesh=None): + dripline_mesh=None, **kwargs): ''' Configures a service with the necessary parameters. @@ -69,6 +69,10 @@ def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling The name of the endpoint, which specifies the binding key for request messages sent to this service. make_connection : bool, optional Flag for indicating whether the service should connect to the broker; if false, the service will be in "dry-run" mode. + endpoints: list, optional + List of endpoint configurations to add to the service + add_endpoints_now: bool, optional + Flag to determine whether endpoints are built in the __init__() function; if False, then endpoints must be built later with add_endpoints_from_config() enable_scheduling : bool, optional broadcast_key : string, optional loop_timeout_ms : int, optional @@ -94,7 +98,6 @@ def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling dripline_mesh : dict, optional Provide optional dripline mesh configuration information (see dripline_config for more information) ''' - # Final dripline_mesh config should be the default updated by the parameters passed by the caller dripline_config = DriplineConfig().to_python() dripline_config.update({} if dripline_mesh is None else dripline_mesh) @@ -128,14 +131,23 @@ def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling _Service.__init__(self, config=scarab.to_param(config), auth=auth, make_connection=make_connection) # Endpoints - if endpoints is not None: - for an_endpoint_conf in endpoints: + self.endpoint_configs = endpoints + if( add_endpoints_now ): + self.add_endpoints_from_config() + + if kwargs: + logger.debug(f'Service received some kwargs that it doesn\'t handle, which will be ignored: {kwargs}') + + def add_endpoints_from_config(self): + if self.endpoint_configs is not None: + for an_endpoint_conf in self.endpoint_configs: an_endpoint = self.create_object(an_endpoint_conf, 'Endpoint') self.add_child( an_endpoint ) if getattr(an_endpoint, 'log_interval', datetime.timedelta(seconds=0)) > datetime.timedelta(seconds=0): logger.debug("queue up start logging for '{}'".format(an_endpoint.name)) an_endpoint.start_logging() + def result_to_scarab_payload(self, result: str): """ Intercept result values and throw error if scarab is unable to convert to param diff --git a/dripline/implementations/postgres_interface.py b/dripline/implementations/postgres_interface.py index 881dc06..3921a4d 100644 --- a/dripline/implementations/postgres_interface.py +++ b/dripline/implementations/postgres_interface.py @@ -26,8 +26,17 @@ __all__.append('PostgreSQLInterface') -class PostgreSQLInterface(Service): +class PostgreSQLInterface(): ''' + A service's interface to a PostgreSQL database. + + This is a mixin class for services that need to connect to PostgreSQL databases. + + To use with a Service, the following order of operations must be followed in the derived class's __init__() function: + 1. Initialize the Service with add_endpoints_now=False + 2. Initialize this PostgreSQLInterface + 3. Connect to the database with PostgreSQLInterface.connect_to_db() + 3. Add endpoints using Service.add_endpoints_from_config() ''' def __init__(self, database_name, database_server, **kwargs): @@ -36,16 +45,21 @@ def __init__(self, database_name, database_server, **kwargs): database_name (str): name of the 'database' to connect to within the database server database_server (str): network resolvable hostname of database server ''' + self.database_name = database_name + self.database_server = database_server if not 'sqlalchemy' in globals(): raise ImportError('SQLAlchemy not found, required for PostgreSQLInterface class') - Service.__init__(self, **kwargs) - if not self.auth.has('postgres'): + def connect_to_db(self, auth): + ''' + Connect to the postgres database using the provided information + ''' + logger.warning(f'auth spec: {auth.spec}') + if not auth.has('postgres'): raise RuntimeError('Authentication is missing "postgres" login details') - - self._connect_to_db(database_server, database_name, self.auth) + self._connect_to_db(self.database_server, self.database_name, auth) def _connect_to_db(self, database_server, database_name, auth): ''' @@ -55,8 +69,12 @@ def _connect_to_db(self, database_server, database_name, auth): self.engine = sqlalchemy.create_engine(engine_str) self.meta = sqlalchemy.MetaData() - def add_child(self, endpoint): - Service.add_child(self, endpoint) + def add_child_table(self, endpoint): + ''' + Add a child endpoint that is an SQLTable. + + This is meant to be called from the serivce that derives from this class, as part of overriding add_child(). + ''' if isinstance(endpoint, SQLTable): logger.debug(f'Adding sqlalchemy.Table object for "{endpoint.table_name}" to Endpoint') endpoint.table = sqlalchemy.Table(endpoint.table_name, self.meta, autoload_with=self.engine, schema=endpoint.schema) diff --git a/dripline/implementations/postgres_sensor_logger.py b/dripline/implementations/postgres_sensor_logger.py index a95b9e7..27a623f 100644 --- a/dripline/implementations/postgres_sensor_logger.py +++ b/dripline/implementations/postgres_sensor_logger.py @@ -28,14 +28,19 @@ class PostgresSensorLogger(AlertConsumer, PostgreSQLInterface): def __init__(self, insertion_table_endpoint_name, **kwargs): ''' ''' - AlertConsumer.__init__(self, **kwargs) + AlertConsumer.__init__(self, add_endpoints_now=False, **kwargs) PostgreSQLInterface.__init__(self, **kwargs) self.insertion_table_endpoint_name = insertion_table_endpoint_name + self.connect_to_db(self.auth) + + self.add_endpoints_from_config() + # add_endpoint is a mess here because of the diamond inheritance, so let's be explicit def add_child(self, endpoint): - PostgreSQLInterface.add_child(self, endpoint) + AlertConsumer.add_child(self, endpoint) + self.add_child_table(endpoint) def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): this_data_table = self.sync_children[self.insertion_table_endpoint_name] diff --git a/module_bindings/dripline_core/_endpoint_pybind.hh b/module_bindings/dripline_core/_endpoint_pybind.hh index da8e547..61e44c8 100644 --- a/module_bindings/dripline_core/_endpoint_pybind.hh +++ b/module_bindings/dripline_core/_endpoint_pybind.hh @@ -17,12 +17,12 @@ namespace dripline_pybind std::list< std::string > all_items; all_items.push_back( "_Endpoint" ); - pybind11::class_< dripline::endpoint, _endpoint_trampoline, std::shared_ptr< dripline::endpoint > >( mod, "_Endpoint", "Endpoint binding" ) + pybind11::classh< dripline::endpoint, _endpoint_trampoline >( mod, "_Endpoint", "Endpoint binding" ) .def( pybind11::init< const std::string& >(), DL_BIND_CALL_GUARD_STREAMS ) // mv_ properties .def_property_readonly( "name", (std::string& (dripline::endpoint::*)()) &dripline::endpoint::name ) - .def_property_readonly( "service", ( dripline::service_ptr_t& (dripline::endpoint::*)()) &dripline::endpoint::service ) + .def_property_readonly( "service", ( dripline::service& (dripline::endpoint::*)()) &dripline::endpoint::parent ) // deal with messages diff --git a/module_bindings/dripline_core/_endpoint_trampoline.hh b/module_bindings/dripline_core/_endpoint_trampoline.hh index dc34a7c..48ba975 100644 --- a/module_bindings/dripline_core/_endpoint_trampoline.hh +++ b/module_bindings/dripline_core/_endpoint_trampoline.hh @@ -8,7 +8,7 @@ namespace dripline_pybind { - class _endpoint_trampoline : public dripline::endpoint + class _endpoint_trampoline : public dripline::endpoint, public pybind11::trampoline_self_life_support { public: diff --git a/module_bindings/dripline_core/_service_pybind.hh b/module_bindings/dripline_core/_service_pybind.hh index 1ca4c95..244876a 100644 --- a/module_bindings/dripline_core/_service_pybind.hh +++ b/module_bindings/dripline_core/_service_pybind.hh @@ -22,13 +22,12 @@ namespace dripline_pybind { std::list< std::string > all_items; all_items.push_back( "_Service" ); - pybind11::class_< _service, + pybind11::classh< _service, _service_trampoline, dripline::core, dripline::endpoint, dripline::scheduler<>, - scarab::cancelable, - std::shared_ptr< _service > + scarab::cancelable >( mod, "_Service", "Service binding" ) .def( pybind11::init< const scarab::param_node&, const scarab::authentication&, diff --git a/module_bindings/dripline_core/_service_trampoline.hh b/module_bindings/dripline_core/_service_trampoline.hh index 0abfa68..5c7047e 100644 --- a/module_bindings/dripline_core/_service_trampoline.hh +++ b/module_bindings/dripline_core/_service_trampoline.hh @@ -8,7 +8,7 @@ namespace dripline_pybind { // we need an extra class so that we can make private/protected methods public for binding - class _service : public dripline::service + class _service : public dripline::service, public pybind11::trampoline_self_life_support { public: //inherit constructor diff --git a/module_bindings/dripline_core/message_pybind.hh b/module_bindings/dripline_core/message_pybind.hh index edbc2a8..c4fe572 100644 --- a/module_bindings/dripline_core/message_pybind.hh +++ b/module_bindings/dripline_core/message_pybind.hh @@ -30,7 +30,7 @@ namespace dripline_pybind message ********/ all_items.push_back( "Message" ); - pybind11::class_< dripline::message, message_trampoline, std::shared_ptr< dripline::message > > message( mod, "Message", "base class for all dripline messages" ); + pybind11::classh< dripline::message, message_trampoline > message( mod, "Message", "base class for all dripline messages" ); // internal types pybind11::enum_< dripline::message::encoding >( message, "encoding", "mime-type of message encoding" ) @@ -85,7 +85,7 @@ namespace dripline_pybind using namespace pybind11::literals; all_items.push_back( "MsgRequest" ); - pybind11::class_< dripline::msg_request, msg_request_trampoline, std::shared_ptr< dripline::msg_request > + pybind11::classh< dripline::msg_request, msg_request_trampoline >( mod, "MsgRequest", message, "dripline messages containing a request to be sent to an endpoint" ) // constructor(s) .def( pybind11::init< >() ) @@ -154,7 +154,7 @@ namespace dripline_pybind msg_reply ************/ all_items.push_back( "MsgReply" ); - pybind11::class_< dripline::msg_reply, msg_reply_trampoline, std::shared_ptr< dripline::msg_reply > + pybind11::classh< dripline::msg_reply, msg_reply_trampoline >( mod, "MsgReply", message, "dripline messages containing a reply to a previously received request" ) // constructor(s) .def( pybind11::init< >() ) @@ -214,7 +214,7 @@ namespace dripline_pybind msg_alert ************/ all_items.push_back( "MsgAlert" ); - pybind11::class_< dripline::msg_alert, msg_alert_trampoline, std::shared_ptr< dripline::msg_alert > + pybind11::classh< dripline::msg_alert, msg_alert_trampoline >( mod, "MsgAlert", message, "dripline message containing alert information" ) // constructor(s) .def( pybind11::init< >() ) diff --git a/module_bindings/dripline_core/message_trampoline.hh b/module_bindings/dripline_core/message_trampoline.hh index a597646..003df58 100644 --- a/module_bindings/dripline_core/message_trampoline.hh +++ b/module_bindings/dripline_core/message_trampoline.hh @@ -7,7 +7,7 @@ namespace dripline_pybind { - class message_trampoline : public dripline::message + class message_trampoline : public dripline::message, public pybind11::trampoline_self_life_support { public: using dripline::message::message; // Inheriting constructors @@ -45,7 +45,7 @@ namespace dripline_pybind }; - class msg_request_trampoline : public dripline::msg_request + class msg_request_trampoline : public dripline::msg_request, public pybind11::trampoline_self_life_support { public: using dripline::msg_request::msg_request; @@ -72,7 +72,7 @@ namespace dripline_pybind }; - class msg_reply_trampoline : public dripline::msg_reply + class msg_reply_trampoline : public dripline::msg_reply, public pybind11::trampoline_self_life_support { public: using dripline::msg_reply::msg_reply; @@ -99,7 +99,7 @@ namespace dripline_pybind }; - class msg_alert_trampoline : public dripline::msg_alert + class msg_alert_trampoline : public dripline::msg_alert, public pybind11::trampoline_self_life_support { public: using dripline::msg_alert::msg_alert;