Adapt to dropping of service_ptr_t and the use of the smart_holder br…

…anch of pybind11

doc/getting_started.rst

@@ -39,8 +39,9 @@ Boost 1.46, rapidjson 1.0, yaml-cpp, rabbitmq-c, and quill.
 Everything should be available from standard package managers, except Quill, which is not in apt (for Ubuntu/Debian).
 
 The Python wrapping of C++ code is done with `Pybind11 <https://pybind11.readthedocs.io/>`_.  
-Version 2.6.0 or higher is required.  
-It can be installed from most package managers or by following their instructions. 
+Version 2.6.0 or higher is required, and it must be on the `smart_holder <https://github.com/pybind/pybind11/tree/smart_holder>`_ branch. 
+The smart_holder branch closely tracks the master branch but includes the critical smart_holder feature. 
+See the `smart_holder docs <https://github.com/pybind/pybind11/blob/smart_holder/README_smart_holder.rst>`_ for more info.
 
 You will need `CMake <https://cmake.org/>`_ to build the C++ wrappers.  
 Version 3.12 or higher is required.  

dripline/core/service.py

@@ -55,11 +55,11 @@ class Service(_Service, ObjectCreator):
     using `process_message(message)`.)
     '''
 
-    def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling=False, 
+    def __init__(self, name, make_connection=True, endpoints=None, add_endpoints_now=True, enable_scheduling=False, 
                  broadcast_key='broadcast', loop_timeout_ms=1000, 
                  message_wait_ms=1000, heartbeat_interval_s=60, 
                  username=None, password=None, authentication_obj=None,
-                 dripline_mesh=None):
+                 dripline_mesh=None, **kwargs):
         '''
         Configures a service with the necessary parameters.
 
@@ -69,6 +69,10 @@ def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling
                 The name of the endpoint, which specifies the binding key for request messages sent to this service.
             make_connection : bool, optional
                 Flag for indicating whether the service should connect to the broker; if false, the service will be in "dry-run" mode.
+            endpoints: list, optional
+                List of endpoint configurations to add to the service
+            add_endpoints_now: bool, optional
+                Flag to determine whether endpoints are built in the __init__() function; if False, then endpoints must be built later with add_endpoints_from_config()
             enable_scheduling : bool, optional
             broadcast_key : string, optional
             loop_timeout_ms : int, optional
@@ -94,7 +98,6 @@ def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling
             dripline_mesh : dict, optional
                 Provide optional dripline mesh configuration information (see dripline_config for more information)
         '''
-
         # Final dripline_mesh config should be the default updated by the parameters passed by the caller
         dripline_config = DriplineConfig().to_python()
         dripline_config.update({} if dripline_mesh is None else dripline_mesh)
@@ -128,14 +131,23 @@ def __init__(self, name, make_connection=True, endpoints=None, enable_scheduling
         _Service.__init__(self, config=scarab.to_param(config), auth=auth, make_connection=make_connection)
 
         # Endpoints
-        if endpoints is not None:
-            for an_endpoint_conf in endpoints:
+        self.endpoint_configs = endpoints
+        if( add_endpoints_now ):
+            self.add_endpoints_from_config()
+
+        if kwargs:
+            logger.debug(f'Service received some kwargs that it doesn\'t handle, which will be ignored: {kwargs}')
+
+    def add_endpoints_from_config(self):
+        if self.endpoint_configs is not None:
+            for an_endpoint_conf in self.endpoint_configs:
                 an_endpoint = self.create_object(an_endpoint_conf, 'Endpoint')
                 self.add_child( an_endpoint )
                 if getattr(an_endpoint, 'log_interval', datetime.timedelta(seconds=0)) > datetime.timedelta(seconds=0):
                     logger.debug("queue up start logging for '{}'".format(an_endpoint.name))
                     an_endpoint.start_logging()
 
+
     def result_to_scarab_payload(self, result: str):
         """
         Intercept result values and throw error if scarab is unable to convert to param

dripline/implementations/postgres_interface.py

@@ -26,8 +26,17 @@
 
 
 __all__.append('PostgreSQLInterface')
-class PostgreSQLInterface(Service):
+class PostgreSQLInterface():
     '''
+    A service's interface to a PostgreSQL database.
+
+    This is a mixin class for services that need to connect to PostgreSQL databases.
+
+    To use with a Service, the following order of operations must be followed in the derived class's __init__() function:
+    1. Initialize the Service with add_endpoints_now=False
+    2. Initialize this PostgreSQLInterface
+    3. Connect to the database with PostgreSQLInterface.connect_to_db()
+    3. Add endpoints using Service.add_endpoints_from_config()
     '''
 
     def __init__(self, database_name, database_server, **kwargs):
@@ -36,16 +45,21 @@ def __init__(self, database_name, database_server, **kwargs):
             database_name (str): name of the 'database' to connect to within the database server
             database_server (str): network resolvable hostname of database server
         '''
+        self.database_name = database_name
+        self.database_server = database_server
 
         if not 'sqlalchemy' in globals():
             raise ImportError('SQLAlchemy not found, required for PostgreSQLInterface class')
 
-        Service.__init__(self, **kwargs)
 
-        if not self.auth.has('postgres'):
+    def connect_to_db(self, auth):
+        '''
+        Connect to the postgres database using the provided information
+        '''
+        logger.warning(f'auth spec: {auth.spec}')
+        if not auth.has('postgres'):
             raise RuntimeError('Authentication is missing "postgres" login details')
-
-        self._connect_to_db(database_server, database_name, self.auth)
+        self._connect_to_db(self.database_server, self.database_name, auth)
 
     def _connect_to_db(self, database_server, database_name, auth):
         '''
@@ -55,8 +69,12 @@ def _connect_to_db(self, database_server, database_name, auth):
         self.engine = sqlalchemy.create_engine(engine_str)
         self.meta = sqlalchemy.MetaData()
 
-    def add_child(self, endpoint):
-        Service.add_child(self, endpoint)
+    def add_child_table(self, endpoint):
+        '''
+        Add a child endpoint that is an SQLTable.
+
+        This is meant to be called from the serivce that derives from this class, as part of overriding add_child().
+        '''
         if isinstance(endpoint, SQLTable):
             logger.debug(f'Adding sqlalchemy.Table object for "{endpoint.table_name}" to Endpoint')
             endpoint.table = sqlalchemy.Table(endpoint.table_name, self.meta, autoload_with=self.engine, schema=endpoint.schema)

dripline/implementations/postgres_sensor_logger.py

@@ -28,14 +28,19 @@ class PostgresSensorLogger(AlertConsumer, PostgreSQLInterface):
     def __init__(self, insertion_table_endpoint_name, **kwargs):
         '''
         '''
-        AlertConsumer.__init__(self, **kwargs)
+        AlertConsumer.__init__(self, add_endpoints_now=False, **kwargs)
         PostgreSQLInterface.__init__(self, **kwargs)
 
         self.insertion_table_endpoint_name = insertion_table_endpoint_name
 
+        self.connect_to_db(self.auth)
+
+        self.add_endpoints_from_config()
+
     # add_endpoint is a mess here because of the diamond inheritance, so let's be explicit
     def add_child(self, endpoint):
-        PostgreSQLInterface.add_child(self, endpoint)
+        AlertConsumer.add_child(self, endpoint)
+        self.add_child_table(endpoint)
 
     def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp):
         this_data_table = self.sync_children[self.insertion_table_endpoint_name]

module_bindings/dripline_core/_endpoint_pybind.hh

@@ -17,12 +17,12 @@ namespace dripline_pybind
         std::list< std::string > all_items;
 
         all_items.push_back( "_Endpoint" );
-        pybind11::class_< dripline::endpoint, _endpoint_trampoline, std::shared_ptr< dripline::endpoint > >( mod, "_Endpoint", "Endpoint binding" )
+        pybind11::classh< dripline::endpoint, _endpoint_trampoline >( mod, "_Endpoint", "Endpoint binding" )
             .def( pybind11::init< const std::string& >(), DL_BIND_CALL_GUARD_STREAMS )
 
             // mv_ properties
             .def_property_readonly( "name", (std::string& (dripline::endpoint::*)()) &dripline::endpoint::name )
-            .def_property_readonly( "service", ( dripline::service_ptr_t& (dripline::endpoint::*)()) &dripline::endpoint::service )
+            .def_property_readonly( "service", ( dripline::service& (dripline::endpoint::*)()) &dripline::endpoint::parent )
 
 
             // deal with messages

module_bindings/dripline_core/_endpoint_trampoline.hh

@@ -8,7 +8,7 @@
 namespace dripline_pybind
 {
 
-    class _endpoint_trampoline : public dripline::endpoint
+    class _endpoint_trampoline : public dripline::endpoint, public pybind11::trampoline_self_life_support
     {
 
         public:

module_bindings/dripline_core/_service_pybind.hh

@@ -22,13 +22,12 @@ namespace dripline_pybind
     {
         std::list< std::string > all_items;
         all_items.push_back( "_Service" );
-        pybind11::class_< _service,
+        pybind11::classh< _service,
                           _service_trampoline,
                           dripline::core,
                           dripline::endpoint,
                           dripline::scheduler<>,
-                          scarab::cancelable,
-                          std::shared_ptr< _service >
+                          scarab::cancelable
                         >( mod, "_Service", "Service binding" )
             .def( pybind11::init< const scarab::param_node&,
                                   const scarab::authentication&,

module_bindings/dripline_core/_service_trampoline.hh

@@ -8,7 +8,7 @@
 namespace dripline_pybind
 {
     // we need an extra class so that we can make private/protected methods public for binding
-    class _service : public dripline::service
+    class _service : public dripline::service, public pybind11::trampoline_self_life_support
     {
         public:
             //inherit constructor

module_bindings/dripline_core/message_pybind.hh

@@ -30,7 +30,7 @@ namespace dripline_pybind
          message
          ********/
         all_items.push_back( "Message" );
-        pybind11::class_< dripline::message, message_trampoline, std::shared_ptr< dripline::message > > message( mod, "Message", "base class for all dripline messages" );
+        pybind11::classh< dripline::message, message_trampoline > message( mod, "Message", "base class for all dripline messages" );
 
         // internal types
         pybind11::enum_< dripline::message::encoding >( message, "encoding", "mime-type of message encoding" )
@@ -85,7 +85,7 @@ namespace dripline_pybind
         using namespace pybind11::literals;
 
         all_items.push_back( "MsgRequest" );
-        pybind11::class_< dripline::msg_request, msg_request_trampoline, std::shared_ptr< dripline::msg_request >
+        pybind11::classh< dripline::msg_request, msg_request_trampoline
                         >( mod, "MsgRequest", message, "dripline messages containing a request to be sent to an endpoint" )
             // constructor(s)
             .def( pybind11::init< >() )
@@ -154,7 +154,7 @@ namespace dripline_pybind
          msg_reply
          ************/
         all_items.push_back( "MsgReply" );
-        pybind11::class_< dripline::msg_reply, msg_reply_trampoline, std::shared_ptr< dripline::msg_reply >
+        pybind11::classh< dripline::msg_reply, msg_reply_trampoline
                         >( mod, "MsgReply", message, "dripline messages containing a reply to a previously received request" )
             // constructor(s)
             .def( pybind11::init< >() )
@@ -214,7 +214,7 @@ namespace dripline_pybind
          msg_alert
          ************/
         all_items.push_back( "MsgAlert" );
-        pybind11::class_< dripline::msg_alert, msg_alert_trampoline, std::shared_ptr< dripline::msg_alert >
+        pybind11::classh< dripline::msg_alert, msg_alert_trampoline
                         >( mod, "MsgAlert", message, "dripline message containing alert information" )
             // constructor(s)
             .def( pybind11::init< >() )

module_bindings/dripline_core/message_trampoline.hh

@@ -7,7 +7,7 @@
 
 namespace dripline_pybind
 {
-    class message_trampoline : public dripline::message
+    class message_trampoline : public dripline::message, public pybind11::trampoline_self_life_support
     {
         public:
             using dripline::message::message; // Inheriting constructors
@@ -45,7 +45,7 @@ namespace dripline_pybind
 
     };
 
-    class msg_request_trampoline : public dripline::msg_request
+    class msg_request_trampoline : public dripline::msg_request, public pybind11::trampoline_self_life_support
     {
         public:
             using dripline::msg_request::msg_request;
@@ -72,7 +72,7 @@ namespace dripline_pybind
 
     };
 
-    class msg_reply_trampoline : public dripline::msg_reply
+    class msg_reply_trampoline : public dripline::msg_reply, public pybind11::trampoline_self_life_support
     {
         public:
             using dripline::msg_reply::msg_reply;
@@ -99,7 +99,7 @@ namespace dripline_pybind
 
     };
 
-    class msg_alert_trampoline : public dripline::msg_alert
+    class msg_alert_trampoline : public dripline::msg_alert, public pybind11::trampoline_self_life_support
     {
         public:
             using dripline::msg_alert::msg_alert;