Mindsdb starting speedup, part 2 (mindsdb#9599)

ramvzz · Oct 4, 2024 · a79e682 · a79e682
1 parent 263a234
commit a79e682
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 34 deletions.
diff --git a/mindsdb/api/http/namespaces/handlers.py b/mindsdb/api/http/namespaces/handlers.py
@@ -50,7 +50,9 @@ class HandlerIcon(Resource):
     @api_endpoint_metrics('GET', '/handlers/handler/icon')
     def get(self, handler_name):
         try:
-            handler_meta = ca.integration_controller.get_handler_meta(handler_name)
+            handler_meta = ca.integration_controller.get_handlers_metadata().get(handler_name)
+            if handler_meta is None:
+                return http_error(HTTPStatus.NOT_FOUND, 'Icon not found', f'Icon for {handler_name} not found')
             icon_name = handler_meta['icon']['name']
             handler_folder = handler_meta['import']['folder']
             mindsdb_path = Path(importlib.util.find_spec('mindsdb').origin).parent

diff --git a/mindsdb/integrations/libs/process_cache.py b/mindsdb/integrations/libs/process_cache.py
@@ -222,11 +222,11 @@ def init(self):
         is_cloud = config.get('cloud', False) # noqa
 
         if config['ml_task_queue']['type'] != 'redis':
-            lightwood_handler = integration_controller.get_handler_module('lightwood')
-            if lightwood_handler is not None and lightwood_handler.Handler is not None:
-                preload_handlers[lightwood_handler.Handler] = 4 if is_cloud else 1
-
             if is_cloud:
+                lightwood_handler = integration_controller.get_handler_module('lightwood')
+                if lightwood_handler is not None and lightwood_handler.Handler is not None:
+                    preload_handlers[lightwood_handler.Handler] = 4 if is_cloud else 1
+
                 huggingface_handler = integration_controller.get_handler_module('huggingface')
                 if huggingface_handler is not None and huggingface_handler.Handler is not None:
                     preload_handlers[huggingface_handler.Handler] = 1

diff --git a/mindsdb/interfaces/database/integrations.py b/mindsdb/interfaces/database/integrations.py
@@ -656,28 +656,6 @@ def _get_handler_meta(self, handler_name):
         for attr in module_attrs:
             handler_meta[attr] = getattr(module, attr)
 
-        # region icon
-        if hasattr(module, 'icon_path'):
-            try:
-                icon_path = handler_dir.joinpath(module.icon_path)
-                icon_type = icon_path.name[icon_path.name.rfind('.') + 1:].lower()
-
-                if icon_type == 'svg':
-                    with open(str(icon_path), 'rt') as f:
-                        handler_meta['icon'] = {
-                            'data': f.read()
-                        }
-                else:
-                    with open(str(icon_path), 'rb') as f:
-                        handler_meta['icon'] = {
-                            'data': base64.b64encode(f.read()).decode('utf-8')
-                        }
-
-                handler_meta['icon']['name'] = icon_path.name
-                handler_meta['icon']['type'] = icon_type
-            except Exception as e:
-                logger.error(f'Error reading icon for {handler_folder_name}, {e}!')
-
         # endregion
         if hasattr(module, 'permanent'):
             handler_meta['permanent'] = module.permanent
@@ -689,6 +667,26 @@ def _get_handler_meta(self, handler_name):
 
         return handler_meta
 
+    def _get_handler_icon(self, handler_dir, icon_path):
+        icon = {}
+        try:
+            icon_path = handler_dir.joinpath(icon_path)
+            icon_type = icon_path.name[icon_path.name.rfind('.') + 1:].lower()
+
+            if icon_type == 'svg':
+                with open(str(icon_path), 'rt') as f:
+                    icon['data'] = f.read()
+            else:
+                with open(str(icon_path), 'rb') as f:
+                    icon['data'] = base64.b64encode(f.read()).decode('utf-8')
+
+            icon['name'] = icon_path.name
+            icon['type'] = icon_type
+
+        except Exception as e:
+            logger.error(f'Error reading icon for {handler_dir}, {e}!')
+        return icon
+
     def _load_handler_modules(self):
         mindsdb_path = Path(importlib.util.find_spec('mindsdb').origin).parent
         handlers_path = mindsdb_path.joinpath('integrations/handlers')
@@ -720,15 +718,12 @@ def _load_handler_modules(self):
                 'name': handler_name,
                 'permanent': handler_info.get('permanent', False),
             }
+            if 'icon_path' in handler_info:
+                icon = self._get_handler_icon(handler_dir, handler_info['icon_path'])
+                if icon:
+                    handler_meta['icon'] = icon
             self.handlers_import_status[handler_name] = handler_meta
 
-        # import all handlers in thread
-        def import_handlers():
-            self.get_handlers_import_status()
-
-        thread = threading.Thread(target=import_handlers)
-        thread.start()
-
     def _get_handler_info(self, handler_dir: Path):
 
         init_file = handler_dir / '__init__.py'