V2.6.0 (#188)

_cmd.py

@@ -29,8 +29,6 @@
 from uuid import uuid1 as uuid, UUID
 from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter
 
-from colorama import Fore
-
 from core import ObdHome
 from _stdio import IO, FormtatText
 from _lock import LockMode
@@ -668,7 +666,7 @@ def background_telemetry_task(self, obd, demploy_name=None):
         if demploy_name is None:
             demploy_name = self.cmds[0]
         data = json.dumps(self.get_obd_namespaces_data(obd))
-        LocalClient.execute_command_background(f"nohup obd telemetry post {demploy_name} --data='{data}' >/dev/null 2>&1 &")
+        LocalClient.execute_command_background("nohup obd telemetry post %s --data='%s' >/dev/null 2>&1 &" % (demploy_name, data))
 
 
 class ClusterConfigStyleChange(ClusterMirrorCommand):
@@ -812,7 +810,7 @@ def _do_command(self, obd):
             res = obd.deploy_cluster(self.cmds[0])
             self.background_telemetry_task(obd)
             if res:
-                obd.stdio.print(FormtatText('Please execute ` obd cluster start %s ` to start' % self.cmds[0], Fore.GREEN))
+                obd.stdio.print(FormtatText.success('Please execute ` obd cluster start %s ` to start' % self.cmds[0]))
             return res
         else:
             return self._show_help()
@@ -1603,7 +1601,7 @@ def _do_command(self, obd):
         except:
             ROOT_IO.print('%s is not trace id' % trace_id)
             return False
-        cmd = 'grep -h "\[{}\]" {}* | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id)
+        cmd = 'grep -h "\[{}\]" $(ls -tr {}*) | sed "s/\[{}\] //g" '.format(trace_id, log_dir, trace_id)
         data = LocalClient.execute_command(cmd)
         ROOT_IO.print(data.stdout)
         return True
@@ -1861,6 +1859,8 @@ class ObdiagAnalyzeLogCommand(ObdiagAnalyzeMirrorCommand):
 
     def init(self, cmd, args):
         super(ObdiagAnalyzeLogCommand, self).init(cmd, args)
+        self.parser.set_usage(
+            '%s <deploy name> [options]' % self.prev_cmd)
         return self
 
     @property
@@ -1884,6 +1884,8 @@ class ObdiagAnalyzeFltTraceCommand(ObdiagAnalyzeMirrorCommand):
 
     def init(self, cmd, args):
         super(ObdiagAnalyzeFltTraceCommand, self).init(cmd, args)
+        self.parser.set_usage(
+            '%s <deploy name> [options]' % self.prev_cmd)
         return self
 
     @property

_deploy.py

@@ -199,11 +199,13 @@ def _get_config(self, item, key):
             key = InnerConfigKey.to_keyword(key)
         return self[item].get(key)
 
+
 class InnerConfigKeywords(object):
 
     DEPLOY_INSTALL_MODE = 'deploy_install_mode'
     DEPLOY_BASE_DIR = 'deploy_base_dir'
 
+
 class InnerConfig(object):
 
     def __init__(self, path, yaml_loader):
@@ -580,6 +582,10 @@ def update_component_attr(self, key, value, save=True):
         self._inner_config.update_attr(key, value)
         return self._deploy_config.dump() if save else True
 
+    def del_component_attr(self, key, save=True):
+        self._inner_config.del_attr(key)
+        return self._deploy_config.dump() if save else True
+
     def get_component_attr(self, key):
         return self._inner_config.get_attr(key)
 
@@ -633,6 +639,9 @@ def get_depend_added_servers(self, name):
 
     def get_deploy_added_components(self):
         return self._deploy_config.added_components
+
+    def get_deploy_changed_components(self):
+        return self._deploy_config.changed_components
 
     def get_depend_config(self, name, server=None, with_default=True):
         if name not in self._depends:
@@ -1045,13 +1054,26 @@ def __init__(self, yaml_path, yaml_loader=yaml, inner_config=None, config_parser
         self._changed_components = []
         self._removed_components = []
         self._do_not_dump = False
+        self._mem_mode = False
+
+    def __deepcopy__(self, memo):
+        deploy_config = self.__class__(self.yaml_path, self.yaml_loader, self._inner_config, self.config_parser_manager, self.stdio)
+        for component_name in self.components:
+            deploy_config.components[component_name] = deepcopy(self.components[component_name])
+        return deploy_config
 
     def set_undumpable(self):
         self._do_not_dump = True
 
     def set_dumpable(self):
         self._do_not_dump = False
 
+    def enable_mem_mode(self):
+        self._mem_mode = True
+
+    def disable_mem_mode(self):
+        self._mem_mode = False
+
     @property
     def user(self):
         return self._user
@@ -1376,6 +1398,8 @@ def del_depend_for_component(self, component_name, depend_component_name, save=T
         return self.dump() if save else True
 
     def update_component_server_conf(self, component_name, server, key, value, save=True):
+        if self._mem_mode: 
+            return True
         if component_name not in self.components:
             return False
         cluster_config = self.components[component_name]
@@ -1386,6 +1410,8 @@ def update_component_server_conf(self, component_name, server, key, value, save=
         return self.dump() if save else True
 
     def update_component_global_conf(self, component_name, key, value, save=True):
+        if self._mem_mode:
+            return True
         if component_name not in self.components:
             return False
         cluster_config = self.components[component_name]
@@ -1438,12 +1464,20 @@ def _merge_component(self, component_name, conf):
             self.stdio.error('The style of the added configuration do not match the style of the existing cluster')
             return False
         src_conf = parser.from_cluster_config(cluster_config)
+
+        global_attr = {}
+        if ComponentInnerConfig.COMPONENT_GLOBAL_ATTRS in src_conf['inner_config']:
+            global_attr = deepcopy(src_conf['inner_config'][(ComponentInnerConfig.COMPONENT_GLOBAL_ATTRS)])
         try:
             merged_cluster_config = parser.merge_config(component_name, src_conf['config'], conf)
         except Exception as e:
             self.stdio.exception(err.EC_COMPONENT_FAILED_TO_MERGE_CONFIG.format(message=str(e)))
             return False
         self.update_component(merged_cluster_config)
+        cluster_config = self.components[component_name]
+        for k in global_attr:
+            v = global_attr.get(k)
+            cluster_config.update_component_attr(k, v, save=False)
         self._changed_components.append(component_name)
 
         # 更新depends config

_errno.py

@@ -164,6 +164,7 @@ class InitDirFailedErrorMessage(object):
 EC_OBSERVER_PRODUCTION_MODE_LIMIT = OBDErrorCodeTemplate(2009, '({server}): when production_mode is True, {key} can not be less then {limit}')
 EC_OBSERVER_SYS_MEM_TOO_LARGE = OBDErrorCodeTemplate(2010, '({server}): system_memory too large. system_memory must be less than memory_limit/memory_limit_percentage.')
 EC_OBSERVER_GET_MEMINFO_FAIL = OBDErrorCodeTemplate(2011, "{server}: fail to get memory info.\nPlease configure 'memory_limit' manually in configuration file")
+EC_OBSERVER_FAIL_TO_START_OCS = OBDErrorCodeTemplate(2012, 'Failed to start {server} obshell')
 
 WC_OBSERVER_SYS_MEM_TOO_LARGE = OBDErrorCodeTemplate(2010, '({server}): system_memory too large. system_memory should be less than {factor} * memory_limit/memory_limit_percentage.')
 

_plugin.py

@@ -292,11 +292,13 @@ def _new_func(
                     idx += 1
             kwargs = namespace_vars
             if method:
+                target_servers = None
                 if cluster_config:
                     servers = cluster_config.servers
                     target_servers = kwargs.get('target_servers')
                     if target_servers is not None:
                         cluster_config.servers = target_servers
+                        stdio.verbose('plugin %s target_servers: %s' % (self, target_servers))
                         del kwargs['target_servers']
                 try:
                     ret = method(self.context, *arg, **kwargs)
@@ -308,8 +310,9 @@ def _new_func(
                     self.context.return_false(exception=e)
                     stdio and getattr(stdio, 'exception', print)('%s RuntimeError: %s' % (self, e))
                 finally:
-                    if cluster_config:
+                    if target_servers:
                         cluster_config.servers = servers
+                        stdio.verbose('plugin %s restore servers: %s' % (self, servers))
         end_time = time.time()
         run_result[method_name]['time'] = end_time - start_time
         self.context.set_variable('run_result', run_result)
@@ -486,7 +489,10 @@ def params(self):
                     'STRING_LIST': StringList,
                     'DICT': Dict,
                     'LIST': List,
-                    'PARAM_LIST': StringOrKvList
+                    'PARAM_LIST': StringOrKvList,
+                    'DB_URL': DBUrl,
+                    'WEB_URL': WebUrl,
+                    'OB_USER': OBUser
                 }
                 self._src_data = {}
                 with open(self.def_param_yaml_path, 'rb') as f:

_stdio.py

@@ -388,7 +388,6 @@ def __init__(self,
         self._root_io = root_io
         self.track_limit = track_limit
         self._verbose_prefix = '-' * self.level
-        self.sub_ios = {}
         self.sync_obj = None
         self.input_stream = None
         self._out_obj = None
@@ -630,21 +629,15 @@ def interrupt_progressbar(self):
             return False
         return self._stop_sync_obj(IOProgressBar, 'interrupt')
 
-    def sub_io(self, pid=None, msg_lv=None):
-        if not pid:
-            pid = os.getpid()
+    def sub_io(self, msg_lv=None):
         if msg_lv is None:
             msg_lv = self.msg_lv
-        key = "%s-%s" % (pid, msg_lv)
-        if key not in self.sub_ios:
-            sub_io = self.__class__(
+        return self.__class__(
                 self.level + 1,
                 msg_lv=msg_lv,
                 track_limit=self.track_limit,
                 root_io=self._root_io if self._root_io else self
             )
-            self.sub_ios[key] = sub_io
-        return self.sub_ios[key]
 
     def print_list(self, ary, field_names=None, exp=lambda x: x if isinstance(x, (list, tuple)) else [x], show_index=False, start=0, **kwargs):
         if not ary:

_types.py

@@ -25,7 +25,7 @@
 import traceback
 
 
-__all__ = ("Moment", "Time", "Capacity", "CapacityMB", "StringList", "Dict", "List", "StringOrKvList", "Double", "Boolean", "Integer", "String", "Path", "SafeString", "PathList", "SafeStringList")
+__all__ = ("Moment", "Time", "Capacity", "CapacityMB", "StringList", "Dict", "List", "StringOrKvList", "Double", "Boolean", "Integer", "String", "Path", "SafeString", "PathList", "SafeStringList", "DBUrl", "WebUrl", "OBUser")
 
 
 class Null(object):
@@ -263,23 +263,31 @@ class String(ConfigItemType):
     def _format(self):
         self.value = self._value = str(self._origin) if self._origin else ''
 
+# this type is used to ensure the parameter is a valid oceanbase user
+class OBUser(ConfigItemType):
+    OB_USER_PATTERN = re.compile("^[a-zA-Z0-9_\.-]+(@[a-zA-Z0-9_\.-]+)?(#[a-zA-Z0-9_\.-]+)?$")
+    def _format(self):
+        if not self.OB_USER_PATTERN.match(str(self._origin)):
+            raise Exception("%s is not a valid config" % self._origin)
+        self.value = self._value = str(self._origin) if self._origin else ''
+
 # this type is used to ensure the parameter not containing special characters to inject command
 class SafeString(ConfigItemType):
-    PATH_PATTERN = re.compile("^[a-zA-Z0-9\u4e00-\u9fa5\-_:@/\.]*$")
+    SAFE_STRING_PATTERN = re.compile("^[a-zA-Z0-9\u4e00-\u9fa5\-_:@/\.]*$")
     def _format(self):
-        if not self.PATH_PATTERN.match(str(self._origin)):
+        if not self.SAFE_STRING_PATTERN.match(str(self._origin)):
             raise Exception("%s is not a valid config" % self._origin)
         self.value = self._value = str(self._origin) if self._origin else ''
 
 # this type is used to ensure the parameter not containing special characters to inject command
 class SafeStringList(ConfigItemType):
-    PATH_PATTERN = re.compile("^[a-zA-Z0-9\u4e00-\u9fa5\-_:@/\.]*$")
+    SAFE_STRING_PATTERN = re.compile("^[a-zA-Z0-9\u4e00-\u9fa5\-_:@/\.]*$")
     def _format(self):
         if self._origin:
             self._origin = str(self._origin).strip()
             self._value = self._origin.split(';')
             for v in self._value:
-                if not self.PATH_PATTERN.match(v):
+                if not self.SAFE_STRING_PATTERN.match(v):
                     raise Exception("%s is not a valid config" % v)
         else:
             self._value = []
@@ -311,3 +319,19 @@ def _format(self):
                     raise Exception("%s is not a valid path" % v)
         else:
             self._value = []
+
+# this type is used to ensure the parameter is a valid database connection url
+class DBUrl(ConfigItemType):
+    DBURL_PATTERN = re.compile("^jdbc:(mysql|oceanbase):(\/\/)([a-zA-Z0-9_.-]+)(:[0-9]{1,5})?\/([a-zA-Z0-9_\-]+)(\?[a-zA-Z0-9_&;=.-]*)?$")
+    def _format(self):
+        if not self.DBURL_PATTERN.match(str(self._origin)):
+            raise Exception("%s is not a valid config" % self._origin)
+        self.value = self._value = str(self._origin) if self._origin else ''
+
+# this type is used to ensure the parameter is a valid web url
+class WebUrl(ConfigItemType):
+    WEBURL_PATTERN = re.compile("^(https?:\/\/)?([\da-z_.-]+)(:[0-9]{1,5})?([\/\w \.-]*)*\/?(?:\?[\w&=_.-]*)?$")
+    def _format(self):
+        if not self.WEBURL_PATTERN.match(str(self._origin)):
+            raise Exception("%s is not a valid config" % self._origin)
+        self.value = self._value = str(self._origin) if self._origin else ''

const.py

@@ -40,4 +40,7 @@
 # obdeploy install path
 CONST_OBD_INSTALL_PATH = "OBD_INSTALL_PATH"
 # obdeploy forbidden variable
-FORBIDDEN_VARS = (CONST_OBD_HOME, CONST_OBD_INSTALL_PRE, CONST_OBD_INSTALL_PATH)
+FORBIDDEN_VARS = (CONST_OBD_HOME, CONST_OBD_INSTALL_PRE, CONST_OBD_INSTALL_PATH)
+
+# service docs url
+DISABLE_SWAGGER = '<DISABLE_SWAGGER>'