fix(mysql): 元数据导入写入机器主键冲突 #3943

TencentBlueKing · Apr 12, 2024 · a920072 · a920072
1 parent 2b02006
commit a920072
Show file tree

Hide file tree

Showing 14 changed files with 100 additions and 100 deletions.
diff --git a/.github/workflows/python_code_check.yml b/.github/workflows/python_code_check.yml
@@ -21,14 +21,14 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.6
+    - name: Set up Python 3.10
       uses: actions/setup-python@v5
       with:
-        python-version: 3.6
+        python-version: 3.10.14
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8==5.0.4 black=="20.8b1"
+        pip install flake8==5.0.4 black=="22.3.0"
     - name: Lint with flake8
       run: |
         FLAKE8_RESULT=$(flake8 dbm-ui/backend --config=dbm-ui/backend/.flake8 || true)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,16 +2,16 @@ files: ^dbm-ui/backend/
 fail_fast: true
 repos:
 - repo: https://github.com/timothycrosley/isort
-  rev: 5.7.0
+  rev: 5.13.2
   hooks:
   - id: isort
     additional_dependencies: [toml]
 - repo: https://github.com/psf/black
-  rev: 20.8b1
+  rev: 22.3.0
   hooks:
   - id: black
     args: [--config=./dbm-ui/pyproject.toml]
-    language_version: "3.6"
+    language_version: python3
 - repo: https://github.com/PyCQA/flake8
   rev: 5.0.4
   hooks:

diff --git a/dbm-services/go.work.sum b/dbm-services/go.work.sum
@@ -914,6 +914,7 @@ github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58 h1:nlG4Wa5+minh3S9LVFtNoY+GVRiudA2e3EVfcCi3RCA=
+github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 h1:ZFfeKAhIQiiOrQaI3/znw0gOmYpO28Tcu1YaqMa/jtQ=
 github.com/sagikazarmark/crypt v0.10.0 h1:96E1qrToLBU6fGzo+PRRz7KGOc9FkYFiPnR3/zf8Smg=
@@ -1051,8 +1052,6 @@ golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBch
 golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
 golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

diff --git a/dbm-ui/backend/constants.py b/dbm-ui/backend/constants.py
@@ -16,7 +16,7 @@
 DEFAULT_BK_CLOUD_ID = 0
 
 # 一些常量值
-INT_MAX = 2 ** 31 - 1
+INT_MAX = 2**31 - 1
 
 # IP 端口分隔符
 IP_PORT_DIVIDER = ":"

diff --git a/dbm-ui/backend/db_monitor/utils.py b/dbm-ui/backend/db_monitor/utils.py
@@ -84,9 +84,9 @@ def render_promql_sql(prom_sql, wheres):
             value = f'"{value}"'
 
         # Check if label already exists in the query
-        if re.search(fr'{label}="[^"]*"|\b{label}=~"[^"]*"', prom_sql):
+        if re.search(rf'{label}="[^"]*"|\b{label}=~"[^"]*"', prom_sql):
             # If label exists, replace its value
-            prom_sql = re.sub(fr'{label}="[^"]*"|\b{label}=~"[^"]*"', fr"{label}={value}", prom_sql)
+            prom_sql = re.sub(rf'{label}="[^"]*"|\b{label}=~"[^"]*"', rf"{label}={value}", prom_sql)
         else:
             # If label does not exist, add before the closing '}'
             # Use double }} to escape a single } in the f-string

diff --git a/dbm-ui/backend/db_services/dbresource/serializers.py b/dbm-ui/backend/db_services/dbresource/serializers.py
@@ -275,14 +275,10 @@ def validate_data_points(self, attrs):
                     _("【{}】后端磁盘挂载点必须包含/data，可选/data1").format(attrs["spec_cluster_type"])
                 )
         # TendisPlus/TendisSSD 磁盘必须包含/data，/data1可选
-        if (
-            attrs["spec_cluster_type"]
-            in [
-                ClusterType.TwemproxyTendisSSDInstance,
-                ClusterType.TendisPredixyTendisplusCluster,
-            ]
-            and attrs["spec_machine_type"] in [MachineType.TENDISPLUS, MachineType.TENDISSSD]
-        ):
+        if attrs["spec_cluster_type"] in [
+            ClusterType.TwemproxyTendisSSDInstance,
+            ClusterType.TendisPredixyTendisplusCluster,
+        ] and attrs["spec_machine_type"] in [MachineType.TENDISPLUS, MachineType.TENDISSSD]:
             if "/data" not in set(mount_points):
                 raise serializers.ValidationError(_("【{}】后端磁盘挂载点必须包含/data").format(attrs["spec_cluster_type"]))
 

diff --git a/dbm-ui/backend/db_services/mysql/constants.py b/dbm-ui/backend/db_services/mysql/constants.py
@@ -10,7 +10,7 @@
 """
 
 # 端口最大最小值
-SERVER_PORT_LIMIT_MAX = 2 ** 16
+SERVER_PORT_LIMIT_MAX = 2**16
 SERVER_PORT_LIMIT_MIN = 0
 
 # 默认起始端口

diff --git a/dbm-ui/backend/db_services/plugin/nameservice/clb.py b/dbm-ui/backend/db_services/plugin/nameservice/clb.py
@@ -24,7 +24,7 @@
 
 @transaction.atomic
 def tendis_add_clb_domain(immute_domain: str, bk_cloud_id: int, created_by: str):
-    """ 增加CLB 域名 """
+    """增加CLB 域名"""
     cluster = Cluster.objects.get(bk_cloud_id=bk_cloud_id, immute_domain=immute_domain)
     clb = cluster.clusterentry_set.filter(cluster_entry_type=ClusterEntryType.CLB.value).first()
     ClusterEntry.objects.create(
@@ -38,7 +38,7 @@ def tendis_add_clb_domain(immute_domain: str, bk_cloud_id: int, created_by: str)
 
 @transaction.atomic
 def tendis_bind_clb_domain(immute_domain: str, bk_cloud_id: int, created_by: str):
-    """ 主域名直接指向CLB """
+    """主域名直接指向CLB"""
     cluster = Cluster.objects.get(bk_cloud_id=bk_cloud_id, immute_domain=immute_domain)
     immute_entry = cluster.clusterentry_set.filter(
         cluster_entry_type=ClusterEntryType.DNS.value, entry=cluster.immute_domain
@@ -51,7 +51,7 @@ def tendis_bind_clb_domain(immute_domain: str, bk_cloud_id: int, created_by: str
 
 @transaction.atomic
 def tendis_unbind_clb_domain(immute_domain: str, bk_cloud_id: int, created_by: str):
-    """ 主域名解绑CLB """
+    """主域名解绑CLB"""
     cluster = Cluster.objects.get(bk_cloud_id=bk_cloud_id, immute_domain=immute_domain)
     immute_entry = cluster.clusterentry_set.filter(
         cluster_entry_type=ClusterEntryType.DNS.value, entry=cluster.immute_domain

diff --git a/dbm-ui/backend/db_services/redis/redis_dts/util.py b/dbm-ui/backend/db_services/redis/redis_dts/util.py
@@ -742,16 +742,12 @@ def is_in_full_transfer(row: TbTendisDtsTask) -> bool:
     判断是否全量传输中
     """
     if row.src_dbtype == ClusterType.TendisTendisSSDInstance:
-        if (
-            row.task_type
-            in [
-                DtsTaskType.TENDISSSD_BACKUP,
-                DtsTaskType.TENDISSSD_BACKUPFILE_FETCH,
-                DtsTaskType.TENDISSSD_TREDISDUMP,
-                DtsTaskType.TENDISSSD_CMDSIMPOTER,
-            ]
-            and row.status in [0, 1]
-        ):
+        if row.task_type in [
+            DtsTaskType.TENDISSSD_BACKUP,
+            DtsTaskType.TENDISSSD_BACKUPFILE_FETCH,
+            DtsTaskType.TENDISSSD_TREDISDUMP,
+            DtsTaskType.TENDISSSD_CMDSIMPOTER,
+        ] and row.status in [0, 1]:
             return True
     if row.src_dbtype == ClusterType.TendisRedisInstance:
         if row.task_type == DtsTaskType.MAKE_CACHE_SYNC and "rdb" in row.message and row.status in [0, 1]:

diff --git a/dbm-ui/backend/db_services/redis/toolbox/handlers.py b/dbm-ui/backend/db_services/redis/toolbox/handlers.py
@@ -133,7 +133,7 @@ def query_master_slave_pairs(self, cluster_id: int) -> list:
         return result
 
     def query_cluster_list(self):
-        """"TODO: 切换为集群列表接口，deprecated"""
+        """ "TODO: 切换为集群列表接口，deprecated"""
 
         clusters = Cluster.objects.filter(
             bk_biz_id=self.bk_biz_id,

diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/mysql_ha_import_metadata.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/mysql_ha_import_metadata.py
@@ -162,29 +162,32 @@ def _create_machine(
             spec_obj = self.storage_spec
 
         machine, _ = Machine.objects.get_or_create(
-            ip=ip,
-            bk_biz_id=self.bk_biz_id,
-            db_module_id=self.db_module_id,
-            access_layer=access_layer.value,
-            machine_type=machine_type.value,
-            cluster_type=ClusterType.TenDBHA.value,
-            bk_city=BKCity.objects.get(pk=bk_city_id),
             bk_host_id=cc_info["bk_host_id"],
-            bk_os_name=cc_info["bk_os_name"],
-            bk_idc_area=cc_info["bk_idc_area"],
-            bk_idc_area_id=cc_info["bk_idc_area_id"],
-            bk_sub_zone=cc_info["sub_zone"],
-            bk_sub_zone_id=cc_info["sub_zone_id"],
-            bk_rack=cc_info["rack"],
-            bk_rack_id=cc_info["rack_id"],
-            bk_svr_device_cls_name=cc_info["bk_svr_device_cls_name"],
-            bk_idc_name=cc_info["idc_name"],
-            bk_idc_id=cc_info["idc_id"],
-            bk_cloud_id=0,  # addition_info["bk_cloud_id"],
-            net_device_id=cc_info["net_device_id"],
-            spec_id=spec_obj.spec_id,
-            spec_config=spec_obj.get_spec_info(),
+            defaults=dict(
+                ip=ip,
+                bk_biz_id=self.bk_biz_id,
+                db_module_id=self.db_module_id,
+                access_layer=access_layer.value,
+                machine_type=machine_type.value,
+                cluster_type=ClusterType.TenDBHA.value,
+                bk_city=BKCity.objects.get(pk=bk_city_id),
+                bk_os_name=cc_info["bk_os_name"],
+                bk_idc_area=cc_info["bk_idc_area"],
+                bk_idc_area_id=cc_info["bk_idc_area_id"],
+                bk_sub_zone=cc_info["sub_zone"],
+                bk_sub_zone_id=cc_info["sub_zone_id"],
+                bk_rack=cc_info["rack"],
+                bk_rack_id=cc_info["rack_id"],
+                bk_svr_device_cls_name=cc_info["bk_svr_device_cls_name"],
+                bk_idc_name=cc_info["idc_name"],
+                bk_idc_id=cc_info["idc_id"],
+                bk_cloud_id=0,
+                net_device_id=cc_info["net_device_id"],
+                spec_id=spec_obj.spec_id,
+                spec_config=spec_obj.get_spec_info(),
+            ),
         )
+
         return machine
 
     def __create_instance(

diff --git a/dbm-ui/backend/flow/plugins/components/collections/spider/import_metadata.py b/dbm-ui/backend/flow/plugins/components/collections/spider/import_metadata.py
@@ -300,29 +300,32 @@ def _create_machine(
             spec_obj = self.remote_spec
 
         machine, _ = Machine.objects.get_or_create(
-            ip=ip,
-            bk_biz_id=self.bk_biz_id,
-            db_module_id=self.db_module_id,
-            access_layer=access_layer.value,
-            machine_type=machine_type.value,
-            cluster_type=ClusterType.TenDBCluster.value,
-            bk_city=BKCity.objects.get(pk=bk_city_id),
             bk_host_id=cc_info["bk_host_id"],
-            bk_os_name=cc_info["bk_os_name"],
-            bk_idc_area=cc_info["bk_idc_area"],
-            bk_idc_area_id=cc_info["bk_idc_area_id"],
-            bk_sub_zone=cc_info["sub_zone"],
-            bk_sub_zone_id=cc_info["sub_zone_id"],
-            bk_rack=cc_info["rack"],
-            bk_rack_id=cc_info["rack_id"],
-            bk_svr_device_cls_name=cc_info["bk_svr_device_cls_name"],
-            bk_idc_name=cc_info["idc_name"],
-            bk_idc_id=cc_info["idc_id"],
-            bk_cloud_id=0,  # addition_info["bk_cloud_id"],
-            net_device_id=cc_info["net_device_id"],
-            spec_id=spec_obj.spec_id,
-            spec_config=spec_obj.get_spec_info(),
+            defaults=dict(
+                ip=ip,
+                bk_biz_id=self.bk_biz_id,
+                db_module_id=self.db_module_id,
+                access_layer=access_layer.value,
+                machine_type=machine_type.value,
+                cluster_type=ClusterType.TenDBCluster.value,
+                bk_city=BKCity.objects.get(pk=bk_city_id),
+                bk_os_name=cc_info["bk_os_name"],
+                bk_idc_area=cc_info["bk_idc_area"],
+                bk_idc_area_id=cc_info["bk_idc_area_id"],
+                bk_sub_zone=cc_info["sub_zone"],
+                bk_sub_zone_id=cc_info["sub_zone_id"],
+                bk_rack=cc_info["rack"],
+                bk_rack_id=cc_info["rack_id"],
+                bk_svr_device_cls_name=cc_info["bk_svr_device_cls_name"],
+                bk_idc_name=cc_info["idc_name"],
+                bk_idc_id=cc_info["idc_id"],
+                bk_cloud_id=0,
+                net_device_id=cc_info["net_device_id"],
+                spec_id=spec_obj.spec_id,
+                spec_config=spec_obj.get_spec_info(),
+            ),
         )
+
         return machine
 
 

diff --git a/dbm-ui/backend/flow/plugins/components/collections/tendbsingle/metadata_import.py b/dbm-ui/backend/flow/plugins/components/collections/tendbsingle/metadata_import.py
@@ -106,29 +106,32 @@ def _create_machine(self, ip: str, addition_info: Dict) -> Machine:
         spec_obj = self.storage_spec
 
         machine, _ = Machine.objects.get_or_create(
-            ip=ip,
-            bk_biz_id=self.bk_biz_id,
-            db_module_id=self.db_module_id,
-            access_layer=AccessLayer.STORAGE.value,
-            machine_type=MachineType.SINGLE.value,
-            cluster_type=ClusterType.TenDBSingle.value,
-            bk_city=BKCity.objects.get(pk=bk_city_id),
             bk_host_id=cc_info["bk_host_id"],
-            bk_os_name=cc_info["bk_os_name"],
-            bk_idc_area=cc_info["bk_idc_area"],
-            bk_idc_area_id=cc_info["bk_idc_area_id"],
-            bk_sub_zone=cc_info["sub_zone"],
-            bk_sub_zone_id=cc_info["sub_zone_id"],
-            bk_rack=cc_info["rack"],
-            bk_rack_id=cc_info["rack_id"],
-            bk_svr_device_cls_name=cc_info["bk_svr_device_cls_name"],
-            bk_idc_name=cc_info["idc_name"],
-            bk_idc_id=cc_info["idc_id"],
-            bk_cloud_id=0,
-            net_device_id=cc_info["net_device_id"],
-            spec_id=spec_obj.spec_id,
-            spec_config=spec_obj.get_spec_info(),
+            defaults=dict(
+                ip=ip,
+                bk_biz_id=self.bk_biz_id,
+                db_module_id=self.db_module_id,
+                access_layer=AccessLayer.STORAGE.value,
+                machine_type=MachineType.SINGLE.value,
+                cluster_type=ClusterType.TenDBSingle.value,
+                bk_city=BKCity.objects.get(pk=bk_city_id),
+                bk_os_name=cc_info["bk_os_name"],
+                bk_idc_area=cc_info["bk_idc_area"],
+                bk_idc_area_id=cc_info["bk_idc_area_id"],
+                bk_sub_zone=cc_info["sub_zone"],
+                bk_sub_zone_id=cc_info["sub_zone_id"],
+                bk_rack=cc_info["rack"],
+                bk_rack_id=cc_info["rack_id"],
+                bk_svr_device_cls_name=cc_info["bk_svr_device_cls_name"],
+                bk_idc_name=cc_info["idc_name"],
+                bk_idc_id=cc_info["idc_id"],
+                bk_cloud_id=0,
+                net_device_id=cc_info["net_device_id"],
+                spec_id=spec_obj.spec_id,
+                spec_config=spec_obj.get_spec_info(),
+            ),
         )
+
         return machine
 
     def __create_storage_instance(

diff --git a/dbm-ui/backend/flow/utils/redis/redis_db_meta.py b/dbm-ui/backend/flow/utils/redis/redis_db_meta.py
@@ -859,7 +859,7 @@ def add_polairs_domain(self):
         polaris_entry.save()
 
     def tendis_add_clb_domain_4_scene(self):
-        """ 增加CLB 域名 """
+        """增加CLB 域名"""
         cluster = Cluster.objects.get(
             bk_cloud_id=self.cluster["bk_cloud_id"], immute_domain=self.cluster["immute_domain"]
         )
@@ -875,7 +875,7 @@ def tendis_add_clb_domain_4_scene(self):
         cluster_entry.save()
 
     def tendis_bind_clb_domain_4_scene(self):
-        """ 主域名直接指向CLB """
+        """主域名直接指向CLB"""
         cluster = Cluster.objects.get(
             bk_cloud_id=self.cluster["bk_cloud_id"], immute_domain=self.cluster["immute_domain"]
         )
@@ -891,7 +891,7 @@ def tendis_bind_clb_domain_4_scene(self):
 
     # 主域名解绑CLB
     def tendis_unBind_clb_domain_4_scene(self):
-        """ 主域名解绑CLB """
+        """主域名解绑CLB"""
         cluster = Cluster.objects.get(
             bk_cloud_id=self.cluster["bk_cloud_id"], immute_domain=self.cluster["immute_domain"]
         )