style(backend): flake8 fix #2633

.pre-commit-config.yaml

@@ -16,16 +16,9 @@ repos:
   - id: black
     args: [--config=./dbm-ui/pyproject.toml]
     language_version: "3.6"
-    exclude: >
-      (?x)^(
-          backend/packages/.*
-      )$
+
 - repo: https://github.com/PyCQA/flake8
-  rev: 3.8.0
+  rev: 5.0.4
   hooks:
   - id: flake8
     args: [--config=./dbm-ui/backend/.flake8]
-    exclude: >
-      (?x)^(
-          backend/packages/.*
-      )$

dbm-ui/backend/.flake8

@@ -1,5 +1,9 @@
 [flake8]
-ignore = W503,E203,W605,C901,F403,F405,F401
+# W504 and W503 conflict with each other in flake8
+# E203 is conflict with black
+# see https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md#flake8
+ignore = W503,E203
+# 排除检查的文件
 exclude =
     *migrations*,
     *.pyc,
@@ -10,10 +14,14 @@ exclude =
     */bin/*,
     */settings/*,
     scripts/*,
-    backend/packages/*,
+
+# 指定文件忽略规则
+per-file-ignores =
+    */__init__.py: F401,F403,F405
+    nginx_template.py: W605
 max-line-length=129
-max-complexity=12
+max-complexity=21
 format=pylint
 show_source = True
 statistics = True
-count = True
+count = True

dbm-ui/backend/db_meta/api/cluster/riak/scale_in.py

@@ -13,8 +13,6 @@
 
 from django.db import transaction
 
-from backend import env
-from backend.components import CCApi
 from backend.db_meta import request_validator
 from backend.db_meta.api import common
 from backend.db_meta.models import Cluster, ClusterEntry, StorageInstance

dbm-ui/backend/db_meta/api/cluster/tendbcluster/handler.py

@@ -7,7 +7,6 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import json
 from typing import List, Optional
 
 from django.db import transaction

dbm-ui/backend/db_meta/api/cluster/tendbha/switch_proxy.py

@@ -12,7 +12,7 @@
 
 from django.db import transaction
 
-from backend.db_meta.enums import ClusterEntryType, InstanceInnerRole, MachineType
+from backend.db_meta.enums import ClusterEntryType, InstanceInnerRole
 from backend.db_meta.models import Cluster, ProxyInstance, StorageInstance
 from backend.flow.utils.cc_manage import CcManage
 from backend.flow.utils.mysql.mysql_module_operate import MysqlCCTopoOperator

dbm-ui/backend/db_meta/api/dbha/apis.py

@@ -10,15 +10,16 @@
 """
 import logging
 from collections import defaultdict
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
+import validators
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import transaction
 from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 
 from backend.constants import DEFAULT_BK_CLOUD_ID, IP_PORT_DIVIDER
-from backend.db_meta import flatten, request_validator, validators
+from backend.db_meta import flatten, meta_validator, request_validator
 from backend.db_meta.enums import (
     ClusterEntryType,
     ClusterStatus,
@@ -43,7 +44,7 @@ def cities():
     return flatten.cities(BKCity.objects.all())
 
 
-def entry_detail(domains: List[str]) -> List[Dict]:
+def entry_detail(domains: List[str]) -> Dict[str, Dict[Any, list]]:
     entries = {}
     for domain in domains:
         clusterentry_set = defaultdict(list)
@@ -116,7 +117,7 @@ def instances(
         for ad in [ad for ad in addresses if len(ad.strip()) > 0]:
             if validators.ipv4(ad):
                 queries |= Q(**{"machine__ip": ad})
-            elif validators.instance(ad):
+            elif meta_validator.instance(ad):
                 queries |= Q(**{"machine__ip": ad.split(IP_PORT_DIVIDER)[0], "port": ad.split(IP_PORT_DIVIDER)[1]})
             elif validators.domain(ad):
                 queries |= Q(**{"cluster__clusterentry__entry": ad})

dbm-ui/backend/db_meta/validators/impl.py → dbm-ui/backend/db_meta/meta_validator.py

@@ -8,18 +8,13 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from validators import *  # pylint: disable=wildcard-import
+import validators
 
 
-@validator
+@validators.validator
 def instance(value):
     groups = value.split(":")
     if len(groups) != 2:
         return False
 
-    return (ipv4(groups[0]) or ipv6(groups[0])) and groups[1].isdigit()
-
-
-@validator
-def port(value):
-    return isinstance(value, int) and 0 < value < 65535
+    return (validators.ipv4(groups[0]) or validators.ipv6(groups[0])) and groups[1].isdigit()

dbm-ui/backend/db_meta/models/cluster_entry.py

@@ -17,7 +17,6 @@
 
 from backend import env
 from backend.bk_web.models import AuditedModel
-from backend.components import DnsApi
 from backend.db_meta.enums import ClusterEntryRole, ClusterEntryType
 from backend.db_meta.models import Cluster
 

dbm-ui/backend/db_meta/models/extra_process.py

@@ -11,7 +11,6 @@
 import logging
 
 from django.db import models
-from django.forms.models import model_to_dict
 from django.utils.translation import ugettext_lazy as _
 
 from backend.bk_web.models import AuditedModel

dbm-ui/backend/db_meta/models/instance.py

@@ -16,7 +16,6 @@
 from django.db.models import Q, QuerySet
 from django.utils.translation import ugettext_lazy as _
 
-from backend import constants
 from backend.bk_web.models import AuditedModel
 from backend.db_meta.enums import (
     AccessLayer,

dbm-ui/backend/db_meta/models/machine.py

@@ -9,7 +9,6 @@
 specific language governing permissions and limitations under the License.
 """
 import copy
-
 from dataclasses import asdict
 
 from django.db import models

dbm-ui/backend/db_meta/request_validator/atom.py

@@ -11,10 +11,9 @@
 import logging
 from typing import List
 
+import validators
 from rest_framework import serializers
 
-from backend.db_meta import validators
-
 logger = logging.getLogger("root")
 
 
@@ -31,33 +30,6 @@ def to_internal_value(self, data):
         return self.field.to_internal_value(data)
 
 
-# def validated_integer(data, min_value=None, max_value=None, allow_null=False) -> int:
-#     field = serializers.IntegerField(min_value=min_value, max_value=max_value, allow_null=allow_null)
-#     slz = _AtomSerializer(data=data, field=field, allow_null=allow_null)
-#     slz.is_valid(raise_exception=True)
-#     return slz.validated_data
-
-
-# def validated_str(data, min_length=None, max_length=None, allow_blank=False, trim_whitespace=True) -> str:
-#     field = serializers.CharField(
-#         min_length=min_length, max_length=max_length, allow_blank=allow_blank, trim_whitespace=trim_whitespace
-#     )
-#     slz = _AtomSerializer(data=data, field=field)
-#     slz.is_valid(raise_exception=True)
-#     return slz.validated_data
-
-
-# def validated_integer_list(data, min_value=None, max_value=None, allow_empty=True, allow_null=True) -> List[int]:
-#     slz = serializers.ListSerializer(
-#         child=serializers.IntegerField(min_value=min_value, max_value=max_value),
-#         data=data,
-#         allow_empty=allow_empty,
-#         allow_null=allow_null,
-#     )
-#     slz.is_valid(raise_exception=True)
-#     return slz.validated_data
-
-
 def validated_str_list(
     data, min_length=None, max_length=None, allow_blank=False, trim_whitespace=True, allow_empty=True, allow_null=True
 ) -> List[str]:
@@ -110,10 +82,3 @@ def validated_domain_list(
     )
     slz.is_valid(raise_exception=True)
     return slz.validated_data
-
-
-# def validated_ip(data) -> str:
-#     field = serializers.IPAddressField()
-#     slz = _AtomSerializer(data=data, field=field)
-#     slz.is_valid(raise_exception=True)
-#     return slz.validated_data

dbm-ui/backend/db_meta/request_validator/common.py

@@ -10,10 +10,9 @@
 """
 import logging
 
+import validators
 from rest_framework import serializers
 
-from backend.db_meta import validators
-
 from .serializers import CommonSerializer
 
 logger = logging.getLogger("root")

dbm-ui/backend/db_meta/request_validator/dbha.py

@@ -8,15 +8,16 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+import validators
 from rest_framework import serializers
 
-from backend.db_meta import validators
+from backend.db_meta import meta_validator
 
 
 class DBHAInstanceRequestSerializer(serializers.Serializer):
     class AddressField(serializers.CharField):
         def to_internal_value(self, data):
-            if not validators.ipv4(data) and not validators.instance(data):  # and not validators.domain(data):
+            if not validators.ipv4(data) and not meta_validator.instance(data):
                 raise serializers.ValidationError('"{}" is not a valid address'.format(data))
 
     logical_city_ids = serializers.ListField(

dbm-ui/backend/db_monitor/management/commands/export_template.py

@@ -8,7 +8,6 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import base64
 import json
 import logging
 import os

dbm-ui/backend/db_monitor/models/alarm.py

@@ -23,7 +23,7 @@
 from backend.components import BKMonitorV3Api
 from backend.configuration.constants import PLAT_BIZ_ID, DBType, SystemSettingsEnum
 from backend.configuration.models import SystemSettings
-from backend.db_meta.models import AppMonitorTopo, DBModule
+from backend.db_meta.models import DBModule
 from backend.db_monitor.constants import (
     APP_PRIORITY,
     BK_MONITOR_SAVE_DISPATCH_GROUP_TEMPLATE,

dbm-ui/backend/db_monitor/tasks.py

@@ -13,8 +13,7 @@
 
 from celery import shared_task
 
-from .. import env
-from ..configuration.constants import PLAT_BIZ_ID
+from backend.configuration.constants import PLAT_BIZ_ID
 
 logger = logging.getLogger("celery")
 

dbm-ui/backend/db_package/views.py

@@ -10,8 +10,6 @@
 """
 import os
 import re
-from collections import defaultdict
-from typing import Dict, List
 
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.db.transaction import atomic

dbm-ui/backend/db_periodic_task/local_tasks/dbmon_heartbeat/heartbeat_report.py

@@ -37,7 +37,6 @@ def check_dbmon_heart_beat():
 
 
 def query_by_cluster_dimension(cluster_domain, cap_key="heartbeat", cluster_type="dbmon"):
-
     logger.info("+===+++++=== cluster is: {} +++++===++++ ".format(cluster_domain))
     query_template = QUERY_TEMPLATE.get(cluster_type)
     if not query_template:
@@ -74,8 +73,29 @@ def query_by_cluster_dimension(cluster_domain, cap_key="heartbeat", cluster_type
     return dbmon_heartbeat_data
 
 
-def _check_dbmon_heart_beat():
+def get_report_subtype_for_storage(cluster_type):
+    if cluster_type == ClusterType.TwemproxyTendisSSDInstance.value:
+        heart_beat_subtype = DbmonHeartbeatReportSubType.REDIS_SSD.value
+    elif cluster_type == ClusterType.TendisPredixyTendisplusCluster.value:
+        heart_beat_subtype = DbmonHeartbeatReportSubType.TENDISPLUS.value
+    elif cluster_type == ClusterType.TendisTwemproxyRedisInstance.value:
+        heart_beat_subtype = DbmonHeartbeatReportSubType.REDIS_CACHE.value
+    else:
+        raise NotImplementedError("Dbmon Not supported tendis type:{}".format(cluster_type))
+    return heart_beat_subtype
+
+
+def get_report_subtype_for_proxy(cluster_type):
+    if cluster_type in [ClusterType.TwemproxyTendisSSDInstance.value, ClusterType.TendisTwemproxyRedisInstance.value]:
+        heart_beat_subtype = DbmonHeartbeatReportSubType.TWEMPROXY.value
+    elif cluster_type == ClusterType.TendisPredixyTendisplusCluster.value:
+        heart_beat_subtype = DbmonHeartbeatReportSubType.PREDIXY.value
+    else:
+        raise NotImplementedError("Dbmon Not supported tendis type:{}".format(cluster_type))
+    return heart_beat_subtype
 
+
+def _check_dbmon_heart_beat():
     """
     获取dbmon心跳信息
     """
@@ -122,14 +142,7 @@ def _check_dbmon_heart_beat():
         for ip in missing_heartbeat_ips:
             # 如果是后端存储节点，再区分cache ,ssd ,tendisplus
             if ip in cluster_info["redis_master_ips_set"] or ip in cluster_info["redis_slave_ips_set"]:
-                if c.cluster_type == ClusterType.TwemproxyTendisSSDInstance.value:
-                    cluster_type = DbmonHeartbeatReportSubType.REDIS_SSD.value
-                elif c.cluster_type == ClusterType.TendisPredixyTendisplusCluster.value:
-                    cluster_type = DbmonHeartbeatReportSubType.TENDISPLUS.value
-                elif c.cluster_type == ClusterType.TendisTwemproxyRedisInstance.value:
-                    cluster_type = DbmonHeartbeatReportSubType.REDIS_CACHE.value
-                else:
-                    raise NotImplementedError("Dbmon Not supported tendis type:{}".format(c.cluster_type))
+                heart_beat_subtype = get_report_subtype_for_storage(c.cluster_type)
                 # 获取端口范围：30000-30010
                 port_ranges = []
                 if ip in cluster_info["redis_master_ips_set"]:
@@ -163,28 +176,19 @@ def _check_dbmon_heart_beat():
             elif ip in cluster_info["twemproxy_ips_set"]:
                 twemproxy_ports = cluster_info.get("twemproxy_ports", [])
                 instance = "{} {}".format(ip, twemproxy_ports[0])
-                if (
-                    c.cluster_type == ClusterType.TwemproxyTendisSSDInstance.value
-                    or c.cluster_type == ClusterType.TendisTwemproxyRedisInstance.value
-                ):
-                    cluster_type = DbmonHeartbeatReportSubType.TWEMPROXY.value
-                elif c.cluster_type == ClusterType.TendisPredixyTendisplusCluster.value:
-                    cluster_type = DbmonHeartbeatReportSubType.PREDIXY.value
-                else:
-                    raise NotImplementedError("Dbmon Not supported tendis type: %s" % c.cluster_type)
+                heart_beat_subtype = get_report_subtype_for_proxy(c.cluster_type)
             else:
                 raise NotImplementedError(" %s is not identified in Dbmon" % ip)
             msg = _("实例 {} dbmon 心跳超时").format(instance)
             try:
-                # create_at = models.DateTimeField(_("创建时间"), auto_now_add=True)
-                # 心跳超时的时间点就用这条记录的创建时间代替了，这里对时间要去不严格
+                # 心跳超时的时间点就用这条记录的创建时间代替了，这里对时间要求不严格
                 DbmonHeartbeatReport.objects.create(
                     creator=c.creator,
                     bk_biz_id=c.bk_biz_id,
                     bk_cloud_id=c.bk_cloud_id,
                     status=False,
                     msg=msg,
-                    cluster_type=cluster_type,
+                    cluster_type=heart_beat_subtype,
                     cluster=c.immute_domain,
                     instance=instance,
                     app=app,

dbm-ui/backend/db_periodic_task/local_tasks/ticket.py

@@ -9,7 +9,6 @@
 specific language governing permissions and limitations under the License.
 """
 from celery.schedules import crontab
-from celery.task import periodic_task
 
 from backend.db_periodic_task.local_tasks import register_periodic_task
 from backend.ticket.tasks.ticket_tasks import TicketTask

dbm-ui/backend/db_proxy/views/db_remote_service/views.py

@@ -12,7 +12,6 @@
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import status
 from rest_framework.decorators import action
-from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 
 from backend.bk_web.swagger import common_swagger_auto_schema