From a387ac50ca30a6011b62d4a00f84f5db9fb0e3a4 Mon Sep 17 00:00:00 2001 From: durant <826035498@qq.com> Date: Wed, 13 Mar 2024 09:28:57 +0800 Subject: [PATCH] =?UTF-8?q?fix(backend):=20cmdb=20dbm=5Fmeta=20=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E6=94=AF=E6=8C=81=E5=8E=8B=E7=BC=A9=EF=BC=8C=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E5=A4=9A=E5=AE=9E=E4=BE=8B=E7=9B=91=E6=8E=A7=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20#3529?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../configuration/handlers/password.py | 10 +- dbm-ui/backend/core/encrypt/aes.py | 3 +- .../db_meta/api/proxy_instance/apis.py | 38 ----- dbm-ui/backend/db_meta/models/machine.py | 145 ++++++++---------- dbm-ui/backend/db_meta/utils.py | 6 +- .../db_periodic_task/local_tasks/db_proxy.py | 4 +- .../db_proxy/views/job_callback/views.py | 4 +- dbm-ui/backend/db_proxy/views/jobapi/views.py | 4 +- .../mysql/fixpoint_rollback/handlers.py | 9 +- .../collections/cloud/exec_service_script.py | 4 +- .../collections/cloud/push_config_file.py | 4 +- .../collections/common/check_resolv_conf.py | 4 +- .../collections/common/dns_server.py | 4 +- .../collections/es/exec_es_actuator_script.py | 8 +- .../collections/es/rewrite_es_config.py | 6 +- .../collections/hdfs/exec_actuator_script.py | 8 +- .../collections/hdfs/rewrite_hdfs_config.py | 4 +- .../influxdb/exec_actuator_script.py | 8 +- .../collections/influxdb/influxdb_config.py | 6 +- .../collections/kafka/exec_actuator_script.py | 8 +- .../collections/kafka/kafka_config.py | 6 +- .../collections/mysql/exec_actuator_script.py | 11 +- .../collections/mysql/mysql_os_init.py | 7 +- .../pulsar/exec_actuator_script.py | 8 +- .../pulsar/rewrite_pulsar_config.py | 8 +- .../collections/redis/exec_actuator_script.py | 8 +- .../exec_data_structure_actuator_script.py | 8 +- .../collections/redis/exec_shell_script.py | 6 +- .../redis/psuh_data_structure_json_script.py | 6 +- .../components/collections/redis/redis_dts.py | 10 +- .../redis/redis_old_backup_records.py | 6 +- .../collections/riak/exec_actuator_script.py | 10 +- .../sqlserver/exec_actuator_script.py | 11 +- .../flow/utils/base/payload_handler.py | 7 +- .../flow/utils/cloud/cloud_act_payload.py | 4 +- dbm-ui/backend/tests/conftest.py | 58 ++++++- .../tests/db_meta/api/machine/test_apis.py | 15 +- .../db_meta/api/proxy_instance/test_apis.py | 13 -- .../backend/tests/db_meta/models/__init__.py | 10 ++ .../tests/db_meta/models/test_machine.py | 84 ++++++++++ dbm-ui/backend/utils/string.py | 23 ++- dbm-ui/bin/pytest.sh | 2 +- dbm-ui/pytest.ini | 2 +- dbm-ui/scripts/ci/code_quality.sh | 2 +- 44 files changed, 341 insertions(+), 271 deletions(-) create mode 100644 dbm-ui/backend/tests/db_meta/models/__init__.py create mode 100644 dbm-ui/backend/tests/db_meta/models/test_machine.py diff --git a/dbm-ui/backend/configuration/handlers/password.py b/dbm-ui/backend/configuration/handlers/password.py index 101c07acaf..996cb7f444 100644 --- a/dbm-ui/backend/configuration/handlers/password.py +++ b/dbm-ui/backend/configuration/handlers/password.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 from collections import defaultdict from typing import Any, Dict, List @@ -24,6 +23,7 @@ from backend.db_periodic_task.models import DBPeriodicTask from backend.db_services.ipchooser.query.resource import ResourceQueryHelper from backend.flow.consts import MySQLPasswordRole +from backend.utils.string import base64_decode, base64_encode class DBPasswordHandler(object): @@ -40,7 +40,7 @@ def verify_password_strength(cls, password: str, echo: bool = False): name=AsymmetricCipherConfigType.PASSWORD.value, content=password, salted=False ) # 密码需要用base64加密后传输 - b64_plain_password = base64.b64encode(plain_password.encode("utf-8")).decode("utf-8") + b64_plain_password = base64_encode(plain_password) check_result = MySQLPrivManagerApi.check_password( {"password": b64_plain_password, "security_rule_name": DBM_PASSWORD_SECURITY_NAME} ) @@ -84,7 +84,7 @@ def query_mysql_admin_password( mysql_admin_password_data["results"] = mysql_admin_password_data.pop("items") cloud_info = ResourceQueryHelper.search_cc_cloud(get_cache=True) for data in mysql_admin_password_data["results"]: - data["password"] = base64.b64decode(data["password"]).decode("utf-8") + data["password"] = base64_decode(data["password"]) data["bk_cloud_name"] = cloud_info[str(data["bk_cloud_id"])]["bk_cloud_name"] return mysql_admin_password_data @@ -120,7 +120,7 @@ def modify_mysql_admin_password(cls, operator: str, password: str, lock_hour: in modify_password_params = { "username": DBM_MYSQL_ADMIN_USER, "component": DBType.MySQL.value, - "password": base64.b64encode(password.encode("utf-8")).decode("utf-8"), + "password": base64_encode(password), "lock_hour": lock_hour, "operator": operator, "clusters": cluster_infos, @@ -176,4 +176,4 @@ def query_proxy_password(cls): } data = MySQLPrivManagerApi.get_password(params)["items"][0] # 注意要用base64解密 - return base64.b64decode(data["password"]).decode("utf8") + return base64_decode(data["password"]) diff --git a/dbm-ui/backend/core/encrypt/aes.py b/dbm-ui/backend/core/encrypt/aes.py index 3191829f0d..486ade1d29 100644 --- a/dbm-ui/backend/core/encrypt/aes.py +++ b/dbm-ui/backend/core/encrypt/aes.py @@ -13,6 +13,7 @@ from Crypto.Cipher import AES from backend.core.encrypt.constants import AES_BLOCK_SIZE, AES_PADDING +from backend.utils.string import base64_encode def pad_it(data): @@ -38,7 +39,7 @@ def encrypt(data: str, aes_key: str) -> str: aes_key = aes_key.encode("utf-8") cipher = AES.new(aes_key, AES.MODE_CBC, aes_key) data = cipher.encrypt(pad_it(data).encode("utf-8")) - return base64.b64encode(data).decode("utf-8") + return base64_encode(data) def decrypt(data: str, aes_key: str) -> str: diff --git a/dbm-ui/backend/db_meta/api/proxy_instance/apis.py b/dbm-ui/backend/db_meta/api/proxy_instance/apis.py index baff762986..b0f01dd49b 100644 --- a/dbm-ui/backend/db_meta/api/proxy_instance/apis.py +++ b/dbm-ui/backend/db_meta/api/proxy_instance/apis.py @@ -77,41 +77,3 @@ def update(proxies): proxy_obj.status = new_status proxy_obj.save() - - -# @transaction.atomic -# def decommission(instances: List[Dict]): -# """ -# TODO：没使用到？待删除 -# 1. 仅支持 下架实例不在任何一个集群 -# 必要条件： -# 1. 不属于任何一个集群 ;属于集群的实例，需要走集群内下架接口 -# -# 场景： -# 1. 上架了，但未添加到集群 -# 2. 从集群内清理掉了 ；调用了 delete_proxies() -# """ -# logger.info("user request decmmission instances {}".format(instances)) -# proxy_objs = common.filter_out_instance_obj(instances, ProxyInstance.objects.all()) -# -# _t = common.in_another_cluster(proxy_objs) -# if _t: -# raise Exception(_("proxy {} 在集群里边").format(_t)) -# -# _t = common.not_exists(instances, ProxyInstance.objects.all()) -# if _t: -# raise Exception(_("proxy {} 不存在").format(_t)) -# -# for proxy_obj in proxy_objs: -# logger.info("remove proxy {} ".format(proxy_obj)) -# CcManage(proxy_obj.bk_biz_id).delete_service_instance(bk_instance_ids=[proxy_obj.bk_instance_id]) -# -# # 需要检查， 是否该机器上所有实例都已经清理干净， -# if len(ProxyInstance.objects.filter(machine__ip=proxy_obj.machine.ip).all()) > 0: -# logger.info("ignore storage machine {} , another instance existed.".format(proxy_obj.machine)) -# else: -# logger.info("proxy machine {}".format(proxy_obj.machine)) -# CcManage( -# proxy_obj.bk_biz_id, -# ).recycle_host([proxy_obj.machine.bk_host_id]) -# proxy_obj.machine.delete() diff --git a/dbm-ui/backend/db_meta/models/machine.py b/dbm-ui/backend/db_meta/models/machine.py index 78818f7b3c..a76eebc4d6 100644 --- a/dbm-ui/backend/db_meta/models/machine.py +++ b/dbm-ui/backend/db_meta/models/machine.py @@ -8,7 +8,9 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import copy +import gzip +import io +import json from dataclasses import asdict from django.db import models @@ -21,6 +23,7 @@ from backend.db_meta.enums import AccessLayer, ClusterType, MachineType from backend.db_meta.exceptions import HostDoseNotExistInCmdbException from backend.db_meta.models import AppCache, BKCity +from backend.utils.string import base64_encode class Machine(AuditedModel): @@ -56,95 +59,81 @@ def __str__(self): return self.ip @property - def dbm_meta(self) -> list: + def dbm_meta(self) -> dict: proxies = self.proxyinstance_set.all() storages = self.storageinstance_set.all() host_labels = [] - def shrink_dbm_meta(dbm_meta): - """数据裁剪""" - - if not dbm_meta: - return [] - - # 剔除实例属性，仅保留集群属性 - first_one = copy.deepcopy(dbm_meta[0]) - for custom_attr in ["instance_role", "instance_port"]: - first_one.pop(custom_attr) - - return { - "version": "v1", - "common": first_one, - "custom": list( - map(lambda x: {"instance_role": x["instance_role"], "instance_port": x["instance_port"]}, dbm_meta) - ), - } - - def remove_duplicates(seq): - unique = set() - for d in seq: - t = tuple(d.items()) - unique.add(t) - - return shrink_dbm_meta([dict(x) for x in unique]) - - if proxies: - for proxy in proxies: - for cluster in proxy.cluster.all(): - tendb_cluster_spider_ext = getattr(proxy, "tendbclusterspiderext", None) - host_labels.append( - asdict( - CommonHostDBMeta( - app=AppCache.get_app_attr(cluster.bk_biz_id, default=cluster.bk_biz_id), - appid=str(cluster.bk_biz_id), - cluster_type=cluster.cluster_type, - cluster_domain=cluster.immute_domain, - db_type=ClusterType.cluster_type_to_db_type(cluster.cluster_type), - # tendbcluster中扩展了proxy的类型，需要特殊处理 - instance_role=tendb_cluster_spider_ext.spider_role - if tendb_cluster_spider_ext - else "proxy", - instance_port=str(proxy.port), - ) + def compress_dbm_meta_content(dbm_meta: dict) -> str: + """ + 压缩 dbm_meta + """ + # 使用gzip压缩 + # python3.6 gzip 不支持 mtime 参数，python3.10 可以直接使用 gzip.compress 压缩 + buf = io.BytesIO() + with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as f: + f.write(json.dumps(dbm_meta).encode("utf-8")) + compressed_data = buf.getvalue() + + # 将压缩后的字节转换为Base64编码的字符串 + base64_encoded_str = base64_encode(compressed_data) + return base64_encoded_str + + for proxy in proxies: + for cluster in proxy.cluster.all(): + tendb_cluster_spider_ext = getattr(proxy, "tendbclusterspiderext", None) + host_labels.append( + asdict( + CommonHostDBMeta( + app=AppCache.get_app_attr(cluster.bk_biz_id, default=cluster.bk_biz_id), + appid=str(cluster.bk_biz_id), + cluster_type=cluster.cluster_type, + cluster_domain=cluster.immute_domain, + db_type=ClusterType.cluster_type_to_db_type(cluster.cluster_type), + # tendbcluster中扩展了proxy的类型，需要特殊处理 + instance_role=tendb_cluster_spider_ext.spider_role + if tendb_cluster_spider_ext + else "proxy", + instance_port=str(proxy.port), ) ) - - if storages: - for storage in storages: - # influxdb需要单独处理 - if storage.cluster_type == ClusterType.Influxdb.value: - host_labels.append( - asdict( - CommonHostDBMeta( - app=AppCache.get_app_attr(storage.bk_biz_id, default=storage.bk_biz_id), - appid=str(storage.bk_biz_id), - cluster_domain=storage.machine.ip, - cluster_type=storage.cluster_type, - db_type=ClusterType.cluster_type_to_db_type(storage.cluster_type), - instance_role=storage.instance_role, - instance_port=str(storage.port), - ) + ) + + for storage in storages: + # influxdb需要单独处理 + if storage.cluster_type == ClusterType.Influxdb.value: + host_labels.append( + asdict( + CommonHostDBMeta( + app=AppCache.get_app_attr(storage.bk_biz_id, default=storage.bk_biz_id), + appid=str(storage.bk_biz_id), + cluster_domain=storage.machine.ip, + cluster_type=storage.cluster_type, + db_type=ClusterType.cluster_type_to_db_type(storage.cluster_type), + instance_role=storage.instance_role, + instance_port=str(storage.port), ) ) - continue - - for cluster in storage.cluster.all(): - host_labels.append( - asdict( - CommonHostDBMeta( - app=AppCache.get_app_attr(cluster.bk_biz_id, default=cluster.bk_biz_id), - appid=str(cluster.bk_biz_id), - cluster_domain=cluster.immute_domain, - cluster_type=cluster.cluster_type, - db_type=ClusterType.cluster_type_to_db_type(cluster.cluster_type), - instance_role=storage.instance_role, - instance_port=str(storage.port), - ) + ) + continue + + for cluster in storage.cluster.all(): + host_labels.append( + asdict( + CommonHostDBMeta( + app=AppCache.get_app_attr(cluster.bk_biz_id, default=cluster.bk_biz_id), + appid=str(cluster.bk_biz_id), + cluster_domain=cluster.immute_domain, + cluster_type=cluster.cluster_type, + db_type=ClusterType.cluster_type_to_db_type(cluster.cluster_type), + instance_role=storage.instance_role, + instance_port=str(storage.port), ) ) + ) - return remove_duplicates(host_labels) + return {"version": "v2", "content": compress_dbm_meta_content({"common": {}, "custom": host_labels})} @classmethod def get_host_info_from_cmdb(cls, bk_host_id: int) -> dict: diff --git a/dbm-ui/backend/db_meta/utils.py b/dbm-ui/backend/db_meta/utils.py index 9d5dbc6baf..4e7a31410c 100644 --- a/dbm-ui/backend/db_meta/utils.py +++ b/dbm-ui/backend/db_meta/utils.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging import os.path from collections import defaultdict @@ -34,6 +33,7 @@ from backend.db_services.ipchooser.constants import DB_MANAGE_SET from backend.db_services.ipchooser.query import resource from backend.flow.utils.cc_manage import CcManage +from backend.utils.string import base64_encode logger = logging.getLogger("root") @@ -69,7 +69,7 @@ def remove_cluster(cluster_id, job_clean=True, cc_clean=True): JobApi.fast_execute_script( { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, - "script_content": str(base64.b64encode(script_content.encode("utf-8")), "utf-8"), + "script_content": base64_encode(script_content), "task_name": _("清理集群"), "account_alias": "root", "script_language": 1, @@ -141,7 +141,7 @@ def remove_cluster_ips(bk_host_ids, job_clean=True, cc_clean=True): JobApi.fast_execute_script( { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, - "script_content": str(base64.b64encode(script_content.encode("utf-8")), "utf-8"), + "script_content": base64_encode(script_content), "task_name": _("清理集群"), "account_alias": "root", "script_language": 1, diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_proxy.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_proxy.py index 85c10f6b6a..60f8a5d376 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/db_proxy.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_proxy.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ -import base64 import copy import logging from collections import defaultdict @@ -31,6 +30,7 @@ from backend.db_proxy.models import ClusterExtension, DBCloudProxy, DBExtension from backend.db_services.ipchooser.query.resource import ResourceQueryHelper from backend.utils.redis import RedisConn +from backend.utils.string import base64_encode logger = logging.getLogger("celery") @@ -110,7 +110,7 @@ def _job_push_config_file(_cloud_id, _file_list, _nginx_list): "service_url": f"http://{extension.ip}:{extension.port}", } file_name = f"{extension.bk_biz_id}_{extension.db_type}_{extension.cluster_name}_nginx.conf" - file_content = str(base64.b64encode(template.render(conf_payload).encode("utf-8")), "utf-8") + file_content = base64_encode(template.render(conf_payload)) file_list.append({"file_name": file_name, "content": file_content}) # 这里先提前写入access url，至于是否执行成功根据is_flush diff --git a/dbm-ui/backend/db_proxy/views/job_callback/views.py b/dbm-ui/backend/db_proxy/views/job_callback/views.py index d1e5d5f436..fc0924aaef 100644 --- a/dbm-ui/backend/db_proxy/views/job_callback/views.py +++ b/dbm-ui/backend/db_proxy/views/job_callback/views.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ -import base64 import json import logging @@ -29,6 +28,7 @@ from backend.flow.consts import SUCCESS_LIST from backend.flow.utils.script_template import fast_execute_script_common_kwargs from backend.utils.redis import RedisConn +from backend.utils.string import base64_encode logger = logging.getLogger("root") @@ -73,7 +73,7 @@ def push_conf_callback(self, request): job_payload = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": "restart_nginx", - "script_content": str(base64.b64encode(restart_nginx_tpl.encode("utf-8")), "utf-8"), + "script_content": base64_encode(restart_nginx_tpl), "script_language": 1, "target_server": { "ip_list": [ diff --git a/dbm-ui/backend/db_proxy/views/jobapi/views.py b/dbm-ui/backend/db_proxy/views/jobapi/views.py index 3e1f8e0be9..5a9e3452d6 100644 --- a/dbm-ui/backend/db_proxy/views/jobapi/views.py +++ b/dbm-ui/backend/db_proxy/views/jobapi/views.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 from typing import Any, Dict from django.utils.translation import ugettext as _ @@ -31,6 +30,7 @@ TransferFileSerializer, ) from backend.db_proxy.views.views import BaseProxyPassViewSet +from backend.utils.string import base64_encode class JobApiProxyPassViewSet(BaseProxyPassViewSet): @@ -55,7 +55,7 @@ def fast_execute_script(self, request): job_payloads: Dict[str, Any] = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": _("DBM 快速脚本执行"), - "script_content": str(base64.b64encode(validated_data["script_content"].encode("utf-8")), "utf-8"), + "script_content": base64_encode(validated_data["script_content"]), "script_language": validated_data["script_language"], "target_server": {"ip_list": validated_data["ip_list"]}, "timeout": validated_data["timeout"], diff --git a/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py b/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py index b13195647c..be734f0b27 100644 --- a/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py +++ b/dbm-ui/backend/db_services/mysql/fixpoint_rollback/handlers.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import copy import json import re @@ -33,7 +32,7 @@ from backend.flow.consts import SUCCESS_LIST, DBActuatorActionEnum, DBActuatorTypeEnum, InstanceStatus, JobStatusEnum from backend.flow.engine.bamboo.scene.mysql.common.get_local_backup import get_local_backup_list from backend.flow.utils.script_template import dba_toolkit_actuator_template, fast_execute_script_common_kwargs -from backend.utils.string import pascal_to_snake +from backend.utils.string import base64_encode, pascal_to_snake from backend.utils.time import compare_time, datetime2str, find_nearby_time @@ -80,7 +79,7 @@ def _find_local_backup_script(self, port: int): } jinja_env = Environment() template = jinja_env.from_string(dba_toolkit_actuator_template) - return template.render(render_params).encode("utf-8") + return template.render(render_params) @staticmethod def _batch_make_job_requests(job_func: Callable, job_payloads: List[Dict]): @@ -427,9 +426,7 @@ def execute_backup_log_script(self) -> List[int]: execute_body: Dict[str, Any] = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": _("查询集群{}的备份日志").format(self.cluster.immute_domain), - "script_content": str( - base64.b64encode(self._find_local_backup_script(target_ip_infos[0]["port"])), "utf-8" - ), + "script_content": base64_encode(self._find_local_backup_script(target_ip_infos[0]["port"])), "script_language": 1, "target_server": { "ip_list": [ diff --git a/dbm-ui/backend/flow/plugins/components/collections/cloud/exec_service_script.py b/dbm-ui/backend/flow/plugins/components/collections/cloud/exec_service_script.py index e78648666f..4f99c9d5fa 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/cloud/exec_service_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/cloud/exec_service_script.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ -import base64 import logging from typing import List @@ -24,6 +23,7 @@ from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.cloud.cloud_act_payload import CloudServiceActPayload from backend.flow.utils.es.es_script_template import fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -93,7 +93,7 @@ def _exec_job_task(self, script_tpl, service_act_payload, target_ip_info, kwargs body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{kwargs['node_name']}_{kwargs['node_id']}", - "script_content": str(base64.b64encode(template.render(service_act_payload).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(service_act_payload)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/cloud/push_config_file.py b/dbm-ui/backend/flow/plugins/components/collections/cloud/push_config_file.py index de7cfe0f63..1986e23c57 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/cloud/push_config_file.py +++ b/dbm-ui/backend/flow/plugins/components/collections/cloud/push_config_file.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ -import base64 import copy import logging @@ -20,6 +19,7 @@ from backend.components import JobApi from backend.core import consts from backend.flow.plugins.components.collections.cloud.exec_service_script import ExecCloudScriptService +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -47,7 +47,7 @@ def _exec_job_task(self, script_tpl, service_act_payload, target_ip_info, kwargs payload["file_list"] = [ { "file_name": kwargs["conf_file_name"], - "content": str(base64.b64encode(template.render(service_act_payload).encode("utf-8")), "utf-8"), + "content": base64_encode(template.render(service_act_payload)), } ] payload["target_server"]["ip_list"] = target_ip_info diff --git a/dbm-ui/backend/flow/plugins/components/collections/common/check_resolv_conf.py b/dbm-ui/backend/flow/plugins/components/collections/common/check_resolv_conf.py index b4f098dbd6..756da4e724 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/common/check_resolv_conf.py +++ b/dbm-ui/backend/flow/plugins/components/collections/common/check_resolv_conf.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging from typing import List @@ -21,6 +20,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.redis.redis_script_template import redis_fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -53,7 +53,7 @@ def _execute(self, data, parent_data) -> bool: # 这里不能换成业务传进来的bk_biz_id，否则会获取作业状态失败 "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(shell_command.encode("utf-8")), "utf-8"), + "script_content": base64_encode(shell_command), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/common/dns_server.py b/dbm-ui/backend/flow/plugins/components/collections/common/dns_server.py index 0ab098f1da..c28183fbd4 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/common/dns_server.py +++ b/dbm-ui/backend/flow/plugins/components/collections/common/dns_server.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging import random from typing import List @@ -24,6 +23,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.redis.redis_script_template import redis_fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -108,7 +108,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(shell_command.encode("utf-8")), "utf-8"), + "script_content": base64_encode(shell_command), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/es/exec_es_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/es/exec_es_actuator_script.py index 9f14e8b510..0e3fd98bc7 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/es/exec_es_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/es/exec_es_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging from typing import List @@ -24,6 +23,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.es.es_script_template import actuator_template, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -78,9 +78,7 @@ def _execute(self, data, parent_data) -> bool: db_act_template["version_id"] = self._runtime_attrs["version"] db_act_template["uid"] = global_data["uid"] - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) @@ -91,7 +89,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/es/rewrite_es_config.py b/dbm-ui/backend/flow/plugins/components/collections/es/rewrite_es_config.py index 69b348cffc..9616d49dca 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/es/rewrite_es_config.py +++ b/dbm-ui/backend/flow/plugins/components/collections/es/rewrite_es_config.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ -import base64 import logging from typing import List @@ -19,6 +18,7 @@ from backend.components.mysql_priv_manager.client import MySQLPrivManagerApi from backend.flow.consts import MySQLPrivComponent, NameSpaceEnum from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -40,7 +40,7 @@ def _execute(self, data, parent_data) -> bool: "bk_cloud_id": global_data["bk_cloud_id"], } ], - "password": base64.b64encode(str(global_data["username"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(str(global_data["username"])), "username": MySQLPrivComponent.ES_FAKE_USER.value, "component": NameSpaceEnum.Es, "operator": "admin", @@ -55,7 +55,7 @@ def _execute(self, data, parent_data) -> bool: "bk_cloud_id": global_data["bk_cloud_id"], } ], - "password": base64.b64encode(str(global_data["password"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(str(global_data["password"])), "username": global_data["username"], "component": NameSpaceEnum.Es, "operator": "admin", diff --git a/dbm-ui/backend/flow/plugins/components/collections/hdfs/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/hdfs/exec_actuator_script.py index 66143da28c..70ffb7ab1c 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/hdfs/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/hdfs/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging from dataclasses import asdict @@ -25,6 +24,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.hdfs.hdfs_script_template import ACTUATOR_TEMPLATE, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -85,9 +85,7 @@ def _execute(self, data, parent_data) -> bool: db_act_template["version_id"] = self._runtime_attrs["version"] db_act_template["uid"] = global_data["uid"] - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) @@ -98,7 +96,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/hdfs/rewrite_hdfs_config.py b/dbm-ui/backend/flow/plugins/components/collections/hdfs/rewrite_hdfs_config.py index 149ccc9514..9f3277f8df 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/hdfs/rewrite_hdfs_config.py +++ b/dbm-ui/backend/flow/plugins/components/collections/hdfs/rewrite_hdfs_config.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging from typing import List @@ -22,6 +21,7 @@ from backend.flow.consts import ConfigTypeEnum, LevelInfoEnum, NameSpaceEnum from backend.flow.plugins.components.collections.common.base_service import BaseService from backend.ticket.constants import TicketType +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -56,7 +56,7 @@ def _execute(self, data, parent_data) -> bool: "bk_cloud_id": global_data["bk_cloud_id"], } ], - "password": base64.b64encode(str(global_data["password"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(global_data["password"]), "username": "root", "component": NameSpaceEnum.Hdfs, "operator": "admin", diff --git a/dbm-ui/backend/flow/plugins/components/collections/influxdb/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/influxdb/exec_actuator_script.py index 98d616a648..68ec7c61a2 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/influxdb/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/influxdb/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging from typing import List @@ -23,6 +22,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.kafka.script_template import ACTUATOR_TEMPLATE, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -70,9 +70,7 @@ def _execute(self, data, parent_data) -> bool: db_act_template["version_id"] = self._runtime_attrs["version"] db_act_template["uid"] = global_data["uid"] - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) @@ -83,7 +81,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/influxdb/influxdb_config.py b/dbm-ui/backend/flow/plugins/components/collections/influxdb/influxdb_config.py index 81a0af1659..f538da819d 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/influxdb/influxdb_config.py +++ b/dbm-ui/backend/flow/plugins/components/collections/influxdb/influxdb_config.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging from typing import List @@ -21,6 +20,7 @@ from backend.db_meta.models import StorageInstance from backend.flow.consts import MySQLPrivComponent, NameSpaceEnum from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -50,7 +50,7 @@ def _execute(self, data, parent_data) -> bool: # 把用户名当密码存 query_params = { "instances": [{"ip": str(storage_obj.id), "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(global_data["username"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(global_data["username"]), "username": MySQLPrivComponent.INFLUXDB_FAKE_USER.value, "component": NameSpaceEnum.Influxdb, "operator": "admin", @@ -59,7 +59,7 @@ def _execute(self, data, parent_data) -> bool: # 存真实的用户名密码 query_params = { "instances": [{"ip": str(storage_obj.id), "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(global_data["password"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(global_data["password"]), "username": global_data["username"], "component": NameSpaceEnum.Influxdb, "operator": "admin", diff --git a/dbm-ui/backend/flow/plugins/components/collections/kafka/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/kafka/exec_actuator_script.py index 98d616a648..68ec7c61a2 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/kafka/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/kafka/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging from typing import List @@ -23,6 +22,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.kafka.script_template import ACTUATOR_TEMPLATE, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -70,9 +70,7 @@ def _execute(self, data, parent_data) -> bool: db_act_template["version_id"] = self._runtime_attrs["version"] db_act_template["uid"] = global_data["uid"] - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) @@ -83,7 +81,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/kafka/kafka_config.py b/dbm-ui/backend/flow/plugins/components/collections/kafka/kafka_config.py index a7bf6426e6..58820a731c 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/kafka/kafka_config.py +++ b/dbm-ui/backend/flow/plugins/components/collections/kafka/kafka_config.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging from typing import List @@ -20,6 +19,7 @@ from backend.components.mysql_priv_manager.client import MySQLPrivManagerApi from backend.flow.consts import ConfigTypeEnum, LevelInfoEnum, MySQLPrivComponent, NameSpaceEnum from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -64,7 +64,7 @@ def _execute(self, data, parent_data) -> bool: # 密码服务，把用户名也当密码存 query_params = { "instances": [{"ip": global_data["domain"], "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(global_data["username"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(global_data["username"]), "username": MySQLPrivComponent.KAFKA_FAKE_USER.value, "component": NameSpaceEnum.Kafka, "operator": "admin", @@ -73,7 +73,7 @@ def _execute(self, data, parent_data) -> bool: # 存真实的账号密码 query_params = { "instances": [{"ip": global_data["domain"], "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(global_data["password"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(global_data["password"]), "username": global_data["username"], "component": NameSpaceEnum.Kafka, "operator": "admin", diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/exec_actuator_script.py index 3b4daf2052..7be43b6e01 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mysql/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import copy import json import logging @@ -28,6 +27,7 @@ from backend.flow.utils.mysql.get_mysql_sys_user import get_mysql_sys_users from backend.flow.utils.mysql.mysql_act_playload import MysqlActPayload from backend.flow.utils.script_template import actuator_template, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") cpl = re.compile("(?P.+?)") # 非贪婪模式，只匹配第一次出现的自定义tag @@ -109,9 +109,8 @@ def _execute(self, data, parent_data) -> bool: ) # payload参数转换base64格式 - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) + FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) # 脚本内容 @@ -121,10 +120,10 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, - "script_param": str(base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8"), + "script_param": base64_encode(json.dumps(db_act_template["payload"])), } # self.log_info("[{}] ready start task with body {}".format(node_name, body)) diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/mysql_os_init.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/mysql_os_init.py index b4d3ce9795..274bf93fa9 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mysql/mysql_os_init.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/mysql_os_init.py @@ -20,6 +20,7 @@ from backend.flow.consts import DBA_ROOT_USER, DEFAULT_INSTANCE, MySQLPrivComponent, UserName from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.script_template import fast_execute_script_common_kwargs +from backend.utils.string import base64_encode cpl = re.compile("(?P.+?)") @@ -99,7 +100,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": "DBM_MySQL_OS_Init", - "script_content": str(base64.b64encode(script_content.encode("utf-8")), "utf-8"), + "script_content": base64_encode(script_content), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } @@ -234,7 +235,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": "DBM-Init-Mysql-Os", - "script_content": str(base64.b64encode(script_content.encode("utf-8")), "utf-8"), + "script_content": base64_encode(script_content), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } @@ -287,7 +288,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": "DBM-Get-Os-Sys-Param", - "script_content": str(base64.b64encode(script_content.encode("utf-8")), "utf-8"), + "script_content": base64_encode(script_content), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/pulsar/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/pulsar/exec_actuator_script.py index 153c92b1b4..27c9ae52dc 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/pulsar/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/pulsar/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json from dataclasses import asdict from typing import List @@ -24,6 +23,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.pulsar.pulsar_script_template import ACTUATOR_TEMPLATE, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode class ExecutePulsarActuatorScriptService(BkJobService): @@ -82,9 +82,7 @@ def _execute(self, data, parent_data) -> bool: db_act_template["version_id"] = self._runtime_attrs["version"] db_act_template["uid"] = global_data["uid"] - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) @@ -95,7 +93,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/pulsar/rewrite_pulsar_config.py b/dbm-ui/backend/flow/plugins/components/collections/pulsar/rewrite_pulsar_config.py index c3038fce5c..55ac237499 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/pulsar/rewrite_pulsar_config.py +++ b/dbm-ui/backend/flow/plugins/components/collections/pulsar/rewrite_pulsar_config.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 from typing import List from pipeline.component_framework.component import Component @@ -24,6 +23,7 @@ from backend.flow.utils.pulsar.consts import PulsarConfigEnum from backend.flow.utils.pulsar.pulsar_context_dataclass import PulsarApplyContext from backend.ticket.constants import TicketType +from backend.utils.string import base64_encode class WriteBackPulsarConfigService(BaseService): @@ -134,7 +134,7 @@ def write_auth_to_prv_manager(self, global_data: dict, token: str): # 写入到密码服务，把用户名当密码存 query_params = { "instances": [{"ip": global_data["domain"], "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(global_data["username"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(str(global_data["username"])), "username": MySQLPrivComponent.PULSAR_FAKE_USER.value, "component": NameSpaceEnum.Pulsar, "operator": "admin", @@ -143,7 +143,7 @@ def write_auth_to_prv_manager(self, global_data: dict, token: str): # 存储真正的账号密码 query_params = { "instances": [{"ip": global_data["domain"], "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(global_data["password"]).encode("utf-8")).decode("utf-8"), + "password": base64_encode(str(global_data["password"])), "username": global_data["username"], "component": NameSpaceEnum.Pulsar, "operator": "admin", @@ -152,7 +152,7 @@ def write_auth_to_prv_manager(self, global_data: dict, token: str): # 存储token query_params = { "instances": [{"ip": global_data["domain"], "port": 0, "bk_cloud_id": global_data["bk_cloud_id"]}], - "password": base64.b64encode(str(f"token:{token}").encode("utf-8")).decode("utf-8"), + "password": base64_encode(str(f"token:{token}")), "username": PulsarConfigEnum.ClientAuthenticationParameters, "component": NameSpaceEnum.Pulsar, "operator": "admin", diff --git a/dbm-ui/backend/flow/plugins/components/collections/redis/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/redis/exec_actuator_script.py index 05ebaa76ab..14dec6d237 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/redis/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/redis/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging import re @@ -29,6 +28,7 @@ redis_actuator_template, redis_fast_execute_script_common_kwargs, ) +from backend.utils.string import base64_encode logger = logging.getLogger("json") cpl = re.compile("(?P.+?)") # 非贪婪模式，只匹配第一次出现的自定义tag @@ -102,9 +102,7 @@ def _execute(self, data, parent_data) -> bool: if getattr(trans_data, "tendis_backup_info"): db_act_template["payload"]["backup_tasks"] = trans_data.tendis_backup_info - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) @@ -115,7 +113,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, "timeout": 86400, diff --git a/dbm-ui/backend/flow/plugins/components/collections/redis/exec_data_structure_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/redis/exec_data_structure_actuator_script.py index bd82bf601f..65419bdb74 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/redis/exec_data_structure_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/redis/exec_data_structure_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging import re @@ -30,6 +29,7 @@ redis_fast_execute_script_common_kwargs, ) from backend.ticket.constants import TicketType +from backend.utils.string import base64_encode logger = logging.getLogger("json") cpl = re.compile("(?P.+?)") # 非贪婪模式，只匹配第一次出现的自定义tag @@ -108,9 +108,7 @@ def _execute(self, data, parent_data) -> bool: if getattr(trans_data, "tendis_backup_info"): db_act_template["payload"]["backup_tasks"] = trans_data.tendis_backup_info - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) db_act_template["file_name"] = ( @@ -123,7 +121,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/redis/exec_shell_script.py b/dbm-ui/backend/flow/plugins/components/collections/redis/exec_shell_script.py index 48031469d4..7b4c91d357 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/redis/exec_shell_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/redis/exec_shell_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import logging import traceback from typing import List @@ -25,6 +24,7 @@ from backend.flow.plugins.components.collections.common.base_service import BaseService, BkJobService from backend.flow.utils.redis.redis_context_dataclass import RedisDataStructureContext from backend.flow.utils.redis.redis_script_template import redis_fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -56,7 +56,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(shell_command.encode("utf-8")), "utf-8"), + "script_content": base64_encode(shell_command), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } @@ -292,7 +292,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(shell_command.encode("utf-8")), "utf-8"), + "script_content": base64_encode(shell_command), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/redis/psuh_data_structure_json_script.py b/dbm-ui/backend/flow/plugins/components/collections/redis/psuh_data_structure_json_script.py index 53a947e56f..772d8e9d56 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/redis/psuh_data_structure_json_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/redis/psuh_data_structure_json_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import copy import json import logging @@ -28,6 +27,7 @@ from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.redis.redis_script_template import redis_data_structure_payload_template from backend.ticket.constants import TicketType +from backend.utils.string import base64_encode logger = logging.getLogger("json") cpl = re.compile("(?P.+?)") # 非贪婪模式，只匹配第一次出现的自定义tag @@ -103,9 +103,7 @@ def _execute(self, data, parent_data) -> bool: self.log_info(_("[{}] kwargs['payload'] 是不完整，需要将{}内容加到payload中").format(node_name, kwargs["cluster"])) db_act_template["payload"].update(kwargs["cluster"]) - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) # 脚本内容 diff --git a/dbm-ui/backend/flow/plugins/components/collections/redis/redis_dts.py b/dbm-ui/backend/flow/plugins/components/collections/redis/redis_dts.py index 7370dfc3bc..de7068c4c5 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/redis/redis_dts.py +++ b/dbm-ui/backend/flow/plugins/components/collections/redis/redis_dts.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import datetime import logging import re @@ -61,6 +60,7 @@ from backend.flow.utils.redis.redis_proxy_util import get_twemproxy_cluster_hash_tag from backend.ticket.constants import TicketType from backend.utils.basic import generate_root_id +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -472,12 +472,8 @@ def _execute(self, data, parent_data): job.save() job_id = job.id - src_password_base64 = base64.b64encode( - kwargs["cluster"]["src"]["redis_password"].encode("utf-8") - ).decode("utf-8") - dst_passsword_base64 = base64.b64encode( - kwargs["cluster"]["dst"]["cluster_password"].encode("utf-8") - ).decode("utf-8") + src_password_base64 = base64_encode(kwargs["cluster"]["src"]["redis_password"]) + dst_passsword_base64 = base64_encode(kwargs["cluster"]["dst"]["cluster_password"]) task_white_regex = get_safe_regex_pattern(kwargs["cluster"]["info"]["key_white_regex"]) task_black_regex = get_safe_regex_pattern(kwargs["cluster"]["info"]["key_black_regex"]) diff --git a/dbm-ui/backend/flow/plugins/components/collections/redis/redis_old_backup_records.py b/dbm-ui/backend/flow/plugins/components/collections/redis/redis_old_backup_records.py index c25454a4f7..d98213c198 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/redis/redis_old_backup_records.py +++ b/dbm-ui/backend/flow/plugins/components/collections/redis/redis_old_backup_records.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging from datetime import datetime, timedelta @@ -24,6 +23,7 @@ from backend.flow.models import FlowNode from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.redis.redis_script_template import redis_fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") @@ -90,7 +90,7 @@ def _execute(self, data, parent_data) -> bool: self.log_info(f"start get last {ndays} days old backup records of {server_ip}") query_result = get_last_n_days_backup_records(ndays, bk_cloud_id, server_ip) - encode_str = str(base64.b64encode(json.dumps(query_result).encode("utf-8")), "utf-8") + encode_str = base64_encode(json.dumps(query_result)) self.log_info(f"success get last {ndays} days old backup records of {server_ip}") @@ -104,7 +104,7 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(shell_command.encode("utf-8")), "utf-8"), + "script_content": base64_encode(shell_command), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/riak/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/riak/exec_actuator_script.py index e2c06b3ee4..cc2a458154 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/riak/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/riak/exec_actuator_script.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import copy import json import logging @@ -26,6 +25,7 @@ from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.riak.riak_act_payload import RiakActPayload from backend.flow.utils.script_template import actuator_template, fast_execute_script_common_kwargs +from backend.utils.string import base64_encode logger = logging.getLogger("json") cpl = re.compile("(?P.+?)") # 非贪婪模式，只匹配第一次出现的自定义tag @@ -99,9 +99,7 @@ def _execute(self, data, parent_data) -> bool: db_act_template["uid"] = global_data["uid"] # payload参数转换base64格式 - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) # 脚本内容 @@ -111,8 +109,8 @@ def _execute(self, data, parent_data) -> bool: body = { "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), - "script_param": str(base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), + "script_param": base64_encode(json.dumps(db_act_template["payload"])), "script_language": 1, "target_server": {"ip_list": target_ip_info}, } diff --git a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/exec_actuator_script.py b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/exec_actuator_script.py index 1b586234b6..b4b39144a9 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/exec_actuator_script.py +++ b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/exec_actuator_script.py @@ -8,7 +8,6 @@ specific language governing permissions and limitations under the License. """ -import base64 import json import logging import re @@ -25,6 +24,7 @@ from backend.flow.plugins.components.collections.common.base_service import BkJobService from backend.flow.utils.script_template import sqlserver_actuator_template from backend.flow.utils.sqlserver.sqlserver_act_payload import SqlserverActPayload +from backend.utils.string import base64_encode logger = logging.getLogger("json") cpl = re.compile("(?P.+?)") # 非贪婪模式，只匹配第一次出现的自定义tag @@ -84,9 +84,8 @@ def _execute(self, data, parent_data) -> bool: ) # payload参数转换base64格式 - db_act_template["payload"] = str( - base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" - ) + db_act_template["payload"] = base64_encode(json.dumps(db_act_template["payload"])) + # 更新节点信息 FlowNode.objects.filter(root_id=root_id, node_id=node_id).update(hosts=exec_ips) @@ -100,10 +99,10 @@ def _execute(self, data, parent_data) -> bool: "is_param_sensitive": 1, "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, "task_name": f"DBM_{node_name}_{node_id}", - "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_content": base64_encode(template.render(db_act_template)), "script_language": 5, "target_server": {"ip_list": exec_ips}, - "script_param": str(base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8"), + "script_param": base64_encode(json.dumps(db_act_template["payload"])), } self.log_debug("[{}] ready start task with body {}".format(node_name, body)) diff --git a/dbm-ui/backend/flow/utils/base/payload_handler.py b/dbm-ui/backend/flow/utils/base/payload_handler.py index 0c944a2122..b23a73915c 100644 --- a/dbm-ui/backend/flow/utils/base/payload_handler.py +++ b/dbm-ui/backend/flow/utils/base/payload_handler.py @@ -25,6 +25,7 @@ from backend.flow.consts import DEFAULT_INSTANCE, ConfigTypeEnum, LevelInfoEnum, MySQLPrivComponent, UserName from backend.flow.utils.mysql.get_mysql_sys_user import generate_mysql_tmp_user from backend.ticket.constants import TicketType +from backend.utils.string import base64_encode apply_list = [ TicketType.MYSQL_SINGLE_APPLY.value, @@ -297,17 +298,17 @@ def redis_save_cluster_password( if redis_password and (not redis_password.isspace()): query_params["component"] = MySQLPrivComponent.REDIS.value - query_params["password"] = base64.b64encode(redis_password.encode("utf-8")).decode("utf-8") + query_params["password"] = base64_encode(redis_password) MySQLPrivManagerApi.modify_password(params=query_params) if redis_proxy_password and (not redis_proxy_password.isspace()): query_params["component"] = MySQLPrivComponent.REDIS_PROXY.value - query_params["password"] = base64.b64encode(redis_proxy_password.encode("utf-8")).decode("utf-8") + query_params["password"] = base64_encode(redis_proxy_password) MySQLPrivManagerApi.modify_password(params=query_params) if redis_proxy_admin_password and (not redis_proxy_admin_password.isspace()): query_params["component"] = MySQLPrivComponent.REDIS_PROXY_ADMIN.value - query_params["password"] = base64.b64encode(redis_proxy_admin_password.encode("utf-8")).decode("utf-8") + query_params["password"] = base64_encode(redis_proxy_admin_password) MySQLPrivManagerApi.modify_password(params=query_params) return True diff --git a/dbm-ui/backend/flow/utils/cloud/cloud_act_payload.py b/dbm-ui/backend/flow/utils/cloud/cloud_act_payload.py index 784d53a553..d79d3324bd 100644 --- a/dbm-ui/backend/flow/utils/cloud/cloud_act_payload.py +++ b/dbm-ui/backend/flow/utils/cloud/cloud_act_payload.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json import logging @@ -30,6 +29,7 @@ ) from backend.flow.engine.exceptions import ServiceDoesNotApply from backend.flow.utils.base.payload_handler import PayloadHandler +from backend.utils.string import base64_encode logger = logging.getLogger("flow") @@ -199,7 +199,7 @@ def get_redis_dts_server_apply_payload(self): "password": redis_os_acc["os_password"], } paylod_json = json.dumps(paylod_obj) - payload_base64 = str(base64.b64encode(paylod_json.encode("utf-8")), "utf-8") + payload_base64 = base64_encode(paylod_json) return { "bk_dbm_nginx_url": nginx_url, "bk_dbm_cloud_id": self.cloud_id, diff --git a/dbm-ui/backend/tests/conftest.py b/dbm-ui/backend/tests/conftest.py index 1d6fa53556..dcd586a645 100644 --- a/dbm-ui/backend/tests/conftest.py +++ b/dbm-ui/backend/tests/conftest.py @@ -8,6 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import ipaddress import os import mock @@ -15,10 +16,12 @@ from django.contrib.auth import get_user_model from django.utils.crypto import get_random_string -from backend.db_meta.enums import ClusterType -from backend.db_meta.models import BKCity, DBModule, LogicalCity +from backend.db_meta import models +from backend.db_meta.enums import AccessLayer, ClusterType, MachineType +from backend.db_meta.models import BKCity, Cluster, DBModule, LogicalCity, Machine from backend.tests.constants import TEST_ADMIN_USERNAME from backend.tests.mock_data import constant +from backend.tests.mock_data.components import cc def mock_bk_user(username): @@ -53,6 +56,46 @@ def create_city(): BKCity.objects.create(logical_city_id=2, bk_idc_city_id=28, bk_idc_city_name="上海") +@pytest.fixture +def machine_fixture(create_city): + bk_city = BKCity.objects.first() + Machine.objects.create( + ip=cc.NORMAL_IP2, + bk_biz_id=constant.BK_BIZ_ID, + machine_type=MachineType.BACKEND.value, + bk_city=bk_city, + bk_cloud_id=1, + bk_host_id=2, + ) + + +@pytest.fixture +def init_proxy_machine(create_city): + bk_city = models.BKCity.objects.first() + machine = models.Machine.objects.create( + ip=cc.NORMAL_IP, + bk_biz_id=constant.BK_BIZ_ID, + machine_type=MachineType.BACKEND.value, + bk_city=bk_city, + access_layer=AccessLayer.PROXY, + ) + return machine + + +@pytest.fixture +def init_storage_machine(create_city): + bk_city = models.BKCity.objects.first() + machine = models.Machine.objects.create( + ip=cc.NORMAL_IP, + bk_biz_id=constant.BK_BIZ_ID, + machine_type=MachineType.BACKEND.value, + bk_city=bk_city, + access_layer=AccessLayer.STORAGE, + bk_host_id=int(ipaddress.IPv4Address(cc.NORMAL_IP)), + ) + return machine + + @pytest.fixture def init_db_module(): DBModule.objects.create( @@ -63,4 +106,15 @@ def init_db_module(): ) +@pytest.fixture +def init_cluster(): + Cluster.objects.create( + bk_biz_id=constant.BK_BIZ_ID, + name=constant.CLUSTER_NAME, + db_module_id=constant.DB_MODULE_ID, + immute_domain=constant.CLUSTER_IMMUTE_DOMAIN, + cluster_type=ClusterType.TenDBHA.value, + ) + + mark_global_skip = pytest.mark.skipif(os.environ.get("GLOBAL_SKIP") == "true", reason="disable in landun WIP") diff --git a/dbm-ui/backend/tests/db_meta/api/machine/test_apis.py b/dbm-ui/backend/tests/db_meta/api/machine/test_apis.py index 06c3e1782c..cd61f22a73 100644 --- a/dbm-ui/backend/tests/db_meta/api/machine/test_apis.py +++ b/dbm-ui/backend/tests/db_meta/api/machine/test_apis.py @@ -14,26 +14,13 @@ from backend.db_meta import api from backend.db_meta.enums import MachineType -from backend.db_meta.models import BKCity, Machine +from backend.db_meta.models import Machine from backend.tests.mock_data import constant from backend.tests.mock_data.components import cc pytestmark = pytest.mark.django_db -@pytest.fixture -def machine_fixture(create_city): - bk_city = BKCity.objects.first() - Machine.objects.create( - ip=cc.NORMAL_IP2, - bk_biz_id=constant.BK_BIZ_ID, - machine_type=MachineType.BACKEND.value, - bk_city=bk_city, - bk_cloud_id=1, - bk_host_id=2, - ) - - class TestCreateMachine: @patch("backend.db_meta.api.machine.apis.CCApi", cc.CCApiMock()) def test_create_success(self, create_city): diff --git a/dbm-ui/backend/tests/db_meta/api/proxy_instance/test_apis.py b/dbm-ui/backend/tests/db_meta/api/proxy_instance/test_apis.py index 6dc46be8d9..0453306585 100644 --- a/dbm-ui/backend/tests/db_meta/api/proxy_instance/test_apis.py +++ b/dbm-ui/backend/tests/db_meta/api/proxy_instance/test_apis.py @@ -23,19 +23,6 @@ TEST_INVALID_PROXY_PORT = 99999 -@pytest.fixture -def init_proxy_machine(create_city): - bk_city = models.BKCity.objects.first() - machine = models.Machine.objects.create( - ip=cc.NORMAL_IP, - bk_biz_id=constant.BK_BIZ_ID, - machine_type=MachineType.BACKEND.value, - bk_city=bk_city, - access_layer=AccessLayer.PROXY, - ) - return machine - - class TestProxyInstance: def test_create_success(self, init_proxy_machine): """创建成功""" diff --git a/dbm-ui/backend/tests/db_meta/models/__init__.py b/dbm-ui/backend/tests/db_meta/models/__init__.py new file mode 100644 index 0000000000..aa5085c628 --- /dev/null +++ b/dbm-ui/backend/tests/db_meta/models/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/dbm-ui/backend/tests/db_meta/models/test_machine.py b/dbm-ui/backend/tests/db_meta/models/test_machine.py new file mode 100644 index 0000000000..90834ea394 --- /dev/null +++ b/dbm-ui/backend/tests/db_meta/models/test_machine.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +import gzip +import json +from unittest.mock import patch + +import pytest + +from backend.db_meta import api +from backend.db_meta.models import Cluster, Machine +from backend.tests.db_meta.api.dbha.test_apis import TEST_PROXY_PORT1, TEST_PROXY_PORT2 +from backend.tests.mock_data.components import cc +from backend.tests.mock_data.components.cc import CCApiMock + +pytestmark = pytest.mark.django_db + + +class TestMachine: + @patch("backend.db_meta.models.app.CCApi", CCApiMock()) + def test_dbm_meta(self, init_proxy_machine, init_cluster): + machine = Machine.objects.first() + cluster = Cluster.objects.first() + proxy_objs = api.proxy_instance.create( + [ + { + "ip": cc.NORMAL_IP, + "port": TEST_PROXY_PORT1, + }, + { + "ip": cc.NORMAL_IP, + "port": TEST_PROXY_PORT2, + }, + ] + ) + cluster.proxyinstance_set.add(*proxy_objs) + dbm_meta = machine.dbm_meta + assert dbm_meta == { + "version": "v2", + "content": "H4sIAAAAAAAC/9WOSwoCMRBErzL0WobMgBuvIjLkJwaTdEwiTAi5u90uRI9gr6rrFUV10BgCRjhNfRwm0" + "M9SMdB37iBTIgFANkln+FmFOAq+lW3tKW7zZjBIxx1wlXc7GzVT63egtmQZVxuNuklGRn3c0MrDs+diqT" + "Jqu2X0b5Iy7u2HJMyVyUIbFqDJf7BTwLiMF+mlgLRqAQAA", + } + + # 将Base64字符串解码回压缩的字节数据 + compressed_bytes_from_base64 = base64.b64decode(dbm_meta["content"]) + + # 使用gzip解压缩 + decompressed_bytes = gzip.decompress(compressed_bytes_from_base64) + + # 将解压缩的字节解码回原始字符串 + decompressed_string = decompressed_bytes.decode("utf-8") + dbm_meta_content = json.loads(decompressed_string) + assert dbm_meta_content == { + "common": {}, + "custom": [ + { + "app": "", + "appid": "2005000002", + "cluster_domain": "fake.db.com", + "cluster_type": "tendbha", + "db_type": "mysql", + "instance_role": "proxy", + "instance_port": "10001", + }, + { + "app": "", + "appid": "2005000002", + "cluster_domain": "fake.db.com", + "cluster_type": "tendbha", + "db_type": "mysql", + "instance_role": "proxy", + "instance_port": "10000", + }, + ], + } diff --git a/dbm-ui/backend/utils/string.py b/dbm-ui/backend/utils/string.py index 8a9d3965bd..ae30ee2f15 100644 --- a/dbm-ui/backend/utils/string.py +++ b/dbm-ui/backend/utils/string.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - +import base64 import json import re from typing import Any, List, Optional, Tuple, Union @@ -186,3 +186,24 @@ def split_str_to_list(string: str) -> List[str]: for char in [" ", "\n", "\t", "\r", "\f", "\v"]: string = string.replace(char, ",") return string.split(",") + + +def base64_encode(content: Union[str, bytes]) -> str: + """ + 将字符串转为base64编码 + :param content: 待转换字符串 + """ + if isinstance(content, str): + content = content.encode("utf-8") + + return base64.b64encode(content).decode("utf-8") + + +def base64_decode(content: Union[str, bytes]) -> str: + """ + 将base64编码的字符串转为原始字符串 + :param content: 待转换字符串 + """ + if isinstance(content, str): + content = content.encode("utf-8") + return base64.b64decode(content).decode("utf-8") diff --git a/dbm-ui/bin/pytest.sh b/dbm-ui/bin/pytest.sh index a2f95c0c1b..8fa90909b5 100755 --- a/dbm-ui/bin/pytest.sh +++ b/dbm-ui/bin/pytest.sh @@ -4,4 +4,4 @@ cd $SCRIPT_DIR && cd .. || exit 1 source bin/environ.sh -pytest backend/tests +pytest backend/tests --cov diff --git a/dbm-ui/pytest.ini b/dbm-ui/pytest.ini index bfa3a1c3c2..5fb637dda0 100644 --- a/dbm-ui/pytest.ini +++ b/dbm-ui/pytest.ini @@ -4,7 +4,7 @@ DJANGO_SETTINGS_MODULE = config.prod addopts = --disable-warnings --no-migrations --reuse-db ; --reuse-db -vv -x -s ; --lf - --cov +; --cov ; --cov-report=html ; --create-db ; -s -m "login or pika" diff --git a/dbm-ui/scripts/ci/code_quality.sh b/dbm-ui/scripts/ci/code_quality.sh index f3dd2a21ff..5a9c5774c0 100755 --- a/dbm-ui/scripts/ci/code_quality.sh +++ b/dbm-ui/scripts/ci/code_quality.sh @@ -5,7 +5,7 @@ source "${VENV_DIR}/bin/activate" DBM_DIR="./dbm-ui" cd $DBM_DIR -TEST_LOGS=$(pytest) +TEST_LOGS=$(pytest --cov) # TEST_LOGS e.g. # ============================= test session starts ============================== # platform linux -- Python 3.6.15, pytest-6.2.4, py-1.11.0, pluggy-0.13.1