feat(backend): 安装包自动化上传 #6153

dbm-ui/backend/dbm_init/medium/handlers.py

@@ -243,7 +243,7 @@ def add_version(version):
             lock_file.write(yaml.safe_dump(lock_info))
 
     @classmethod
-    def build_medium(cls, bkrepo_tmp_dir):
+    def build_medium(cls, bkrepo_tmp_dir, installation=False):
         # 加载lock文件，获取介质的版本信息
         medium_lock_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "medium.lock")
         with open(medium_lock_path, "r") as lock_file:
@@ -252,6 +252,9 @@ def build_medium(cls, bkrepo_tmp_dir):
         for db_type, mediums in lock_info.items():
             for medium in mediums:
                 for medium_type, medium_info in medium.items():
+                    # 如果介质和安装模式不匹配，忽略
+                    if medium_info.get("installation", False) != installation:
+                        continue
                     # 将编译好的介质复制到指定目录
                     target_medium_path = f"{bkrepo_tmp_dir}/{db_type}/{medium_type}/{medium_info['version']}"
                     result = subprocess.run(

dbm-ui/backend/dbm_init/medium/installs/Dockerfile-ES

@@ -0,0 +1,14 @@
+## busybox，集合了常用的linux工具箱
+FROM busybox:latest
+
+ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download
+ENV VERSION=v1.0.0
+
+## 获取二进制介质，将静态介质存放到/data目录
+RUN set -ex && \
+    mkdir /data && cd /data && \
+    wget -qO espack-7.10.2.tar.gz ${DOWNLOAD_URL}/${VERSION}/espack-7.10.2.tar.gz --no-check-certificate  && \
+    wget -qO dbm_elasticsearch_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_elasticsearch_exporter.tgz --no-check-certificate
+
+## 将install的文件移动到/install/xxx，这样通过挂载/install可以共享文件
+ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/es && mv /data/* /install/es/"]

dbm-ui/backend/dbm_init/medium/installs/Dockerfile-HDFS

@@ -0,0 +1,14 @@
+## busybox，集合了常用的linux工具箱
+FROM busybox:latest
+
+ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download
+ENV VERSION=v1.0.0
+
+## 获取二进制介质，将静态介质存放到/data目录
+RUN set -ex && \
+    mkdir /data && cd /data && \
+    wget -qO hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz ${DOWNLOAD_URL}/${VERSION}/hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz --no-check-certificate  && \
+    wget -qO dbm_hdfs_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_hdfs_exporter.tgz --no-check-certificate
+
+## 将install的文件移动到/install/xxx，这样通过挂载/install可以共享文件
+ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/hdfs && mv /data/* /install/hdfs/"]

dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Kafka

@@ -0,0 +1,15 @@
+## busybox，集合了常用的linux工具箱
+FROM busybox:latest
+
+ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download
+ENV VERSION=v1.0.0
+
+## 获取二进制介质，将静态介质存放到/data目录
+RUN set -ex && \
+    mkdir /data && cd /data && \
+    wget -qO kafkapack-2.4.0.tar.gz ${DOWNLOAD_URL}/${VERSION}/kafkapack-2.4.0.tar.gz --no-check-certificate  && \
+    wget -qO dbm_kafka_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_kafka_exporter.tgz --no-check-certificate && \
+    wget -qO dbm_kafka_bkpull.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_kafka_bkpull.tgz --no-check-certificate
+
+## 将install的文件移动到/install/xxx，这样通过挂载/install可以共享文件
+ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/kafka && mv /data/* /install/kafka/"]

dbm-ui/backend/dbm_init/medium/installs/Dockerfile-MySQL

@@ -0,0 +1,19 @@
+## busybox，集合了常用的linux工具箱
+FROM busybox:latest
+
+ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download
+ENV VERSION=v1.0.0
+
+## 获取二进制介质，将静态介质存放到/data目录
+RUN set -ex && \
+    mkdir /data && cd /data && \
+    wget -qO dbm_mysqld_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_mysqld_exporter.tgz --no-check-certificate  && \
+    wget -qO dbm_mysqlproxy_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_mysqlproxy_exporter.tgz --no-check-certificate  && \
+    wget -qO dbm_tbinlogdumper_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_tbinlogdumper_exporter.tgz --no-check-certificate  && \
+    wget -qO mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz ${DOWNLOAD_URL}/${VERSION}/mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz --no-check-certificate  && \
+    wget -qO mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz ${DOWNLOAD_URL}/${VERSION}/mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz --no-check-certificate && \
+    wget -qO mysql-proxy-0.82.15.tar.gz ${DOWNLOAD_URL}/${VERSION}/mysql-proxy-0.82.15.tar.gz --no-check-certificate
+
+
+## 将install的文件移动到/mysql-install，这样通过挂载/mysql-install可以共享文件
+ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/mysql && mv /data/* /install/mysql/"]

dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Redis

@@ -0,0 +1,18 @@
+## busybox，集合了常用的linux工具箱
+FROM busybox:latest
+
+ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download
+ENV VERSION=v1.0.0
+
+## 获取二进制介质，将静态介质存放到/data目录
+RUN set -ex && \
+    mkdir /data && cd /data && \
+    wget -qO dbm_predixy_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_predixy_exporter.tgz --no-check-certificate  && \
+    wget -qO dbm_redis_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_redis_exporter.tgz --no-check-certificate  && \
+    wget -qO predixy-1.4.3.tar.gz ${DOWNLOAD_URL}/${VERSION}/predixy-1.4.3.tar.gz --no-check-certificate  && \
+    wget -qO redis-5.0.14.tar.gz ${DOWNLOAD_URL}/${VERSION}/redis-5.0.14.tar.gz --no-check-certificate  && \
+    wget -qO redis-6.2.14.tar.gz ${DOWNLOAD_URL}/${VERSION}/redis-6.2.14.tar.gz --no-check-certificate && \
+    wget -qO tendisplus-2.7.3-rocksdb-v8.5.3.tgz ${DOWNLOAD_URL}/${VERSION}/tendisplus-2.7.3-rocksdb-v8.5.3.tgz --no-check-certificate
+
+## 将install的文件移动到/install/xxx，这样通过挂载/install可以共享文件
+ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/redis && mv /data/* /install/redis/"]

dbm-ui/backend/dbm_init/medium/main.py

@@ -36,6 +36,7 @@
     elif args.type == "build":
         MediumHandler.build_medium(bkrepo_tmp_dir=path)
     elif args.type == "upload":
+        MediumHandler.build_medium(bkrepo_tmp_dir=path, installation=True)
         MediumHandler().upload_medium(path=args.db, bkrepo_tmp_dir=path)
     elif args.type == "sync":
         MediumHandler().sync_from_bkrepo(db_type=args.db)

dbm-ui/backend/dbm_init/medium/medium.lock

@@ -36,13 +36,35 @@ es:
     commitId: 7fa118a30171a55154958a376a81ff1830e9c247
     name: dbactuator
     version: 1.0.4
+# ---安装包文件---
+- es:
+    buildPath: /install/es/espack-7.10.2.tar.gz
+    name: espack-7.10.2.tar.gz
+    version: 7.2.0
+    installation: true
+- exporter:
+    buildPath: /install/es/dbm_elasticsearch_exporter.tgz
+    name: dbm_elasticsearch_exporter.tgz
+    version: dbm_elasticsearch_exporter
+    installation: true
 hdfs:
 - actuator:
     buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator
     commitDate: '202406251536'
     commitId: 7fa118a30171a55154958a376a81ff1830e9c247
     name: dbactuator
     version: 1.0.4
+# ---安装包文件---
+- hdfs:
+    buildPath: /install/hdfs/hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz
+    name: hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz
+    version: 2.6.0-cdh5.4.11-tendataV0.2
+    installation: true
+- exporter:
+    buildPath: /install/hdfs/dbm_hdfs_exporter.tgz
+    name: dbm_hdfs_exporter.tgz
+    version: dbm_hdfs_exporter
+    installation: true
 influxdb:
 - actuator:
     buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator
@@ -51,12 +73,29 @@ influxdb:
     name: dbactuator
     version: 1.0.4
 kafka:
+# ---介质文件---
 - actuator:
     buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator
     commitDate: '202406251536'
     commitId: 7fa118a30171a55154958a376a81ff1830e9c247
     name: dbactuator
     version: 1.0.4
+# ---安装包文件---
+- kafka:
+    buildPath: /install/kafka/kafkapack-2.4.0.tar.gz
+    name: kafkapack-2.4.0.tar.gz
+    version: 2.4.0
+    installation: true
+- exporter:
+    buildPath: /install/kafka/dbm_kafka_exporter.tgz
+    name: dbm_kafka_exporter.tgz
+    version: dbm_kafka_exporter
+    installation: true
+- exporter:
+    buildPath: /install/kafka/dbm_kafka_bkpull.tgz
+    name: dbm_kafka_bkpull.tgz
+    version: dbm_kafka_bkpull
+    installation: true
 mongodb:
 - actuator:
     buildPath: /blueking-dbm/dbm-services/mongodb/db-tools/dbactuator/build/mongo-dbactuator
@@ -81,6 +120,7 @@ mongodb:
     name: mongotools.tar.gz
     version: 1.0.0
 mysql:
+# ----介质列表----
 - actuator:
     buildPath: /blueking-dbm/dbm-services/mysql/db-tools/dbactuator/build/dbactuator
     commitDate: '202406272009'
@@ -121,6 +161,37 @@ mysql:
     buildPath: /toolkit/dba-toolkit.tar.gz
     name: dba-toolkit.tar.gz
     version: 1.0.0
+# ----安装包列表----
+- mysql:
+    buildPath: /install/mysql/mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz
+    name: mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz
+    version: MySQL-5.7
+    installation: true
+- mysql:
+    buildPath: /install/mysql/mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz
+    name: mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz
+    version: MySQL-8.0
+    installation: true
+- mysql-proxy:
+    buildPath: /install/mysql/mysql-proxy-0.82.15.tar.gz
+    name: mysql-proxy-0.82.15.tar.gz
+    version: latest
+    installation: true
+- exporter:
+    buildPath: /install/mysql/dbm_tbinlogdumper_exporter.tgz
+    name: dbm_tbinlogdumper_exporter.tgz
+    version: dbm_tbinlogdumper_exporter
+    installation: true
+- exporter:
+    buildPath: /install/mysql/dbm_mysqlproxy_exporter.tgz
+    name: dbm_mysqlproxy_exporter.tgz
+    version: dbm_mysqlproxy_exporter
+    installation: true
+- exporter:
+    buildPath: /install/mysql/dbm_mysqld_exporter.tgz
+    name: dbm_mysqld_exporter.tgz
+    version: dbm_mysqld_exporter
+    installation: true
 pulsar:
 - actuator:
     buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator
@@ -129,6 +200,7 @@ pulsar:
     name: dbactuator
     version: 1.0.4
 redis:
+# ---介质列表---
 - actuator:
     buildPath: /blueking-dbm/dbm-services/redis/db-tools/dbactuator/build/dbactuator_redis
     commitDate: '202406261537'
@@ -149,6 +221,37 @@ redis:
     buildPath: /toolkit/dbtools.tar.gz
     name: dbtools.tar.gz
     version: 1.0.0
+# ---安装包列表---
+- redis:
+    buildPath: /install/redis/redis-5.0.14.tar.gz
+    name: redis-5.0.14.tar.gz
+    version: Redis-5
+    installation: true
+- redis:
+    buildPath: /install/redis/redis-6.2.14.tar.gz
+    name: redis-6.2.14.tar.gz
+    version: Redis-6
+    installation: true
+- tendisplus:
+    buildPath: /install/redis/tendisplus-2.7.3-rocksdb-v8.5.3.tgz
+    name: tendisplus-2.7.3-rocksdb-v8.5.3.tgz
+    version: Tendisplus-2.7
+    installation: true
+- predixy:
+    buildPath: /install/redis/predixy-1.4.3.tar.gz
+    name: predixy-1.4.3.tar.gz
+    version: Predixy-latest
+    installation: true
+- exporter:
+    buildPath: /install/redis/dbm_predixy_exporter.tgz
+    name: dbm_predixy_exporter.tgz
+    version: dbm_predixy_exporter
+    installation: true
+- exporter:
+    buildPath: /install/redis/dbm_redis_exporter.tgz
+    name: dbm_redis_exporter.tgz
+    version: dbm_redis_exporter
+    installation: true
 riak:
 - actuator:
     buildPath: /blueking-dbm/dbm-services/riak/db-tools/dbactuator/build/dbactuator

dbm-ui/backend/flow/consts.py

@@ -255,6 +255,7 @@ class MediumEnum(str, StructuredEnum):
     DbBackup = EnumField("dbbackup", _("dbbackup"))
     DbBackupTXSQL = EnumField("dbbackup-txsql", _("dbbackup-txsql"))
     DBActuator = EnumField("actuator", _("actuator"))
+    Exporter = EnumField("exporter", _("exporter"))
     Latest = EnumField("latest", _("最新版本"))
     Twemproxy = EnumField("twemproxy", _("twemproxy"))
     Predixy = EnumField("predixy", _("predixy"))

helm-charts/bk-dbm/charts/dbm/templates/_helpers.tpl

@@ -159,15 +159,14 @@ environment variables
 {{- end -}}
 
 {{- define "dbm.initContainersWaitForSaaS" -}}
-initContainers:
-  - name: check-saas-api
-    image: {{ include "dbm.migration.k8sWaitFor.image" . }}
-    imagePullPolicy: {{ .Values.image.pullPolicy }}
-    args:
-      - pod
-      - -lapp.kubernetes.io/component={{ include "dbm.saas-api.fullname" .}}
-    resources:
-      {{- toYaml .Values.initJob.resources | nindent 6 }}
+- name: check-saas-api
+  image: {{ include "dbm.migration.k8sWaitFor.image" . }}
+  imagePullPolicy: {{ .Values.image.pullPolicy }}
+  args:
+    - pod
+    - -lapp.kubernetes.io/component={{ include "dbm.saas-api.fullname" .}}
+  resources:
+    {{- toYaml .Values.initJob.resources | nindent 4 }}
 {{- end }}
 
 {{- define "dbm.initContainersWaitForMigrate" -}}
@@ -182,6 +181,17 @@ initContainers:
       {{- toYaml .Values.initJob.resources | nindent 6 }}
 {{- end }}
 
+{{- define "dbm.initContainerMediumInstall" -}}
+{{- $root := first . -}}
+{{- $db_type := last . -}}
+- name: dbm-medium-install-{{ $db_type }}
+  image: "{{ $root.Values.global.imageRegistry | default $root.Values.dbmedium.installImage.registry }}/{{ $root.Values.dbmedium.installImage.repository }}-{{ $db_type }}:{{ $root.Values.dbmedium.installImage.tag | default $root.Chart.AppVersion }}"
+  imagePullPolicy: {{ $root.Values.dbmedium.installImage.pullPolicy }}
+  volumeMounts:
+    - mountPath: /install
+      name: medium-install
+{{- end }}
+
 {{- define "dbm.initMedium" -}}
 {{- $root := first . -}}
 {{- $db_type := last . -}}
@@ -200,6 +210,9 @@ initContainers:
     {{- end }}
   resources:
     {{- toYaml $root.Values.initJob.resources | nindent 4 }}
+  volumeMounts:
+    - mountPath: /install
+      name: medium-install
 {{- end }}
 
 {{- define "dbm.container_env" -}}

helm-charts/bk-dbm/charts/dbm/templates/jobs/medium-init-job.yaml

@@ -17,6 +17,13 @@ spec:
       {{- end }}
       serviceAccountName: {{ include "dbm.serviceAccountName" . }}
       restartPolicy: Never
+      initContainers:
+        {{- include "dbm.initContainersWaitForSaaS" . | nindent 8 }}
+        {{- range $db_type, $enable := .Values.dbmedium.installEnable }}
+          {{- if $enable -}}
+            {{- include "dbm.initContainerMediumInstall" (list $ $db_type) | nindent 8 }}
+          {{- end }}
+        {{- end }}
       containers:
         {{- include "dbm.initMedium" (list . "mysql") | nindent 8 }}
         {{- include "dbm.initMedium" (list . "es") | nindent 8 }}
@@ -29,5 +36,7 @@ spec:
         {{- include "dbm.initMedium" (list . "riak") | nindent 8 }}
         {{- include "dbm.initMedium" (list . "sqlserver") | nindent 8 }}
         {{- include "dbm.initMedium" (list . "cloud") | nindent 8 }}
-      {{- include "dbm.initContainersWaitForSaaS" . | nindent 6 }}
+      volumes:
+        - name: medium-install
+          emptyDir: {}
 {{- end }}

helm-charts/bk-dbm/templates/configmaps/dbm-configmap.yaml

@@ -30,7 +30,7 @@ data:
   BK_SAAS_HOST: "{{ .Values.dbm.envs.bkSaasUrl }}"
   BROKER_URL: "{{ .Values.dbm.envs.brokerUrl }}"
   DBA_APP_BK_BIZ_ID: "{{ .Values.dbm.envs.dbaAppBkBizId }}"
-  CORS_ALLOWED_ORIGINS: "{{ .Values.dbm.envs.corsAllowedOrigins | default (list) }}"
+  CORS_ALLOWED_ORIGINS: "{{ .Values.dbm.envs.corsAllowedOrigins | default "" }}"
   # REDIS
   REDIS_HOST: "{{ .Values.externalRedis.host }}"
   REDIS_PORT: "{{ .Values.externalRedis.port }}"

helm-charts/bk-dbm/values.yaml

@@ -174,9 +174,24 @@ dbm:
       repository: "build/blueking/dbmedium"
       pullPolicy: IfNotPresent
       tag: ""
+    installImage:
+      registry: "mirrors.tencent.com"
+      repository: "build/blueking/dbmedium-install"
+      pullPolicy: IfNotPresent
+      tag: ""
+    installEnable:
+      mysql: false
+      es: false
+      kafka: false
+      hdfs: false
+      pulsar: false
+      influxdb: false
+      redis: false
+      mongodb: false
+      riak: false
+      sqlserver: false
 
 # bk-dbconfig values
-
 dbconfig:
   extraEnvVarsCM: bk-dbconfig-db-env
   enabled: true