diff --git a/dbm-ui/backend/dbm_init/medium/handlers.py b/dbm-ui/backend/dbm_init/medium/handlers.py index 7b69b3d4fe..d43edafb2e 100644 --- a/dbm-ui/backend/dbm_init/medium/handlers.py +++ b/dbm-ui/backend/dbm_init/medium/handlers.py @@ -243,7 +243,7 @@ def add_version(version): lock_file.write(yaml.safe_dump(lock_info)) @classmethod - def build_medium(cls, bkrepo_tmp_dir): + def build_medium(cls, bkrepo_tmp_dir, installation=False): # 加载lock文件,获取介质的版本信息 medium_lock_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "medium.lock") with open(medium_lock_path, "r") as lock_file: @@ -252,6 +252,9 @@ def build_medium(cls, bkrepo_tmp_dir): for db_type, mediums in lock_info.items(): for medium in mediums: for medium_type, medium_info in medium.items(): + # 如果介质和安装模式不匹配,忽略 + if medium_info.get("installation", False) != installation: + continue # 将编译好的介质复制到指定目录 target_medium_path = f"{bkrepo_tmp_dir}/{db_type}/{medium_type}/{medium_info['version']}" result = subprocess.run( diff --git a/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-ES b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-ES new file mode 100644 index 0000000000..b8c90ca529 --- /dev/null +++ b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-ES @@ -0,0 +1,14 @@ +## busybox,集合了常用的linux工具箱 +FROM busybox:latest + +ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download +ENV VERSION=v1.0.0 + +## 获取二进制介质,将静态介质存放到/data目录 +RUN set -ex && \ + mkdir /data && cd /data && \ + wget -qO espack-7.10.2.tar.gz ${DOWNLOAD_URL}/${VERSION}/espack-7.10.2.tar.gz --no-check-certificate && \ + wget -qO dbm_elasticsearch_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_elasticsearch_exporter.tgz --no-check-certificate + +## 将install的文件移动到/install/xxx,这样通过挂载/install可以共享文件 +ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/es && mv /data/* /install/es/"] diff --git a/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-HDFS b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-HDFS new file mode 100644 index 0000000000..32e9a0615c --- /dev/null +++ b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-HDFS @@ -0,0 +1,14 @@ +## busybox,集合了常用的linux工具箱 +FROM busybox:latest + +ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download +ENV VERSION=v1.0.0 + +## 获取二进制介质,将静态介质存放到/data目录 +RUN set -ex && \ + mkdir /data && cd /data && \ + wget -qO hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz ${DOWNLOAD_URL}/${VERSION}/hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz --no-check-certificate && \ + wget -qO dbm_hdfs_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_hdfs_exporter.tgz --no-check-certificate + +## 将install的文件移动到/install/xxx,这样通过挂载/install可以共享文件 +ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/hdfs && mv /data/* /install/hdfs/"] diff --git a/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Kafka b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Kafka new file mode 100644 index 0000000000..71759f72a0 --- /dev/null +++ b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Kafka @@ -0,0 +1,15 @@ +## busybox,集合了常用的linux工具箱 +FROM busybox:latest + +ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download +ENV VERSION=v1.0.0 + +## 获取二进制介质,将静态介质存放到/data目录 +RUN set -ex && \ + mkdir /data && cd /data && \ + wget -qO kafkapack-2.4.0.tar.gz ${DOWNLOAD_URL}/${VERSION}/kafkapack-2.4.0.tar.gz --no-check-certificate && \ + wget -qO dbm_kafka_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_kafka_exporter.tgz --no-check-certificate && \ + wget -qO dbm_kafka_bkpull.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_kafka_bkpull.tgz --no-check-certificate + +## 将install的文件移动到/install/xxx,这样通过挂载/install可以共享文件 +ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/kafka && mv /data/* /install/kafka/"] diff --git a/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-MySQL b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-MySQL new file mode 100644 index 0000000000..0e6c96c5c8 --- /dev/null +++ b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-MySQL @@ -0,0 +1,19 @@ +## busybox,集合了常用的linux工具箱 +FROM busybox:latest + +ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download +ENV VERSION=v1.0.0 + +## 获取二进制介质,将静态介质存放到/data目录 +RUN set -ex && \ + mkdir /data && cd /data && \ + wget -qO dbm_mysqld_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_mysqld_exporter.tgz --no-check-certificate && \ + wget -qO dbm_mysqlproxy_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_mysqlproxy_exporter.tgz --no-check-certificate && \ + wget -qO dbm_tbinlogdumper_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_tbinlogdumper_exporter.tgz --no-check-certificate && \ + wget -qO mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz ${DOWNLOAD_URL}/${VERSION}/mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz --no-check-certificate && \ + wget -qO mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz ${DOWNLOAD_URL}/${VERSION}/mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz --no-check-certificate && \ + wget -qO mysql-proxy-0.82.15.tar.gz ${DOWNLOAD_URL}/${VERSION}/mysql-proxy-0.82.15.tar.gz --no-check-certificate + + +## 将install的文件移动到/mysql-install,这样通过挂载/mysql-install可以共享文件 +ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/mysql && mv /data/* /install/mysql/"] diff --git a/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Redis b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Redis new file mode 100644 index 0000000000..b68c1a6251 --- /dev/null +++ b/dbm-ui/backend/dbm_init/medium/installs/Dockerfile-Redis @@ -0,0 +1,18 @@ +## busybox,集合了常用的linux工具箱 +FROM busybox:latest + +ENV DOWNLOAD_URL=https://github.com/TencentBlueKing/blueking-dbm/releases/download +ENV VERSION=v1.0.0 + +## 获取二进制介质,将静态介质存放到/data目录 +RUN set -ex && \ + mkdir /data && cd /data && \ + wget -qO dbm_predixy_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_predixy_exporter.tgz --no-check-certificate && \ + wget -qO dbm_redis_exporter.tgz ${DOWNLOAD_URL}/${VERSION}/dbm_redis_exporter.tgz --no-check-certificate && \ + wget -qO predixy-1.4.3.tar.gz ${DOWNLOAD_URL}/${VERSION}/predixy-1.4.3.tar.gz --no-check-certificate && \ + wget -qO redis-5.0.14.tar.gz ${DOWNLOAD_URL}/${VERSION}/redis-5.0.14.tar.gz --no-check-certificate && \ + wget -qO redis-6.2.14.tar.gz ${DOWNLOAD_URL}/${VERSION}/redis-6.2.14.tar.gz --no-check-certificate && \ + wget -qO tendisplus-2.7.3-rocksdb-v8.5.3.tgz ${DOWNLOAD_URL}/${VERSION}/tendisplus-2.7.3-rocksdb-v8.5.3.tgz --no-check-certificate + +## 将install的文件移动到/install/xxx,这样通过挂载/install可以共享文件 +ENTRYPOINT ["/bin/sh", "-c", "mkdir -p /install/redis && mv /data/* /install/redis/"] diff --git a/dbm-ui/backend/dbm_init/medium/main.py b/dbm-ui/backend/dbm_init/medium/main.py index 9139cac550..4ecd3660eb 100644 --- a/dbm-ui/backend/dbm_init/medium/main.py +++ b/dbm-ui/backend/dbm_init/medium/main.py @@ -36,6 +36,7 @@ elif args.type == "build": MediumHandler.build_medium(bkrepo_tmp_dir=path) elif args.type == "upload": + MediumHandler.build_medium(bkrepo_tmp_dir=path, installation=True) MediumHandler().upload_medium(path=args.db, bkrepo_tmp_dir=path) elif args.type == "sync": MediumHandler().sync_from_bkrepo(db_type=args.db) diff --git a/dbm-ui/backend/dbm_init/medium/medium.lock b/dbm-ui/backend/dbm_init/medium/medium.lock index a2d09216fd..18f89d34e5 100644 --- a/dbm-ui/backend/dbm_init/medium/medium.lock +++ b/dbm-ui/backend/dbm_init/medium/medium.lock @@ -36,6 +36,17 @@ es: commitId: 7fa118a30171a55154958a376a81ff1830e9c247 name: dbactuator version: 1.0.4 +# ---安装包文件--- +- es: + buildPath: /install/es/espack-7.10.2.tar.gz + name: espack-7.10.2.tar.gz + version: 7.2.0 + installation: true +- exporter: + buildPath: /install/es/dbm_elasticsearch_exporter.tgz + name: dbm_elasticsearch_exporter.tgz + version: dbm_elasticsearch_exporter + installation: true hdfs: - actuator: buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator @@ -43,6 +54,17 @@ hdfs: commitId: 7fa118a30171a55154958a376a81ff1830e9c247 name: dbactuator version: 1.0.4 +# ---安装包文件--- +- hdfs: + buildPath: /install/hdfs/hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz + name: hdfspack-2.6.0-cdh5.4.11-tendataV0.2.tar.gz + version: 2.6.0-cdh5.4.11-tendataV0.2 + installation: true +- exporter: + buildPath: /install/hdfs/dbm_hdfs_exporter.tgz + name: dbm_hdfs_exporter.tgz + version: dbm_hdfs_exporter + installation: true influxdb: - actuator: buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator @@ -51,12 +73,29 @@ influxdb: name: dbactuator version: 1.0.4 kafka: +# ---介质文件--- - actuator: buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator commitDate: '202406251536' commitId: 7fa118a30171a55154958a376a81ff1830e9c247 name: dbactuator version: 1.0.4 +# ---安装包文件--- +- kafka: + buildPath: /install/kafka/kafkapack-2.4.0.tar.gz + name: kafkapack-2.4.0.tar.gz + version: 2.4.0 + installation: true +- exporter: + buildPath: /install/kafka/dbm_kafka_exporter.tgz + name: dbm_kafka_exporter.tgz + version: dbm_kafka_exporter + installation: true +- exporter: + buildPath: /install/kafka/dbm_kafka_bkpull.tgz + name: dbm_kafka_bkpull.tgz + version: dbm_kafka_bkpull + installation: true mongodb: - actuator: buildPath: /blueking-dbm/dbm-services/mongodb/db-tools/dbactuator/build/mongo-dbactuator @@ -81,6 +120,7 @@ mongodb: name: mongotools.tar.gz version: 1.0.0 mysql: +# ----介质列表---- - actuator: buildPath: /blueking-dbm/dbm-services/mysql/db-tools/dbactuator/build/dbactuator commitDate: '202406272009' @@ -121,6 +161,37 @@ mysql: buildPath: /toolkit/dba-toolkit.tar.gz name: dba-toolkit.tar.gz version: 1.0.0 +# ----安装包列表---- +- mysql: + buildPath: /install/mysql/mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz + name: mysql-5.7.20-linux-x86_64-tmysql-3.3-gcs.tar.gz + version: MySQL-5.7 + installation: true +- mysql: + buildPath: /install/mysql/mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz + name: mysql-8.0.32-linux-glibc2.12-x86_64.tar.xz + version: MySQL-8.0 + installation: true +- mysql-proxy: + buildPath: /install/mysql/mysql-proxy-0.82.15.tar.gz + name: mysql-proxy-0.82.15.tar.gz + version: latest + installation: true +- exporter: + buildPath: /install/mysql/dbm_tbinlogdumper_exporter.tgz + name: dbm_tbinlogdumper_exporter.tgz + version: dbm_tbinlogdumper_exporter + installation: true +- exporter: + buildPath: /install/mysql/dbm_mysqlproxy_exporter.tgz + name: dbm_mysqlproxy_exporter.tgz + version: dbm_mysqlproxy_exporter + installation: true +- exporter: + buildPath: /install/mysql/dbm_mysqld_exporter.tgz + name: dbm_mysqld_exporter.tgz + version: dbm_mysqld_exporter + installation: true pulsar: - actuator: buildPath: /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator/build/dbactuator @@ -129,6 +200,7 @@ pulsar: name: dbactuator version: 1.0.4 redis: +# ---介质列表--- - actuator: buildPath: /blueking-dbm/dbm-services/redis/db-tools/dbactuator/build/dbactuator_redis commitDate: '202406261537' @@ -149,6 +221,37 @@ redis: buildPath: /toolkit/dbtools.tar.gz name: dbtools.tar.gz version: 1.0.0 +# ---安装包列表--- +- redis: + buildPath: /install/redis/redis-5.0.14.tar.gz + name: redis-5.0.14.tar.gz + version: Redis-5 + installation: true +- redis: + buildPath: /install/redis/redis-6.2.14.tar.gz + name: redis-6.2.14.tar.gz + version: Redis-6 + installation: true +- tendisplus: + buildPath: /install/redis/tendisplus-2.7.3-rocksdb-v8.5.3.tgz + name: tendisplus-2.7.3-rocksdb-v8.5.3.tgz + version: Tendisplus-2.7 + installation: true +- predixy: + buildPath: /install/redis/predixy-1.4.3.tar.gz + name: predixy-1.4.3.tar.gz + version: Predixy-latest + installation: true +- exporter: + buildPath: /install/redis/dbm_predixy_exporter.tgz + name: dbm_predixy_exporter.tgz + version: dbm_predixy_exporter + installation: true +- exporter: + buildPath: /install/redis/dbm_redis_exporter.tgz + name: dbm_redis_exporter.tgz + version: dbm_redis_exporter + installation: true riak: - actuator: buildPath: /blueking-dbm/dbm-services/riak/db-tools/dbactuator/build/dbactuator diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index b7ff3a440f..13c9c35370 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -255,6 +255,7 @@ class MediumEnum(str, StructuredEnum): DbBackup = EnumField("dbbackup", _("dbbackup")) DbBackupTXSQL = EnumField("dbbackup-txsql", _("dbbackup-txsql")) DBActuator = EnumField("actuator", _("actuator")) + Exporter = EnumField("exporter", _("exporter")) Latest = EnumField("latest", _("最新版本")) Twemproxy = EnumField("twemproxy", _("twemproxy")) Predixy = EnumField("predixy", _("predixy")) diff --git a/helm-charts/bk-dbm/charts/dbm/templates/_helpers.tpl b/helm-charts/bk-dbm/charts/dbm/templates/_helpers.tpl index 9d583b8ee8..9b7f9b7179 100644 --- a/helm-charts/bk-dbm/charts/dbm/templates/_helpers.tpl +++ b/helm-charts/bk-dbm/charts/dbm/templates/_helpers.tpl @@ -159,15 +159,14 @@ environment variables {{- end -}} {{- define "dbm.initContainersWaitForSaaS" -}} -initContainers: - - name: check-saas-api - image: {{ include "dbm.migration.k8sWaitFor.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - pod - - -lapp.kubernetes.io/component={{ include "dbm.saas-api.fullname" .}} - resources: - {{- toYaml .Values.initJob.resources | nindent 6 }} +- name: check-saas-api + image: {{ include "dbm.migration.k8sWaitFor.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - pod + - -lapp.kubernetes.io/component={{ include "dbm.saas-api.fullname" .}} + resources: + {{- toYaml .Values.initJob.resources | nindent 4 }} {{- end }} {{- define "dbm.initContainersWaitForMigrate" -}} @@ -182,6 +181,17 @@ initContainers: {{- toYaml .Values.initJob.resources | nindent 6 }} {{- end }} +{{- define "dbm.initContainerMediumInstall" -}} +{{- $root := first . -}} +{{- $db_type := last . -}} +- name: dbm-medium-install-{{ $db_type }} + image: "{{ $root.Values.global.imageRegistry | default $root.Values.dbmedium.installImage.registry }}/{{ $root.Values.dbmedium.installImage.repository }}-{{ $db_type }}:{{ $root.Values.dbmedium.installImage.tag | default $root.Chart.AppVersion }}" + imagePullPolicy: {{ $root.Values.dbmedium.installImage.pullPolicy }} + volumeMounts: + - mountPath: /install + name: medium-install +{{- end }} + {{- define "dbm.initMedium" -}} {{- $root := first . -}} {{- $db_type := last . -}} @@ -200,6 +210,9 @@ initContainers: {{- end }} resources: {{- toYaml $root.Values.initJob.resources | nindent 4 }} + volumeMounts: + - mountPath: /install + name: medium-install {{- end }} {{- define "dbm.container_env" -}} diff --git a/helm-charts/bk-dbm/charts/dbm/templates/jobs/medium-init-job.yaml b/helm-charts/bk-dbm/charts/dbm/templates/jobs/medium-init-job.yaml index acb0e20a7c..bae5ba725f 100644 --- a/helm-charts/bk-dbm/charts/dbm/templates/jobs/medium-init-job.yaml +++ b/helm-charts/bk-dbm/charts/dbm/templates/jobs/medium-init-job.yaml @@ -17,6 +17,13 @@ spec: {{- end }} serviceAccountName: {{ include "dbm.serviceAccountName" . }} restartPolicy: Never + initContainers: + {{- include "dbm.initContainersWaitForSaaS" . | nindent 8 }} + {{- range $db_type, $enable := .Values.dbmedium.installEnable }} + {{- if $enable -}} + {{- include "dbm.initContainerMediumInstall" (list $ $db_type) | nindent 8 }} + {{- end }} + {{- end }} containers: {{- include "dbm.initMedium" (list . "mysql") | nindent 8 }} {{- include "dbm.initMedium" (list . "es") | nindent 8 }} @@ -29,5 +36,7 @@ spec: {{- include "dbm.initMedium" (list . "riak") | nindent 8 }} {{- include "dbm.initMedium" (list . "sqlserver") | nindent 8 }} {{- include "dbm.initMedium" (list . "cloud") | nindent 8 }} - {{- include "dbm.initContainersWaitForSaaS" . | nindent 6 }} + volumes: + - name: medium-install + emptyDir: {} {{- end }} diff --git a/helm-charts/bk-dbm/templates/configmaps/dbm-configmap.yaml b/helm-charts/bk-dbm/templates/configmaps/dbm-configmap.yaml index 3383af6fc8..4978b389e9 100644 --- a/helm-charts/bk-dbm/templates/configmaps/dbm-configmap.yaml +++ b/helm-charts/bk-dbm/templates/configmaps/dbm-configmap.yaml @@ -30,7 +30,7 @@ data: BK_SAAS_HOST: "{{ .Values.dbm.envs.bkSaasUrl }}" BROKER_URL: "{{ .Values.dbm.envs.brokerUrl }}" DBA_APP_BK_BIZ_ID: "{{ .Values.dbm.envs.dbaAppBkBizId }}" - CORS_ALLOWED_ORIGINS: "{{ .Values.dbm.envs.corsAllowedOrigins | default (list) }}" + CORS_ALLOWED_ORIGINS: "{{ .Values.dbm.envs.corsAllowedOrigins | default "" }}" # REDIS REDIS_HOST: "{{ .Values.externalRedis.host }}" REDIS_PORT: "{{ .Values.externalRedis.port }}" diff --git a/helm-charts/bk-dbm/values.yaml b/helm-charts/bk-dbm/values.yaml index 018d4cf89e..4df5861599 100644 --- a/helm-charts/bk-dbm/values.yaml +++ b/helm-charts/bk-dbm/values.yaml @@ -174,9 +174,24 @@ dbm: repository: "build/blueking/dbmedium" pullPolicy: IfNotPresent tag: "" + installImage: + registry: "mirrors.tencent.com" + repository: "build/blueking/dbmedium-install" + pullPolicy: IfNotPresent + tag: "" + installEnable: + mysql: false + es: false + kafka: false + hdfs: false + pulsar: false + influxdb: false + redis: false + mongodb: false + riak: false + sqlserver: false # bk-dbconfig values - dbconfig: extraEnvVarsCM: bk-dbconfig-db-env enabled: true