diff --git a/dbm-services/bigdata/db-tools/dbactuator/Makefile b/dbm-services/bigdata/db-tools/dbactuator/Makefile index 0597f378bb..f2740f560f 100644 --- a/dbm-services/bigdata/db-tools/dbactuator/Makefile +++ b/dbm-services/bigdata/db-tools/dbactuator/Makefile @@ -1,9 +1,10 @@ SHELL := /bin/bash BASE_DIR = $(shell pwd) VERSION = 0.0.1 +GITHASH = "" APPNAME = dbactuator GOOS ?= linux -BUILD_FLAG = "-X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=`git rev-parse HEAD` " +BUILD_FLAG = "-X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=${GITHASH} " .PHONY: all build clean diff --git a/dbm-services/mysql/db-tools/dbactuator/Makefile b/dbm-services/mysql/db-tools/dbactuator/Makefile index f5e0ed8904..346cf0a727 100644 --- a/dbm-services/mysql/db-tools/dbactuator/Makefile +++ b/dbm-services/mysql/db-tools/dbactuator/Makefile @@ -1,11 +1,12 @@ SHELL := /bin/bash BASE_DIR = $(shell pwd) VERSION = 0.0.1 +GITHASH = "" APPNAME = dbactuator GOOS ?= linux -BUILD_FLAG = " -X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=`git rev-parse HEAD` " -BUILD_EXTERNAL_FLAG = " -X main.external=ON -X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=`git rev-parse HEAD` " -BUILD_MINI_FLAG = " -s -w -X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=`git rev-parse HEAD` " +BUILD_FLAG = " -X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=${GITHASH} " +BUILD_EXTERNAL_FLAG = " -X main.external=ON -X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=${GITHASH} " +BUILD_MINI_FLAG = " -s -w -X main.version=${VERSION} -X main.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X main.githash=${GITHASH} " .PHONY: all build clean diff --git a/dbm-services/mysql/db-tools/mysql-crond/Makefile b/dbm-services/mysql/db-tools/mysql-crond/Makefile index a9d77f72f6..2893b55541 100644 --- a/dbm-services/mysql/db-tools/mysql-crond/Makefile +++ b/dbm-services/mysql/db-tools/mysql-crond/Makefile @@ -1,10 +1,11 @@ PROJ="mysql-crond" MODULE="dbm-services/mysql/db-tools/mysql-crond" VERSION = $(error please set VERSION flag) +GITHASH = "" #VERSION=$(shell date +'%y%m%d.%H.%M') PKG=${PROJ}.tar.gz OUTPUT_DIR=build -RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=`git rev-parse HEAD` " +RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=${GITHASH} " DEV_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash="" " .PHONY: release-bin diff --git a/dbm-services/mysql/db-tools/mysql-monitor/Makefile b/dbm-services/mysql/db-tools/mysql-monitor/Makefile index 1c1298776d..93824a5211 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/Makefile +++ b/dbm-services/mysql/db-tools/mysql-monitor/Makefile @@ -1,9 +1,10 @@ PROJ="mysql-monitor" MODULE="dbm-services/mysql/db-tools/mysql-monitor" VERSION = $(error please set VERSION flag) +GITHASH = "" PKG = ${PROJ}.tar.gz OUTPUT_DIR = build -RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=`git rev-parse HEAD` " +RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=${GITHASH} " DEV_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash="" " diff --git a/dbm-services/mysql/db-tools/mysql-rotatebinlog/Makefile b/dbm-services/mysql/db-tools/mysql-rotatebinlog/Makefile index 5bcb0ea909..22053541b0 100644 --- a/dbm-services/mysql/db-tools/mysql-rotatebinlog/Makefile +++ b/dbm-services/mysql/db-tools/mysql-rotatebinlog/Makefile @@ -2,9 +2,10 @@ PROJ="mysql-rotatebinlog" PROJ_BIN="rotatebinlog" MODULE="dbm-services/mysql/db-tools/mysql-rotatebinlog" VERSION = $(error please set VERSION flag) +GITHASH = "" PROJ_PKG = ${PROJ}.tar.gz OUTPUT_DIR = build -RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=`git rev-parse HEAD` " +RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=${GITHASH} " BETA_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash="" " BASE_DIR = $(shell pwd) diff --git a/dbm-services/mysql/db-tools/mysql-table-checksum/Makefile b/dbm-services/mysql/db-tools/mysql-table-checksum/Makefile index 1de5074a03..a05d489038 100644 --- a/dbm-services/mysql/db-tools/mysql-table-checksum/Makefile +++ b/dbm-services/mysql/db-tools/mysql-table-checksum/Makefile @@ -1,9 +1,10 @@ PROJ="mysql-table-checksum" MODULE="dbm-services/mysql/db-tools/mysql-table-checksum" VERSION = $(error please set VERSION flag) +GITHASH = "" PKG="mysql-checksum.tar.gz" # 这是个不太好改的错误了 OUTPUT_DIR=build -RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=`git rev-parse HEAD` " +RELEASE_BUILD_FLAG = "-X ${MODULE}/cmd.version=${VERSION} -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash=${GITHASH} " DEV_BUILD_FLAG = "-X ${MODULE}/cmd.version="develop" -X ${MODULE}/cmd.buildStamp=`date -u '+%Y-%m-%d_%I:%M:%S%p'` -X ${MODULE}/cmd.gitHash="" " .PHONY: release-bin diff --git a/dbm-services/redis/db-tools/dbactuator/Makefile b/dbm-services/redis/db-tools/dbactuator/Makefile index 568a1c226a..daba8db33b 100644 --- a/dbm-services/redis/db-tools/dbactuator/Makefile +++ b/dbm-services/redis/db-tools/dbactuator/Makefile @@ -2,10 +2,11 @@ SHELL := /bin/bash BASE_DIR = $(shell pwd) SRV_NAME=dbactuator_redis VERSION = 0.0.1 +GITHASH = "" PACKAGE_PATH="dbm-services/redis/db-tools/dbactuator/cmd" -BUILD_FLAG = " -X ${PACKAGE_PATH}.version=${VERSION} -X ${PACKAGE_PATH}.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%z'` -X ${PACKAGE_PATH}.githash=`git rev-parse HEAD` " +BUILD_FLAG = " -X ${PACKAGE_PATH}.version=${VERSION} -X ${PACKAGE_PATH}.buildstamp=`date -u '+%Y-%m-%d_%I:%M:%S%z'` -X ${PACKAGE_PATH}.githash=${GITHASH} " build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -gcflags="all=-trimpath=${PWD}" -asmflags="all=-trimpath=${PWD}" -ldflags ${BUILD_FLAG} -o ${BASE_DIR}/build/$(SRV_NAME) -v main.go diff --git a/dbm-ui/backend/db_meta/api/cluster/riak/detail.py b/dbm-ui/backend/db_meta/api/cluster/riak/detail.py new file mode 100644 index 0000000000..7094c319b9 --- /dev/null +++ b/dbm-ui/backend/db_meta/api/cluster/riak/detail.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import gettext as _ + +from backend.db_meta.api.cluster.base.graph import Graphic, LineLabel +from backend.db_meta.enums import InstanceRole +from backend.db_meta.models import Cluster + + +def scan_cluster(cluster: Cluster) -> Graphic: + """ + 绘制kafka的拓扑结构图 + """ + graph = Graphic(node_id=Graphic.generate_graphic_id(cluster)) + graph.add_instance_nodes(cluster=cluster, roles=InstanceRole.RIAK_NODE, group_name=_("Riak 节点")) + return graph diff --git a/dbm-ui/backend/db_package/views.py b/dbm-ui/backend/db_package/views.py index 00e8850307..0f51b1e221 100644 --- a/dbm-ui/backend/db_package/views.py +++ b/dbm-ui/backend/db_package/views.py @@ -53,7 +53,7 @@ def create(self, request, *args, **kwargs): @action(methods=["POST"], detail=False, serializer_class=UpdateOrCreateSerializer) def update_or_create(self, request, *args, **kwargs): data = self.params_validate(self.get_serializer_class()) - Package.objects.update_or_create(md5=data["md5"], defaults=data) + Package.objects.update_or_create(md5=data["md5"], db_type=data["db_type"], defaults=data) return Response() @common_swagger_auto_schema( diff --git a/dbm-ui/backend/db_services/bigdata/riak/__init__.py b/dbm-ui/backend/db_services/bigdata/riak/__init__.py new file mode 100644 index 0000000000..aa5085c628 --- /dev/null +++ b/dbm-ui/backend/db_services/bigdata/riak/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/dbm-ui/backend/db_services/bigdata/riak/constants.py b/dbm-ui/backend/db_services/bigdata/riak/constants.py new file mode 100644 index 0000000000..d55ccb2c27 --- /dev/null +++ b/dbm-ui/backend/db_services/bigdata/riak/constants.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +RESOURCE_TAG = "db_services/resources/riak" diff --git a/dbm-ui/backend/db_services/bigdata/riak/query.py b/dbm-ui/backend/db_services/bigdata/riak/query.py new file mode 100644 index 0000000000..872e764646 --- /dev/null +++ b/dbm-ui/backend/db_services/bigdata/riak/query.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ + +from backend.db_meta.api.cluster.riak.detail import scan_cluster +from backend.db_meta.enums import InstanceRole +from backend.db_meta.enums.cluster_type import ClusterType +from backend.db_meta.models import Cluster +from backend.db_services.bigdata.resources.query import BigDataBaseListRetrieveResource + + +class RiakListRetrieveResource(BigDataBaseListRetrieveResource): + + cluster_types = [ClusterType.Riak] + instance_roles = [InstanceRole.RIAK_NODE] + fields = [ + *BigDataBaseListRetrieveResource.fields, + {"name": _("riak节点"), "key": "riak_node"}, + ] + + @classmethod + def get_topo_graph(cls, bk_biz_id: int, cluster_id: int) -> dict: + cluster = Cluster.objects.get(bk_biz_id=bk_biz_id, id=cluster_id) + graph = scan_cluster(cluster).to_dict() + return graph diff --git a/dbm-ui/backend/db_services/bigdata/riak/urls.py b/dbm-ui/backend/db_services/bigdata/riak/urls.py new file mode 100644 index 0000000000..c8c4bc35d0 --- /dev/null +++ b/dbm-ui/backend/db_services/bigdata/riak/urls.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from rest_framework.routers import DefaultRouter + +from backend.db_services.bigdata.riak.views import RiakClusterViewSet + +router = DefaultRouter(trailing_slash=True) +router.register(r"riak_resources", RiakClusterViewSet, basename="riak_resources") + +urlpatterns = router.urls diff --git a/dbm-ui/backend/db_services/bigdata/riak/views.py b/dbm-ui/backend/db_services/bigdata/riak/views.py new file mode 100644 index 0000000000..4cd9740398 --- /dev/null +++ b/dbm-ui/backend/db_services/bigdata/riak/views.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.decorators import method_decorator +from django.utils.translation import ugettext as _ +from rest_framework import status + +from backend.bk_web.swagger import common_swagger_auto_schema +from backend.db_services.bigdata.kafka import constants +from backend.db_services.bigdata.resources import yasg_slz +from backend.db_services.bigdata.resources.views import ResourceViewSet +from backend.db_services.bigdata.riak.query import RiakListRetrieveResource +from backend.db_services.dbbase.resources import serializers + + +@method_decorator( + name="list", + decorator=common_swagger_auto_schema( + operation_summary=_("获取集群列表"), + query_serializer=serializers.ListResourceSLZ(), + responses={status.HTTP_200_OK: yasg_slz.PaginatedResourceSLZ()}, + tags=[constants.RESOURCE_TAG], + ), +) +@method_decorator( + name="retrieve", + decorator=common_swagger_auto_schema( + operation_summary=_("获取集群详情"), + responses={status.HTTP_200_OK: yasg_slz.ResourceSLZ()}, + tags=[constants.RESOURCE_TAG], + ), +) +@method_decorator( + name="list_instances", + decorator=common_swagger_auto_schema( + operation_summary=_("获取实例列表"), + query_serializer=serializers.ListInstancesSerializer(), + responses={status.HTTP_200_OK: yasg_slz.PaginatedResourceSLZ()}, + tags=[constants.RESOURCE_TAG], + ), +) +@method_decorator( + name="retrieve_instance", + decorator=common_swagger_auto_schema( + operation_summary=_("获取实例详情"), + query_serializer=serializers.RetrieveInstancesSerializer(), + tags=[constants.RESOURCE_TAG], + ), +) +@method_decorator( + name="get_table_fields", + decorator=common_swagger_auto_schema( + operation_summary=_("获取查询返回字段"), + responses={status.HTTP_200_OK: yasg_slz.ResourceFieldSLZ()}, + tags=[constants.RESOURCE_TAG], + ), +) +@method_decorator( + name="get_topo_graph", + decorator=common_swagger_auto_schema( + operation_summary=_("获取集群拓扑"), + responses={status.HTTP_200_OK: yasg_slz.ResourceTopoGraphSLZ()}, + tags=[constants.RESOURCE_TAG], + ), +) +class RiakClusterViewSet(ResourceViewSet): + query_class = RiakListRetrieveResource + query_serializer_class = serializers.ListResourceSLZ diff --git a/dbm-ui/backend/db_services/bigdata/urls.py b/dbm-ui/backend/db_services/bigdata/urls.py index 98b131e4be..2c5301690b 100644 --- a/dbm-ui/backend/db_services/bigdata/urls.py +++ b/dbm-ui/backend/db_services/bigdata/urls.py @@ -17,4 +17,5 @@ path("bizs//kafka/", include("backend.db_services.bigdata.kafka.urls")), path("bizs//pulsar/", include("backend.db_services.bigdata.pulsar.urls")), path("bizs//influxdb/", include("backend.db_services.bigdata.influxdb.urls")), + path("bizs//riak/", include("backend.db_services.bigdata.riak.urls")), ] diff --git a/dbm-ui/backend/dbm_init/medium/Dockerfile b/dbm-ui/backend/dbm_init/medium/Dockerfile index 2f05d9457a..d8068aaa8c 100644 --- a/dbm-ui/backend/dbm_init/medium/Dockerfile +++ b/dbm-ui/backend/dbm_init/medium/Dockerfile @@ -1,7 +1,8 @@ FROM golang:1.21 as go-builder -## 标准化时区 -RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ +## 标准化时区,安装依赖 +RUN set -ex && \ + ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo "Asia/Shanghai" > /etc/timezone ## 安装go import工具 @@ -16,40 +17,6 @@ ADD ./dbm-services ./dbm-services RUN set -ex && \ goimports -w . && find . -name go.mod -execdir go mod tidy \; -FROM go-mod-builder as mysql-medium-builder - -## 构建mysql介质 -RUN set -ex && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/dbactuator && make VERSION= -j4 && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-dbbackup && sh build.sh -t txsql && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-table-checksum && make release-bin VERSION= -j4 && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-crond && make release-bin VERSION= -j4 && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-rotatebinlog && make release VERSION= -j4 && \ - cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-monitor && make release-bin VERSION= -j4 - - -FROM go-mod-builder as redis-medium-builder - -## 构建redis介质 -RUN set -ex && \ - cd /blueking-dbm/dbm-services/redis/db-tools/dbactuator && make VERSION= -j4 && \ - cd /blueking-dbm/dbm-services/redis/db-tools/dbmon && bash package.sh - - -FROM go-mod-builder as bigdata-medium-builder - -## 构建大数据介质 -RUN set -ex && \ - cd /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator && make VERSION= -j4 - - -FROM go-builder as medium-builder - -WORKDIR /blueking-dbm -COPY --from=mysql-medium-builder /blueking-dbm/dbm-services/mysql ./dbm-services/mysql -COPY --from=redis-medium-builder /blueking-dbm/dbm-services/redis ./dbm-services/redis -COPY --from=bigdata-medium-builder /blueking-dbm/dbm-services/bigdata ./dbm-services/bigdata - FROM python:3.6.12-slim-buster AS base @@ -76,7 +43,7 @@ RUN set -ex && \ RUN set -ex && mkdir ~/.pip && printf '[global]\nindex-url = https://mirrors.tencent.com/pypi/simple/' > ~/.pip/pip.conf -FROM base AS builder +FROM base AS python-builder WORKDIR / @@ -108,10 +75,7 @@ WORKDIR /app USER root ## 拷贝虚拟环境 -COPY --from=builder /venv /venv - -## 拷贝第一阶段的文件,构建制品 -COPY --from=medium-builder /blueking-dbm /blueking-dbm +COPY --from=python-builder /venv /venv ## 拷贝脚本文件 ADD ./dbm-ui/backend/dbm_init/medium/ ./ @@ -124,25 +88,91 @@ ARG GITHUB_TOKEN='' ARG GITHUB_USERNAME='' ARG GITHUB_USER_EMAIL='' -## 将相关制品移动到指定目录,提交medicum.lock文件 +##更新并提交medicum.lock文件 RUN set -ex && \ # 克隆线上的dbm仓库 git config --global user.email ${GITHUB_USER_EMAIL} && \ git config --global user.name ${GITHUB_USERNAME} && \ git clone https://${GITHUB_TOKEN}@github.com/TencentBlueKing/blueking-dbm.git && \ - # 注意:这里需要将.git文件复制给前面构建文件夹,用作判断commit是否需要更新 - cp -r blueking-dbm/.git /blueking-dbm && \ - python main.py --type build && \ - rm -rf /blueking-dbm && \ + # 将clone的blueking-dbm转移到跟目录操作 + mv blueking-dbm/ / && cd /blueking-dbm && git checkout -f ${MEDIUM_BUILDER_BRANCH} && \ + # 更新lock文件 + cd /app && python main.py --type update_lock && \ # 将更新后的medium.lock文件放到clone的dbm仓库 - cd blueking-dbm/ && \ - git checkout -f ${MEDIUM_BUILDER_BRANCH} && \ - cp ../medium.lock ${MEDIUM_PATH} && \ + cp medium.lock /blueking-dbm/${MEDIUM_PATH} && \ # 判断有diff才提pr,否则会非正常结束。注意这里也需要删除.git - git add ${MEDIUM_PATH}/medium.lock && \ - git diff-index --quiet HEAD && rm -rf /app/blueking-dbm || \ + cd /blueking-dbm && git add ${MEDIUM_PATH}/medium.lock && \ + git diff-index --quiet HEAD && rm -rf /blueking-dbm || \ ( \ git commit -m "minor: [$(date +"%Y-%m-%d %H:%M:%S")]update medium.lock" && \ git push --set-upstream --force origin ${MEDIUM_BUILDER_BRANCH}:medicum_lock_${MEDIUM_BUILDER_BRANCH} && \ - rm -rf /app/blueking-dbm \ + rm -rf /blueking-dbm \ ) + + +FROM go-mod-builder as medium-builder + +WORKDIR / +## 安装jq依赖 +RUN set -ex && \ + apt-get update && apt-get install -y --no-install-recommends wget && \ + wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && \ + chmod a+x /usr/local/bin/yq && \ + rm -rf /var/lib/apt/lists/* + +## 保存jq的脚本用于下一阶段用 +RUN set -ex && \ + echo 'yq e ".${1}[].${2} | .${3}" /medium.lock | grep -v null' >> lock.sh && \ + chmod 777 lock.sh + +## 获取上一阶段的lock文件 +COPY --from=base-app /app/medium.lock / + + +FROM medium-builder as mysql-medium-builder + +WORKDIR / +## 构建mysql介质 +RUN set -ex && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/dbactuator && \ + make VERSION=$(/lock.sh mysql actuator version) GITHASH=$(/lock.sh mysql actuator commitId) -j4 && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-dbbackup && \ + sh build.sh -t txsql && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-table-checksum && \ + make release-bin VERSION=$(/lock.sh mysql mysql-checksum version) GITHASH=$(/lock.sh mysql mysql-checksum commitId) -j4 && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-crond && \ + make release-bin VERSION=$(/lock.sh mysql mysql-crond version) GITHASH=$(/lock.sh mysql mysql-crond commitId) -j4 && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-rotatebinlog && \ + make release VERSION=$(/lock.sh mysql rotate-binlog version) GITHASH=$(/lock.sh mysql rotate-binlog commitId) -j4 && \ + cd /blueking-dbm/dbm-services/mysql/db-tools/mysql-monitor && \ + make release-bin VERSION=$(/lock.sh mysql mysql-monitor version) GITHASH=$(/lock.sh mysql mysql-monitor commitId) -j4 + + +FROM medium-builder as redis-medium-builder + +## 构建redis介质 +RUN set -ex && \ + cd /blueking-dbm/dbm-services/redis/db-tools/dbactuator && \ + make VERSION=$(/lock.sh redis actuator version) GITHASH=$(/lock.sh redis actuator commitId) -j4 && \ + cd /blueking-dbm/dbm-services/redis/db-tools/dbmon && \ + bash package.sh + + +FROM medium-builder as bigdata-medium-builder + +## 构建大数据介质 +RUN set -ex && \ + cd /blueking-dbm/dbm-services/bigdata/db-tools/dbactuator && \ + make VERSION=$(/lock.sh es actuator version) GITHASH=$(/lock.sh es actuator commitId) -j4 + + +FROM base-app as medium-app + +WORKDIR /app + +COPY --from=mysql-medium-builder /blueking-dbm/dbm-services/mysql /blueking-dbm/dbm-services/mysql +COPY --from=redis-medium-builder /blueking-dbm/dbm-services/redis /blueking-dbm/dbm-services/redis +COPY --from=bigdata-medium-builder /blueking-dbm/dbm-services/bigdata /blueking-dbm/dbm-services/bigdata + +RUN set -ex && \ + python main.py --type build && rm -rf /blueking-dbm diff --git a/dbm-ui/backend/dbm_init/medium/handlers.py b/dbm-ui/backend/dbm_init/medium/handlers.py index 78a6443da8..946db7e2c6 100644 --- a/dbm-ui/backend/dbm_init/medium/handlers.py +++ b/dbm-ui/backend/dbm_init/medium/handlers.py @@ -188,8 +188,8 @@ def sync_from_bkrepo(self, db_type): http.post(url="apis/packages/update_or_create/", data=package_params) @classmethod - def build_medium(cls, bkrepo_tmp_dir): - """同步构建好的介质,并更新.lock文件""" + def update_lock(cls, bkrepo_tmp_dir): + """更新.lock文件""" def add_version(version): # TODO: 这里版本号叠加规则是怎样?默认只是小版本+1 @@ -220,6 +220,20 @@ def add_version(version): medium_info["version"] = add_version(medium_info["version"]) medium_info["commitId"] = dir_commit + # 更新lock文件 + with open(medium_lock_path, "w") as lock_file: + lock_file.write(yaml.safe_dump(lock_info)) + + @classmethod + def build_medium(cls, bkrepo_tmp_dir): + # 加载lock文件,获取介质的版本信息 + medium_lock_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "medium.lock") + with open(medium_lock_path, "r") as lock_file: + lock_info = yaml.safe_load(lock_file) + + for db_type, mediums in lock_info.items(): + for medium in mediums: + for medium_type, medium_info in medium.items(): # 将编译好的介质复制到指定目录 target_medium_path = f"{bkrepo_tmp_dir}/{db_type}/{medium_type}/{medium_info['version']}" result = subprocess.run( @@ -230,7 +244,3 @@ def add_version(version): ) if result.returncode: logger.error("Error: move medium fail! message: %s", result.stderr) - - # 更新lock文件 - with open(medium_lock_path, "w") as lock_file: - lock_file.write(yaml.safe_dump(lock_info)) diff --git a/dbm-ui/backend/dbm_init/medium/main.py b/dbm-ui/backend/dbm_init/medium/main.py index 59a093d7fe..9139cac550 100644 --- a/dbm-ui/backend/dbm_init/medium/main.py +++ b/dbm-ui/backend/dbm_init/medium/main.py @@ -31,7 +31,9 @@ """版本镜像脚本执行入口""" path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "medium") - if args.type == "build": + if args.type == "update_lock": + MediumHandler.update_lock(bkrepo_tmp_dir=path) + elif args.type == "build": MediumHandler.build_medium(bkrepo_tmp_dir=path) elif args.type == "upload": MediumHandler().upload_medium(path=args.db, bkrepo_tmp_dir=path) diff --git a/dbm-ui/backend/flow/plugins/components/collections/riak/get_riak_resource.py b/dbm-ui/backend/flow/plugins/components/collections/riak/get_riak_resource.py index 4718cd4c16..4e16255494 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/riak/get_riak_resource.py +++ b/dbm-ui/backend/flow/plugins/components/collections/riak/get_riak_resource.py @@ -30,22 +30,19 @@ def _execute(self, data, parent_data) -> bool: global_data = data.get_one_of_inputs("global_data") trans_data = data.get_one_of_inputs("trans_data") - if global_data["ip_source"] == "manual_input": - ips = [node["ip"] for node in global_data["nodes"]] - if global_data["ticket_type"] == TicketType.RIAK_CLUSTER_APPLY and len(ips) >= 3: - trans_data.nodes = ips - trans_data.base_node = ips[0] - trans_data.operate_nodes = ips[1:] - elif ( - global_data["ticket_type"] == TicketType.RIAK_CLUSTER_SCALE_OUT - or global_data["ticket_type"] == TicketType.RIAK_CLUSTER_SCALE_IN - ) and len(ips) >= 1: - trans_data.operate_nodes = ips - else: - self.log_error(_("获取机器资源失败,新建集群至少选择3台机器,扩容或缩容至少选择1台机器")) - return False - elif self.data["ip_source"] == "resource_pool": - pass + ips = [node["ip"] for node in global_data["nodes"]] + if global_data["ticket_type"] == TicketType.RIAK_CLUSTER_APPLY and len(ips) >= 3: + trans_data.nodes = ips + trans_data.base_node = ips[0] + trans_data.operate_nodes = ips[1:] + elif ( + global_data["ticket_type"] == TicketType.RIAK_CLUSTER_SCALE_OUT + or global_data["ticket_type"] == TicketType.RIAK_CLUSTER_SCALE_IN + ) and len(ips) >= 1: + trans_data.operate_nodes = ips + else: + self.log_error(_("获取机器资源失败,新建集群至少选择3台机器,扩容或缩容至少选择1台机器")) + return False self.log_info(_("获取机器资源成功。 {}").format(trans_data)) data.outputs["trans_data"] = trans_data diff --git a/dbm-ui/backend/ticket/builders/mysql/base.py b/dbm-ui/backend/ticket/builders/mysql/base.py index 6c12e71cf7..cdfd91e7bb 100644 --- a/dbm-ui/backend/ticket/builders/mysql/base.py +++ b/dbm-ui/backend/ticket/builders/mysql/base.py @@ -97,7 +97,7 @@ def validate_cluster_can_access(self, attrs): for status_flag, whitelist in self.unavailable_whitelist__status_flag.items(): if cluster.status_flag & status_flag and ticket_type not in whitelist: raise serializers.ValidationError( - _("高可用实例状态异常:{},暂时无法执行该单据类型:{}").format(status_flag.flag_text(), ticket_type) + _("集群实例状态异常:{},暂时无法执行该单据类型:{}").format(status_flag.flag_text(), ticket_type) ) return attrs diff --git a/dbm-ui/backend/ticket/builders/riak/base.py b/dbm-ui/backend/ticket/builders/riak/base.py index 50d0c24ec2..3995546bdf 100644 --- a/dbm-ui/backend/ticket/builders/riak/base.py +++ b/dbm-ui/backend/ticket/builders/riak/base.py @@ -12,6 +12,9 @@ from backend.configuration.constants import DBType from backend.ticket.builders import TicketFlowBuilder +# 默认riak的版本为2.2 +RIAK_VERSION = "2.2" + class BaseRiakTicketFlowBuilder(TicketFlowBuilder): group = DBType.Riak.value diff --git a/dbm-ui/backend/ticket/builders/riak/riak_apply.py b/dbm-ui/backend/ticket/builders/riak/riak_apply.py index 5e94592614..eaeb425353 100644 --- a/dbm-ui/backend/ticket/builders/riak/riak_apply.py +++ b/dbm-ui/backend/ticket/builders/riak/riak_apply.py @@ -18,7 +18,7 @@ from backend.flow.engine.controller.riak import RiakController from backend.ticket import builders from backend.ticket.builders.common.base import CommonValidate -from backend.ticket.builders.riak.base import BaseRiakTicketFlowBuilder +from backend.ticket.builders.riak.base import RIAK_VERSION, BaseRiakTicketFlowBuilder from backend.ticket.constants import TicketType @@ -34,7 +34,7 @@ class RiakApplyDetailSerializer(serializers.Serializer): ip_source = serializers.ChoiceField( help_text=_("主机来源"), choices=IpSource.get_choices(), default=IpSource.RESOURCE_POOL.value ) - db_version = serializers.CharField(help_text=_("riak数据库版本")) + db_version = serializers.CharField(help_text=_("riak数据库版本"), required=False, default=RIAK_VERSION) resource_spec = serializers.JSONField(help_text=_("部署规格")) # display fields @@ -88,8 +88,6 @@ def patch_ticket_detail(self): details = self.ticket.details db_module_name = DBModule.objects.get(db_module_id=details["db_module_id"]).db_module_name riak_cluster_name = riak_domain = f"{details['cluster_name']}.{db_module_name}" - - # TODO: 可能需要补充version等信息 details.update( db_module_name=db_module_name, cluster_name=riak_cluster_name, diff --git a/dbm-ui/backend/ticket/builders/riak/riak_shrink.py b/dbm-ui/backend/ticket/builders/riak/riak_shrink.py index 0492dddcfe..7b56e57e35 100644 --- a/dbm-ui/backend/ticket/builders/riak/riak_shrink.py +++ b/dbm-ui/backend/ticket/builders/riak/riak_shrink.py @@ -38,7 +38,7 @@ def format_ticket_data(self): self.ticket_data["bk_cloud_id"] = cluster.bk_cloud_id -@builders.BuilderFactory.register(TicketType.RIAK_CLUSTER_SCALE_IN, phase=ClusterPhase.OFFLINE) +@builders.BuilderFactory.register(TicketType.RIAK_CLUSTER_SCALE_IN) class RiakShrinkFlowBuilder(BaseRiakTicketFlowBuilder): serializer = RiakShrinkDetailSerializer inner_flow_builder = RiakShrinkFlowParamBuilder diff --git a/dbm-ui/backend/ticket/exclusive_ticket.xlsx b/dbm-ui/backend/ticket/exclusive_ticket.xlsx index 4bd66fb734..7126df3287 100644 Binary files a/dbm-ui/backend/ticket/exclusive_ticket.xlsx and b/dbm-ui/backend/ticket/exclusive_ticket.xlsx differ diff --git a/helm-charts/bk-dbm/Chart.lock b/helm-charts/bk-dbm/Chart.lock index 3edc47e208..a255e2baaa 100644 --- a/helm-charts/bk-dbm/Chart.lock +++ b/helm-charts/bk-dbm/Chart.lock @@ -4,7 +4,7 @@ dependencies: version: 1.13.0 - name: mysql repository: https://charts.bitnami.com/bitnami - version: 9.14.1 + version: 9.14.2 - name: redis repository: https://charts.bitnami.com/bitnami version: 16.13.2 @@ -56,5 +56,5 @@ dependencies: - name: backup-consumer repository: file://charts/backup-consumer version: 0.0.3 -digest: sha256:2980ed56f192ffd9fa531dc1668522eefacd827580a900509ba0e4a78a826946 -generated: "2023-11-01T15:09:09.075761+08:00" +digest: sha256:6a298ec313dd81bf8304ff4851852594333711addfc65f44aff1aeda34c7ed24 +generated: "2023-11-06T21:45:38.599672+08:00"