feat: 新增mysql开区功能 #1036

dbm-services/mysql/db-partition/handler/handler.go

@@ -2,7 +2,6 @@
 package handler
 
 import (
-	"dbm-services/mysql/db-partition/cron"
 	"errors"
 	"fmt"
 	"net/http"
@@ -11,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"dbm-services/mysql/db-partition/cron"
+
 	cron_pkg "github.com/robfig/cron/v3"
 
 	"dbm-services/common/go-pubpkg/errno"

dbm-services/mysql/db-partition/service/execute_partition_object.go

@@ -1,3 +1,4 @@
+// Package service TODO
 package service
 
 import (
@@ -117,3 +118,18 @@ type PartitionObject struct {
 	ShardName      string         `json:"shard_name"`
 	ExecuteObjects []PartitionSql `json:"execute_objects"`
 }
+
+// PartitionLogs TODO
+type PartitionLogs struct {
+	ID          int       `gorm:"column:id;primary_key;auto_increment`
+	BkBizId     int       `gorm:"column:bk_biz_id"`
+	Operator    string    `gorm:"column:operator"`
+	Para        string    `gorm:"column:para"`
+	ExecuteTime time.Time `gorm:"column:execute_time"`
+}
+
+// PartitionLogsParam TODO
+type PartitionLogsParam struct {
+	Para string
+	Err  error
+}

dbm-services/mysql/db-partition/service/manage_config.go

@@ -11,6 +11,7 @@
 package service
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"regexp"
@@ -305,6 +306,7 @@ func (m *CreatePartitionsInput) UpdatePartitionsConfig() error {
 
 	reservedPartition := m.ExpireTime / m.PartitionTimeInterval
 	partitionType := 0
+
 	switch m.PartitionColumnType {
 	case "datetime":
 		partitionType = 0
@@ -318,37 +320,53 @@ func (m *CreatePartitionsInput) UpdatePartitionsConfig() error {
 	var errs []string
 	for _, dblike := range m.DbLikes {
 		for _, tblike := range m.TbLikes {
-			updateCondition := fmt.Sprintf("bk_biz_id=%d and immute_domain='%s' and dblike='%s' and tblike='%s'",
-				m.BkBizId, m.ImmuteDomain, dblike, tblike)
-			var updateColumn struct {
-				PartitionColumn       string
-				PartitionColumnType   string
-				ReservedPartition     int
-				ExtraPartition        int
-				PartitionTimeInterval int
-				PartitionType         int
-				ExpireTime            int
-				Creator               string
-				Updator               string
+			update_column_map := map[string]interface{}{
+				"partition_column":        m.PartitionColumn,
+				"partition_column_type":   m.PartitionColumnType,
+				"reserved_partition":      reservedPartition,
+				"extra_partition":         extraTime,
+				"partition_time_interval": m.PartitionTimeInterval,
+				"partition_type":          partitionType,
+				"expire_time":             m.ExpireTime,
+				"updator":                 m.Updator,
+				"update_time":             time.Now(),
+			}
+			result := model.DB.Self.Debug().Table(tbName).
+				Where(
+					"bk_biz_id=? and immute_domain=? and dblike=? and tblike=?",
+					m.BkBizId, m.ImmuteDomain, dblike, tblike).
+				Updates(update_column_map)
+			var para PartitionLogsParam
+			jString, jerr := json.Marshal(update_column_map)
+			if jerr != nil {
+				return jerr
 			}
-			updateColumn.PartitionColumn = m.PartitionColumn
-			updateColumn.PartitionColumnType = m.PartitionColumnType
-			updateColumn.ReservedPartition = reservedPartition
-			updateColumn.ExtraPartition = extraTime
-			updateColumn.PartitionTimeInterval = m.PartitionTimeInterval
-			updateColumn.PartitionType = partitionType
-			updateColumn.ExpireTime = m.ExpireTime
-			updateColumn.Creator = m.Creator
-			updateColumn.Updator = m.Updator
-			result := model.DB.Self.Debug().Table(tbName).Where(updateCondition).Updates(&updateColumn)
+			para.Para = string(jString)
 			if result.Error != nil {
 				errs = append(errs, result.Error.Error())
+				para.Err = result.Error
+			}
+			var plog PartitionLogs
+			plog.BkBizId = m.BkBizId
+			plog.ExecuteTime = time.Now()
+			plog.Operator = m.Updator
+
+			jString, jerr = json.Marshal(update_column_map)
+			if jerr != nil {
+				return jerr
+			}
+			plog.Para = string(jString)
+			res := model.DB.Self.Debug().Create(&plog)
+			if res.Error != nil {
+				return res.Error
 			}
 		}
 	}
+
 	if len(errs) > 0 {
 		return fmt.Errorf("errors: %s", strings.Join(errs, "\n"))
 	}
+
 	return nil
 }
 

dbm-services/mysql/db-tools/dbactuator/.gitignore

@@ -25,4 +25,7 @@ sync_test.sh
 .vscode/
 scripts/upload_media.sh
 scripts/upload.sh
-localtest/
+localtest/
+.codecc
+.idea
+.vscode

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/mysqlcmd.go

@@ -58,6 +58,10 @@ func NewMysqlCommand() *cobra.Command {
 				NewDropTableCommand(),
 				InstallBackupClientCommand(),
 				NewEnableTokudbPluginCommand(),
+				NewOpenAreaDumpSchemaCommand(),
+				NewOpenAreaImportSchemaCommand(),
+				NewOpenAreaDumpData(),
+				NewOpenAreaImportData(),
 			},
 		},
 		{

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/open_area_dump_data.go

@@ -0,0 +1,96 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package mysqlcmd
+
+import (
+	"fmt"
+
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/internal/subcmd"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+
+	"github.com/spf13/cobra"
+)
+
+// OpenAreaDumpDataAct TODO
+type OpenAreaDumpDataAct struct {
+	*subcmd.BaseOptions
+	Service mysql.OpenAreaDumpSchemaComp
+}
+
+// NewOpenAreaDumpData TODO
+func NewOpenAreaDumpData() *cobra.Command {
+	act := OpenAreaDumpDataAct{
+		BaseOptions: subcmd.GBaseOptions,
+	}
+	cmd := &cobra.Command{
+		Use:   "open_area_dumpdata",
+		Short: "开区导出数据",
+		Example: fmt.Sprintf(
+			`dbactuator mysql open_area_dumpdata %s %s`,
+			subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()),
+		),
+		Run: func(cmd *cobra.Command, args []string) {
+			util.CheckErr(act.Validate())
+			util.CheckErr(act.Init())
+			util.CheckErr(act.Run())
+		},
+	}
+	return cmd
+}
+
+// Validate TODO
+func (d *OpenAreaDumpDataAct) Validate() (err error) {
+	return d.BaseOptions.Validate()
+}
+
+// Init TODO
+func (d *OpenAreaDumpDataAct) Init() (err error) {
+	if err = d.Deserialize(&d.Service.Params); err != nil {
+		logger.Error("DeserializeAndValidate failed, %v", err)
+		return err
+	}
+	// d.Deserialize方法执行后，并直接返回值，
+	d.Service.GeneralParam = subcmd.GeneralRuntimeParam
+	return nil
+}
+
+// Run TODO
+func (d *OpenAreaDumpDataAct) Run() (err error) {
+	steps := subcmd.Steps{
+		{
+			FunName: "init",
+			Func:    d.Service.Init,
+		},
+		{
+			FunName: "precheck",
+			Func:    d.Service.Precheck,
+		},
+		{
+			FunName: "运行导出指定表数据",
+			Func:    d.Service.OpenAreaDumpData,
+		},
+		{
+			FunName: "压缩开区文件",
+			Func:    d.Service.CompressDumpDir,
+		},
+		{
+			FunName: "上传指定表数据文件",
+			Func:    d.Service.Upload,
+		},
+	}
+	if err := steps.Run(); err != nil {
+		return err
+	}
+	logger.Info("开区导出表数据成功")
+	return nil
+}

dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/open_area_dump_schema.go

@@ -0,0 +1,98 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package mysqlcmd
+
+import (
+	"fmt"
+
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/internal/subcmd"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util"
+
+	"github.com/spf13/cobra"
+)
+
+// OpenAreaDumpSchemaAct TODO
+type OpenAreaDumpSchemaAct struct {
+	*subcmd.BaseOptions
+	Service mysql.OpenAreaDumpSchemaComp
+}
+
+// NewOpenAreaDumpSchemaCommand TODO
+func NewOpenAreaDumpSchemaCommand() *cobra.Command {
+	// *subcmd.BaseOptions是指针变量，需要初始化， subcmd.GBaseOptions在subcmd的init中已被初始化
+	act := OpenAreaDumpSchemaAct{
+		BaseOptions: subcmd.GBaseOptions,
+	}
+
+	cmd := &cobra.Command{
+		Use:   "open_area_dumpschema",
+		Short: "开区导出表结构",
+		Example: fmt.Sprintf(
+			`dbactuator mysql open_area_dumpschema %s %s`,
+			subcmd.CmdBaseExampleStr, subcmd.ToPrettyJson(act.Service.Example()),
+		),
+		Run: func(cmd *cobra.Command, args []string) {
+			util.CheckErr(act.Validate())
+			util.CheckErr(act.Init())
+			util.CheckErr(act.Run())
+		},
+	}
+	return cmd
+}
+
+// Validate TODO
+func (d *OpenAreaDumpSchemaAct) Validate() (err error) {
+	return d.BaseOptions.Validate()
+}
+
+// Init TODO
+func (d *OpenAreaDumpSchemaAct) Init() (err error) {
+	if err = d.Deserialize(&d.Service.Params); err != nil {
+		logger.Error("DeserializeAndValidate failed, %v", err)
+		return err
+	}
+	// d.Deserialize方法执行后，并直接返回值，
+	d.Service.GeneralParam = subcmd.GeneralRuntimeParam
+	return nil
+}
+
+// Run TODO
+func (d *OpenAreaDumpSchemaAct) Run() (err error) {
+	steps := subcmd.Steps{
+		{
+			FunName: "init",
+			Func:    d.Service.Init,
+		},
+		{
+			FunName: "precheck",
+			Func:    d.Service.Precheck,
+		},
+		{
+			FunName: "运行导出表结构",
+			Func:    d.Service.OpenAreaDumpSchema,
+		},
+		{
+			FunName: "压缩开区文件",
+			Func:    d.Service.CompressDumpDir,
+		},
+		{
+			FunName: "上传表结构",
+			Func:    d.Service.Upload,
+		},
+	}
+	if err := steps.Run(); err != nil {
+		return err
+	}
+	logger.Info("开区导出表结构成功")
+	return nil
+}