From f2dded1af313235f9e096017a7ac89dfcdcfbb63 Mon Sep 17 00:00:00 2001 From: ymakedaq <996156275@qq.com> Date: Mon, 11 Sep 2023 19:23:59 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix(dbm-services):=20=E4=BF=AE=E5=A4=8Dspid?= =?UTF-8?q?er=E5=B7=A5=E5=85=B7=E4=BD=93=E9=AA=8Cbugs=20close=20#1035?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/db-simulation/app/service/kubernets.go | 5 +++-- .../db-simulation/app/syntax/builtin_rule.go | 9 +++++++-- .../mysql/db-simulation/handler/handler.go | 16 +++++++++++++++- .../bk-dbm/charts/db-simulation/Chart.yaml | 2 +- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dbm-services/mysql/db-simulation/app/service/kubernets.go b/dbm-services/mysql/db-simulation/app/service/kubernets.go index 70e798b8c9..fd69c05285 100644 --- a/dbm-services/mysql/db-simulation/app/service/kubernets.go +++ b/dbm-services/mysql/db-simulation/app/service/kubernets.go @@ -289,8 +289,9 @@ func (k *DbPodSets) DeletePod() (err error) { // GetLoadSchemaSQLCmd TODO func (k *DbPodSets) GetLoadSchemaSQLCmd(bkpath, file string) (cmd string) { cmd = fmt.Sprintf( - "curl -o %s %s && mysql --defaults-file=/etc/my.cnf -uroot -p%s --default-character-set=%s -vvv < %s", - file, getdownloadUrl(bkpath, file), k.BaseInfo.RootPwd, k.BaseInfo.Charset, file) + `curl -o %s %s && sed -i '/50720 SET tc_admin=0/d' %s && + mysql --defaults-file=/etc/my.cnf -uroot -p%s --default-character-set=%s -vvv < %s`, + file, getdownloadUrl(bkpath, file), file, k.BaseInfo.RootPwd, k.BaseInfo.Charset, file) return } diff --git a/dbm-services/mysql/db-simulation/app/syntax/builtin_rule.go b/dbm-services/mysql/db-simulation/app/syntax/builtin_rule.go index 0360cde4ed..a86593339a 100644 --- a/dbm-services/mysql/db-simulation/app/syntax/builtin_rule.go +++ b/dbm-services/mysql/db-simulation/app/syntax/builtin_rule.go @@ -22,12 +22,13 @@ const ( // SpecialCharRegex = `[￥$!@#%^&*()+={}\[\];:'"<>,.?/\\| ]` // AllowWordRegex TODO - AllowWordRegex = `^[a-zA-Z0-9][a-zA-Z0-9_-]+[a-zA-Z0-9]$` + AllowWordRegex = `^[a-zA-Z0-9]([a-zA-Z0-9_-])*[a-zA-Z0-9]$` // SysReservesPrefixName TODO ) var reAllowWord *regexp.Regexp +var reAllowOneWord *regexp.Regexp // var mysql55WordMap map[string]struct{} var mysql56WordMap map[string]struct{} @@ -39,6 +40,7 @@ var sysReservesPrefixNames []string func init() { sysReservesPrefixNames = []string{"stage_truncate"} reAllowWord = regexp.MustCompile(AllowWordRegex) + reAllowOneWord = regexp.MustCompile(`^[a-zA-Z0-9]$`) mysql56WordMap = sliceToMap(keyworld.MySQL56_KEYWORD) mysql57WordMap = sliceToMap(keyworld.MySQL57_KEYWORD) mysql80WordMap = sliceToMap(keyworld.MySQL80_KEYWORD) @@ -66,8 +68,11 @@ func KeyWordValidator(ver, name string) (matched bool, msg string) { // SpecialCharValidator TODO func SpecialCharValidator(name string) (matched bool, msg string) { + if reAllowOneWord.MatchString(name) { + return false, "" + } if !reAllowWord.MatchString(name) { - return true, "Only allowed " + AllowWordRegex + " characters " + return true, name + " : Must match the regexp " + AllowWordRegex + " characters " } for _, sysPrefix := range sysReservesPrefixNames { re := regexp.MustCompile(fmt.Sprintf("^%s", sysPrefix)) diff --git a/dbm-services/mysql/db-simulation/handler/handler.go b/dbm-services/mysql/db-simulation/handler/handler.go index 58ddfb88c2..5fc5eea9a4 100644 --- a/dbm-services/mysql/db-simulation/handler/handler.go +++ b/dbm-services/mysql/db-simulation/handler/handler.go @@ -1,3 +1,13 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + // Package handler TODO package handler @@ -80,6 +90,10 @@ func SpiderClusterSimulation(r *gin.Context) { DbPodSets: service.NewDbPodSets(), BaseParam: ¶m.BaseParam, } + rootPwd := cmutil.RandStr(10) + if !service.DelPod { + logger.Info("the pwd %s", rootPwd) + } tsk.DbImage = img tsk.SpiderImage = param.GetSpiderImg() tsk.TdbCtlImage = param.GetTdbctlImg() @@ -88,7 +102,7 @@ func SpiderClusterSimulation(r *gin.Context) { replaceUnderSource(param.TaskId)), Lables: map[string]string{"task_id": replaceUnderSource(param.TaskId), "request_id": requestId}, - RootPwd: cmutil.RandStr(10), + RootPwd: rootPwd, Charset: param.MySQLCharSet, } service.SpiderTaskChan <- tsk diff --git a/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml b/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml index 182b6c4114..853f02dbd3 100644 --- a/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml +++ b/helm-charts/bk-dbm/charts/db-simulation/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 0.0.1-alpha.32 +appVersion: 0.0.1-alpha.33 description: A Helm chart for Kubernetes name: db-simulation type: application From 0e922006ca549b7f308f6d7f3129910d24066771 Mon Sep 17 00:00:00 2001 From: ymakedaq <996156275@qq.com> Date: Tue, 12 Sep 2023 11:45:50 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(dbm-services):=20=E4=BF=AE=E5=A4=8Dspid?= =?UTF-8?q?er=E5=B7=A5=E5=85=B7=E4=BD=93=E9=AA=8Cbugs=20close=20#1035?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db-simulation/app/service/kubernets.go | 10 +- dbm-services/mysql/db-simulation/rule.yaml | 109 ++++++++++++------ .../templates/dbsimulation-configmap.yaml | 2 +- 3 files changed, 82 insertions(+), 39 deletions(-) diff --git a/dbm-services/mysql/db-simulation/app/service/kubernets.go b/dbm-services/mysql/db-simulation/app/service/kubernets.go index fd69c05285..fdcd3cd897 100644 --- a/dbm-services/mysql/db-simulation/app/service/kubernets.go +++ b/dbm-services/mysql/db-simulation/app/service/kubernets.go @@ -136,7 +136,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) { Command: []string{"/bin/bash", "-c", fmt.Sprintf("mysql -uroot -p%s -e 'select 1'", k.BaseInfo.RootPwd)}, }, }, - InitialDelaySeconds: 2, + InitialDelaySeconds: 3, PeriodSeconds: 5, }, }, { @@ -156,7 +156,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) { Command: []string{"/bin/bash", "-c", fmt.Sprintf("mysql -uroot -p%s -e 'select 1'", k.BaseInfo.RootPwd)}, }, }, - InitialDelaySeconds: 2, + InitialDelaySeconds: 3, PeriodSeconds: 5, }, }, @@ -179,7 +179,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) { Command: []string{"/bin/bash", "-c", fmt.Sprintf("mysql -uroot -p%s -e 'select 1'", k.BaseInfo.RootPwd)}, }, }, - InitialDelaySeconds: 2, + InitialDelaySeconds: 3, PeriodSeconds: 5, }, }, @@ -193,6 +193,7 @@ func (k *DbPodSets) CreateClusterPod() (err error) { logger.Info("connect tdbctl success ~") // create cluster relation for _, ql := range k.getCreateClusterSqls() { + logger.Info("exec init cluster sql %s", ql) if _, err = k.DbWork.Db.Exec(ql); err != nil { return err } @@ -289,8 +290,7 @@ func (k *DbPodSets) DeletePod() (err error) { // GetLoadSchemaSQLCmd TODO func (k *DbPodSets) GetLoadSchemaSQLCmd(bkpath, file string) (cmd string) { cmd = fmt.Sprintf( - `curl -o %s %s && sed -i '/50720 SET tc_admin=0/d' %s && - mysql --defaults-file=/etc/my.cnf -uroot -p%s --default-character-set=%s -vvv < %s`, + `curl -o %s %s && sed -i '/50720 SET tc_admin=0/d' %s && mysql -uroot -p%s --default-character-set=%s -vvv < %s`, file, getdownloadUrl(bkpath, file), file, k.BaseInfo.RootPwd, k.BaseInfo.Charset, file) return } diff --git a/dbm-services/mysql/db-simulation/rule.yaml b/dbm-services/mysql/db-simulation/rule.yaml index 27a04b36a6..2e91bd086b 100644 --- a/dbm-services/mysql/db-simulation/rule.yaml +++ b/dbm-services/mysql/db-simulation/rule.yaml @@ -14,60 +14,103 @@ BuiltInRule: CommandRule: HighRiskCommandRule: - expr: ' Val in Item ' - desc: '高危命令' - item: [ "drop_table", "drop_index", "lock_tables", "analyze","rename_table", "drop_procedure", "drop_view", "drop_trigger","drop_function", "drop_server", "drop_event", "drop_compression_dictionary","optimize", "alter_tablespace"] - + expr: " Val in Item " + desc: "高危命令" + item: + [ + "drop_table", + "drop_index", + "lock_tables", + "analyze", + "rename_table", + "drop_procedure", + "drop_view", + "drop_trigger", + "drop_function", + "drop_server", + "drop_event", + "drop_compression_dictionary", + "optimize", + "alter_tablespace", + ] + BanCommandRule: - expr: ' Val in Item ' - desc: '禁用命令' + expr: " Val in Item " + desc: "禁用命令" ban: true - item: ["truncate", "revoke", "kill", "reset", "drop_db","drop_user", "grant","create_user", "revoke_all", "shutdown", "lock_tables_for_backup","reset", "purge", "lock_binlog_for_backup","lock_tables_for_backup","install_plugin", "uninstall_plugin","alter_user"] - + item: + [ + "truncate", + "revoke", + "kill", + "reset", + "drop_db", + "drop_user", + "grant", + "create_user", + "revoke_all", + "shutdown", + "lock_tables_for_backup", + "reset", + "purge", + "lock_binlog_for_backup", + "lock_tables_for_backup", + "install_plugin", + "uninstall_plugin", + "alter_user", + ] CreateTableRule: SuggestBlobColumCount: - expr: ' Val >= Item ' - item: 10 - desc: "建议单表Blob字段不要过多" + expr: " Val >= Item " + item: 10 + desc: "建议单表Blob字段不要过多" SuggestEngine: - expr: ' not (Val contains Item) and ( len(Val) != 0 ) ' - item: 'innodb' - desc: "建议使用Innodb表" + expr: " not (Val contains Item) and ( len(Val) != 0 ) " + item: "innodb" + desc: "建议使用Innodb表" NeedPrimaryKey: - expr: ' Val == Item ' - item: 1 - desc: "建议包含主键" + expr: " Val == Item " + item: 1 + desc: "建议包含主键" DefinerRule: - expr: ' Val in Item ' - desc: '不允许指定definer' - ban: true - item: ["create_function","create_trigger","create_event","create_procedure","create_view"] + expr: " Val in Item " + desc: "不允许指定definer" + ban: true + item: + [ + "create_function", + "create_trigger", + "create_event", + "create_procedure", + "create_view", + ] NormalizedName: - expr: ' Val in Item ' - desc: '规范化命名' - item: ["first_char_exception", "special_char", "Keyword_exception"] + expr: " Val in Item " + desc: "规范化命名" + item: ["first_char_exception", "special_char", "Keyword_exception"] AlterTableRule: HighRiskType: - expr: ' Val in Item ' - item: ["drop_column", "drop_key","change_column","rename_table", "rename_key"] + expr: " Val in Item " + item: + ["drop_column", "drop_key", "change_column", "rename_table", "rename_key"] desc: "高危变更类型" HighRiskPkAlterType: - expr: ' Val in Item ' + expr: " Val in Item " item: ["add_column", "add_key", "change_column"] desc: "高危主键变更类型" AlterUseAfter: - expr: ' Val != Item ' + expr: " Val != Item " item: "" desc: "变更表时使用了after" - AddColumnMixed: - expr: ' ( Item in Val ) && ( len(Val) > 1 ) ' - item: 'add_column' + AddColumnMixed: + expr: " ( Item in Val ) && ( len(Val) > 1 ) " + item: "add_column" desc: "加字段和其它alter table 类型混用，可能导致非在线加字段" DmlRule: DmlNotHasWhere: - # expr: ' Val != Item ' - item: true + # expr: ' Val != Item ' + item: true desc: "没有使用WHERE或者LIMIT,可能会导致全表数据更改" diff --git a/helm-charts/bk-dbm/templates/dbsimulation-configmap.yaml b/helm-charts/bk-dbm/templates/dbsimulation-configmap.yaml index 2381696e53..c1ecab40b1 100644 --- a/helm-charts/bk-dbm/templates/dbsimulation-configmap.yaml +++ b/helm-charts/bk-dbm/templates/dbsimulation-configmap.yaml @@ -34,7 +34,7 @@ data: - version: "8.0" image: "{{ .Values.global.imageRegistry | default "mirrors.tencent.com" }}/build/blueking/tendb-8.0.18:4.1_v2" - version: "tdbctl" - image: "{{ .Values.global.imageRegistry | default "mirrors.tencent.com" }}/build/blueking/tdbctl:2.4.1_v3" + image: "{{ .Values.global.imageRegistry | default "mirrors.tencent.com" }}/build/blueking/tdbctl:2.4.3_v1" - version: "spider" image: "{{ .Values.global.imageRegistry | default "mirrors.tencent.com" }}/build/blueking/tspider3:3.7.8_v1" dbconf: From eb212744d06708984d021e0ca5cf1c22d5dd60ae Mon Sep 17 00:00:00 2001 From: ymakedaq <996156275@qq.com> Date: Thu, 14 Sep 2023 16:57:18 +0800 Subject: [PATCH 3/3] update chart version --- dbm-services/go.work.sum | 4 +- helm-charts/bk-dbm/Chart.yaml | 114 +++++++++++++++++----------------- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/dbm-services/go.work.sum b/dbm-services/go.work.sum index 19ee46992d..21e3d070c3 100644 --- a/dbm-services/go.work.sum +++ b/dbm-services/go.work.sum @@ -462,6 +462,7 @@ github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuD github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0 h1:WW2B2uxx9KWF6bGlHqhm8Okiafwwx7Y2kcpn8lCpjgo= github.com/checkpoint-restore/go-criu/v5 v5.3.0 h1:wpFFOoomK3389ue2lAb0Boag6XPht5QYpipxmSNL4d8= @@ -856,8 +857,6 @@ github.com/sclevine/agouti v3.0.0+incompatible h1:8IBJS6PWz3uTlMP3YBIR5f+KAldcGu github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921 h1:58EBmR2dMNL2n/FnbQewK3D14nXr0V9CObDSvMJLq+Y= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca h1:3fECS8atRjByijiI8yYiuwLwQ2ZxXobW7ua/8GRB3pI= @@ -1024,6 +1023,7 @@ golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= diff --git a/helm-charts/bk-dbm/Chart.yaml b/helm-charts/bk-dbm/Chart.yaml index 97d7c0d2a6..70c36c9043 100644 --- a/helm-charts/bk-dbm/Chart.yaml +++ b/helm-charts/bk-dbm/Chart.yaml @@ -1,62 +1,62 @@ apiVersion: v2 dependencies: -- name: common - repository: https://charts.bitnami.com/bitnami - version: 1.13.0 -#- condition: mysql.enabled -# name: mysql -# version: 8.x.x -# repository: https://charts.bitnami.com/bitnami -- condition: redis.enabled - name: redis - version: 16.x.x - repository: https://charts.bitnami.com/bitnami -- name: grafana - repository: file://./charts/grafana - version: 7.x.x - condition: grafana.enabled -- condition: dbm.enabled - name: dbm - repository: file://charts/dbm - version: 0.1.20 -- condition: dbconfig.enabled - name: dbconfig - repository: file://charts/dbconfig - version: 0.1.6 -- condition: dbpriv.enabled - name: dbpriv - repository: file://charts/dbpriv - version: 0.1.28 -- condition: dbpartition.enabled - name: dbpartition - repository: file://charts/dbpartition - version: 0.1.3 -- condition: db-simulation.enabled - name: db-simulation - repository: file://charts/db-simulation - version: 0.1.7 -- condition: dbRemoteService.enabled - name: db-remote-service - repository: file://charts/db-remote-service - version: 0.9.5 -- condition: db-dns-api.enabled - name: db-dns-api - repository: file://charts/db-dns-api - version: 0.1.3 -- condition: hadb-api.enabled - name: hadb-api - repository: file://charts/hadb-api - version: 0.0.10 -- condition: dbResource.enabled - name: db-resource - repository: file://charts/db-resource - version: 0.0.7 -- condition: db-celery-service.enabled - name: db-celery-service - repository: file://charts/db-celery-service - version: 0.0.4 + - name: common + repository: https://charts.bitnami.com/bitnami + version: 1.13.0 + #- condition: mysql.enabled + # name: mysql + # version: 8.x.x + # repository: https://charts.bitnami.com/bitnami + - condition: redis.enabled + name: redis + version: 16.x.x + repository: https://charts.bitnami.com/bitnami + - name: grafana + repository: file://./charts/grafana + version: 7.x.x + condition: grafana.enabled + - condition: dbm.enabled + name: dbm + repository: file://charts/dbm + version: 0.1.20 + - condition: dbconfig.enabled + name: dbconfig + repository: file://charts/dbconfig + version: 0.1.6 + - condition: dbpriv.enabled + name: dbpriv + repository: file://charts/dbpriv + version: 0.1.28 + - condition: dbpartition.enabled + name: dbpartition + repository: file://charts/dbpartition + version: 0.1.3 + - condition: db-simulation.enabled + name: db-simulation + repository: file://charts/db-simulation + version: 0.1.7 + - condition: dbRemoteService.enabled + name: db-remote-service + repository: file://charts/db-remote-service + version: 0.9.5 + - condition: db-dns-api.enabled + name: db-dns-api + repository: file://charts/db-dns-api + version: 0.1.3 + - condition: hadb-api.enabled + name: hadb-api + repository: file://charts/hadb-api + version: 0.0.10 + - condition: dbResource.enabled + name: db-resource + repository: file://charts/db-resource + version: 0.0.7 + - condition: db-celery-service.enabled + name: db-celery-service + repository: file://charts/db-celery-service + version: 0.0.4 description: A Helm chart for bkdbm name: bk-dbm type: application -version: 1.2.0-alpha.26 -appVersion: 1.2.0-alpha.26 +version: 1.2.0-alpha.27 +appVersion: 1.2.0-alpha.27