Merge pull request #2038 from actiontech/release-2.9999.x

merge 2.9999.x to main
actiontech · Nov 23, 2023 · 2c989c9 · 2c989c9
2 parents 25cec16 + 089e0b5
commit 2c989c9
Show file tree

Hide file tree

Showing 23 changed files with 647 additions and 204 deletions.
diff --git a/.github/workflows/check-pr-files.yml b/.github/workflows/check-pr-files.yml
@@ -12,5 +12,5 @@ jobs:
         name: Prevent file change
         with:
           githubToken: ${{ secrets.GITHUB_TOKEN }}
-          pattern: ^(?!.*_ee\.go$)(?!.*_ee_test\.go$)(?!.*_rel\.go$)(?!.*_rel_test\.go$)(?!go\.mod$)(?!go\.sum$)(?!\.github\/workflows\/check-pr-files\.yml$)(?!vendor\/.*)(?!.*_qa\.go$).*
+          pattern: ^(?!.*_ee\/.*$)(?!.*_ee\.go$)(?!.*_ee_test\.go$)(?!.*_rel\.go$)(?!.*_rel_test\.go$)(?!go\.mod$)(?!go\.sum$)(?!\.github\/workflows\/check-pr-files\.yml$)(?!vendor\/.*)(?!.*_qa\.go$).*
           trustedAuthors: xalvarez
diff --git a/.github/workflows/pr-ce-release-2-main.yml b/.github/workflows/pr-ce-release-2-main.yml
@@ -6,16 +6,20 @@ on:
 
 jobs:
   cherry_pick_release_2_main:
-    if: github.event.pull_request.merged == true && github.repository == 'actiontech/sqle'
+    if: contains(github.event.pull_request.labels.*.name, 'need_cherry_pick')
     runs-on: ubuntu-latest
     name: Cherry pick into main
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Cherry pick into main
-        uses: Nathanmalnoury/gh-backport-action@master
-        with:
-          pr_branch: 'main'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+    - name: checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Create PR to main
+      uses: carloscastrojumo/[email protected]
+      with:
+        branch: ${{ github.repository == 'actiontech/sqle-ee' && 'main-ee' || 'main' }}
+        labels: |
+          cherry-pick
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/spelling_dict.txt b/spelling_dict.txt
@@ -13,13 +13,15 @@ asensitive
 atoi
 attrs
 auditplan
+armscii
 backoff
 bcda
 beeefore
 bigint
 binlog
 blkid
 btree
+chinese
 cardinalities
 ccug
 chanxuehong
@@ -439,4 +441,6 @@ XtraBackup
 mysqldump
 datediff
 xxx
-sprintln
+sprintln
+sessionctx
+stmtctx
diff --git a/sqle/api/controller/v1/audit_plan.go b/sqle/api/controller/v1/audit_plan.go
@@ -798,7 +798,7 @@ type AuditPlanSQLReqV1 struct {
 	QueryTimeMax         *float64  `json:"query_time_max" from:"query_time_max" example:"5.22"`
 	FirstQueryAt         time.Time `json:"first_query_at" from:"first_query_at" example:"2023-09-12T02:48:01.317880Z"`
 	DBUser               string    `json:"db_user" from:"db_user" example:"database_user001"`
-	EndPoint             string    `json:"end_point" from:"end_point" example:"10.186.1.2"`
+	Endpoint             string    `json:"endpoint" from:"endpoint" example:"10.186.1.2"`
 }
 
 // @Summary 全量同步SQL到扫描任务
@@ -974,6 +974,7 @@ func convertToModelAuditPlanSQL(c echo.Context, auditPlan *model.AuditPlan, reqS
 			SQLContent:  reqSQL.LastReceiveText,
 			Info:        info,
 			Schema:      reqSQL.Schema,
+			Endpoint:    reqSQL.Endpoint,
 		})
 	}
 	return sqls, nil

diff --git a/sqle/api/controller/v1/sql_manage.go b/sqle/api/controller/v1/sql_manage.go
@@ -16,7 +16,7 @@ type GetSqlManageListReq struct {
 	FilterStatus                 *string `query:"filter_status" json:"filter_status,omitempty"`
 	FilterDbType                 *string `query:"filter_db_type" json:"filter_db_type,omitempty"`
 	FilterRuleName               *string `query:"filter_rule_name" json:"filter_rule_name,omitempty"`
-	FilterEndpoint               *string `query:"filter_endpoint" json:"filter_endpoint,omitempty"`
+	FuzzySearchEndpoint          *string `query:"fuzzy_search_endpoint" json:"fuzzy_search_endpoint,omitempty"`
 	SortField                    *string `query:"sort_field" json:"sort_field,omitempty" valid:"omitempty,oneof=first_appear_timestamp last_receive_timestamp fp_count" enums:"first_appear_timestamp,last_receive_timestamp,fp_count"`
 	SortOrder                    *string `query:"sort_order" json:"sort_order,omitempty" valid:"omitempty,oneof=asc desc" enums:"asc,desc"`
 	PageIndex                    uint32  `query:"page_index" valid:"required" json:"page_index"`
@@ -78,7 +78,7 @@ type Source struct {
 // @Param filter_status query string false "status" Enums(unhandled,solved,ignored,manual_audited)
 // @Param filter_rule_name query string false "rule name"
 // @Param filter_db_type query string false "db type"
-// @Param filter_endpoint query string false "endpoint"
+// @Param fuzzy_search_endpoint query string false "fuzzy search endpoint"
 // @Param sort_field query string false "sort field" Enums(first_appear_timestamp,last_receive_timestamp,fp_count)
 // @Param sort_order query string false "sort order" Enums(asc,desc)
 // @Param page_index query uint32 true "page index"
@@ -121,7 +121,7 @@ type ExportSqlManagesReq struct {
 	FilterStatus                 *string `query:"filter_status" json:"filter_status,omitempty"`
 	FilterDbType                 *string `query:"filter_db_type" json:"filter_db_type,omitempty"`
 	FilterRuleName               *string `query:"filter_rule_name" json:"filter_rule_name,omitempty"`
-	FilterEndpoint               *string `query:"filter_endpoint" json:"filter_endpoint,omitempty"`
+	FuzzySearchEndpoint          *string `query:"fuzzy_search_endpoint" json:"fuzzy_search_endpoint,omitempty"`
 	SortField                    *string `query:"sort_field" json:"sort_field,omitempty" valid:"omitempty,oneof=first_appear_timestamp last_receive_timestamp fp_count" enums:"first_appear_timestamp,last_receive_timestamp,fp_count"`
 	SortOrder                    *string `query:"sort_order" json:"sort_order,omitempty" valid:"omitempty,oneof=asc desc" enums:"asc,desc"`
 }
@@ -143,7 +143,7 @@ type ExportSqlManagesReq struct {
 // @Param filter_status query string false "status" Enums(unhandled,solved,ignored,manual_audited)
 // @Param filter_db_type query string false "db type"
 // @Param filter_rule_name query string false "rule name"
-// @Param filter_endpoint query string false "endpoint"
+// @Param fuzzy_search_endpoint query string false "fuzzy search endpoint"
 // @Param sort_field query string false "sort field" Enums(first_appear_timestamp,last_receive_timestamp,fp_count)
 // @Param sort_order query string false "sort order" Enums(asc,desc)
 // @Success 200 {file} file "export sql manage"

diff --git a/sqle/cmd/scannerd/scanners/scanner.go b/sqle/cmd/scannerd/scanners/scanner.go
@@ -18,6 +18,7 @@ type SQL struct {
 	QueryTime   float64   // 慢日志执行时长
 	QueryAt     time.Time // 慢日志发生时间
 	DBUser      string    // 执行SQL的用户
+	Endpoint    string    // 下发SQL的端点信息
 }
 
 // Scanner is a interface for all Scanners.

diff --git a/sqle/docs/docs.go b/sqle/docs/docs.go
@@ -3020,8 +3020,8 @@ var doc = `{
                     },
                     {
                         "type": "string",
-                        "description": "endpoint",
-                        "name": "filter_endpoint",
+                        "description": "fuzzy search endpoint",
+                        "name": "fuzzy_search_endpoint",
                         "in": "query"
                     },
                     {
@@ -3210,8 +3210,8 @@ var doc = `{
                     },
                     {
                         "type": "string",
-                        "description": "endpoint",
-                        "name": "filter_endpoint",
+                        "description": "fuzzy search endpoint",
+                        "name": "fuzzy_search_endpoint",
                         "in": "query"
                     },
                     {
@@ -7338,7 +7338,7 @@ var doc = `{
                     "type": "string",
                     "example": "database_user001"
                 },
-                "end_point": {
+                "endpoint": {
                     "type": "string",
                     "example": "10.186.1.2"
                 },

diff --git a/sqle/docs/swagger.json b/sqle/docs/swagger.json
@@ -3004,8 +3004,8 @@
                     },
                     {
                         "type": "string",
-                        "description": "endpoint",
-                        "name": "filter_endpoint",
+                        "description": "fuzzy search endpoint",
+                        "name": "fuzzy_search_endpoint",
                         "in": "query"
                     },
                     {
@@ -3194,8 +3194,8 @@
                     },
                     {
                         "type": "string",
-                        "description": "endpoint",
-                        "name": "filter_endpoint",
+                        "description": "fuzzy search endpoint",
+                        "name": "fuzzy_search_endpoint",
                         "in": "query"
                     },
                     {
@@ -7322,7 +7322,7 @@
                     "type": "string",
                     "example": "database_user001"
                 },
-                "end_point": {
+                "endpoint": {
                     "type": "string",
                     "example": "10.186.1.2"
                 },

diff --git a/sqle/docs/swagger.yaml b/sqle/docs/swagger.yaml
@@ -144,7 +144,7 @@ definitions:
       db_user:
         example: database_user001
         type: string
-      end_point:
+      endpoint:
         example: 10.186.1.2
         type: string
       first_query_at:
@@ -5475,9 +5475,9 @@ paths:
         in: query
         name: filter_db_type
         type: string
-      - description: endpoint
+      - description: fuzzy search endpoint
         in: query
-        name: filter_endpoint
+        name: fuzzy_search_endpoint
         type: string
       - description: sort field
         enum:
@@ -5603,9 +5603,9 @@ paths:
         in: query
         name: filter_rule_name
         type: string
-      - description: endpoint
+      - description: fuzzy search endpoint
         in: query
-        name: filter_endpoint
+        name: fuzzy_search_endpoint
         type: string
       - description: sort field
         enum:

diff --git a/sqle/driver/mysql/audit_executed_sql_test.go b/sqle/driver/mysql/audit_executed_sql_test.go
@@ -112,9 +112,9 @@ ALTER TABLE exist_db.not_exist_tb_1 ADD INDEX idx_2(b1);
 ALTER TABLE exist_db.not_exist_tb_1 ADD COLUMN b2 blob UNIQUE KEY COMMENT "unit test";
 ALTER TABLE exist_db.not_exist_tb_1 MODIFY COLUMN b1 blob UNIQUE KEY COMMENT "unit test";
 `,
-					newTestResult().addResult(rulepkg.DDLCheckIndexedColumnWithBlob).
-						add(driverV2.RuleLevelWarn, "", "建表DDL必须包含CREATE_TIME字段且默认值为CURRENT_TIMESTAMP").
-						add(driverV2.RuleLevelWarn, "", "建表DDL必须包含UPDATE_TIME字段且默认值为CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP").
+					newTestResult().addResult(rulepkg.DDLCheckPKName).addResult(rulepkg.DDLCheckIndexedColumnWithBlob).
+						add(driverV2.RuleLevelWarn, "", "建议建表DDL包含CREATE_TIME字段且默认值为CURRENT_TIMESTAMP").
+						add(driverV2.RuleLevelWarn, "", "建表DDL需要包含UPDATE_TIME字段且默认值为CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP").
 						add(driverV2.RuleLevelWarn, "", "这些索引字段(b1)需要有非空约束"),
 					newTestResult().addResult(rulepkg.DDLCheckIndexNotNullConstraint, "b1"),
 					newTestResult().addResult(rulepkg.DDLCheckIndexNotNullConstraint, "b1"),
@@ -226,12 +226,12 @@ id bigint unsigned NOT NULL KEY DEFAULT "unit test" COMMENT "unit test",
 v1 varchar(255) NOT NULL DEFAULT "unit test" COMMENT "unit test",
 v2 varchar(255) NOT NULL DEFAULT "unit test" COMMENT "unit test"
 )ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COMMENT="unit test";
-ALTER TABLE exist_db.exist_tb_1 Add primary key(v1);
+ALTER TABLE exist_db.exist_tb_1 Add primary key(v1); 
 			`,
 					newTestResult().addResult(rulepkg.DDLCheckPKWithoutAutoIncrement).
-						add(driverV2.RuleLevelWarn, "", "建表DDL必须包含CREATE_TIME字段且默认值为CURRENT_TIMESTAMP").
-						add(driverV2.RuleLevelWarn, "", "建表DDL必须包含UPDATE_TIME字段且默认值为CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"),
-					newTestResult(),
+						add(driverV2.RuleLevelWarn, "", "建议建表DDL包含CREATE_TIME字段且默认值为CURRENT_TIMESTAMP").
+						add(driverV2.RuleLevelWarn, "", "建表DDL需要包含UPDATE_TIME字段且默认值为CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"),
+					newTestResult().addResult(rulepkg.DDLCheckPKName),
 				)
 			})
 
@@ -249,10 +249,10 @@ PRIMARY KEY (id)
 )ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COMMENT="unit test";
 ALTER TABLE exist_db.exist_tb_1 Add primary key(v1);
 			`,
-					newTestResult().addResult(rulepkg.DDLCheckPKWithoutBigintUnsigned).
-						add(driverV2.RuleLevelWarn, "", "建表DDL必须包含CREATE_TIME字段且默认值为CURRENT_TIMESTAMP").
-						add(driverV2.RuleLevelWarn, "", "建表DDL必须包含UPDATE_TIME字段且默认值为CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"),
-					newTestResult(),
+					newTestResult().addResult(rulepkg.DDLCheckPKName).addResult(rulepkg.DDLCheckPKWithoutBigintUnsigned).
+						add(driverV2.RuleLevelWarn, "", "建议建表DDL包含CREATE_TIME字段且默认值为CURRENT_TIMESTAMP").
+						add(driverV2.RuleLevelWarn, "", "建表DDL需要包含UPDATE_TIME字段且默认值为CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"),
+					newTestResult().addResult(rulepkg.DDLCheckPKName),
 				)
 			})