Skip to content

Commit

Permalink
Merge pull request #2541 from actiontech/report_push_job
Browse files Browse the repository at this point in the history 
Report push job
  • Loading branch information
@sjjian
sjjian authored Aug 16, 2024
2 parents c0009fd + 8794699 commit 0ea5602
Show file tree
Hide file tree
Showing 26 changed files with 662 additions and 75 deletions.
41 changes: 37 additions & 4 deletions sqle/api/controller/v1/audit_plan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,11 @@ type GetAuditPlanMetasResV1 struct {
}

type AuditPlanMetaV1 struct {
Type string `json:"audit_plan_type"`
Desc string `json:"audit_plan_type_desc"`
InstanceType string `json:"instance_type"`
Params []AuditPlanParamResV1 `json:"audit_plan_params,omitempty"`
Type string `json:"audit_plan_type"`
Desc string `json:"audit_plan_type_desc"`
InstanceType string `json:"instance_type"`
Params []AuditPlanParamResV1 `json:"audit_plan_params,omitempty"`
HighPriorityConditions []HighPriorityCondition `json:"high_priority_conditions"`
}

type AuditPlanParamResV1 struct {
Expand All @@ -65,6 +66,20 @@ type AuditPlanParamResV1 struct {
EnumsValues []params.EnumsValue `json:"enums_value"`
}

type HighPriorityCondition struct {
Key string `json:"key"`
Desc string `json:"desc"`
Value string `json:"value"`
Type string `json:"type" enums:"string,int,bool,password"`
EnumsValues []params.EnumsValue `json:"enums_value"`
Operator Operator `json:"operator"`
}

type Operator struct {
Value string `json:"operator_value"`
EnumsValue []params.EnumsValue `json:"operator_enums_value"`
}

func ConvertAuditPlanMetaWithInstanceIdToRes(meta auditplan.Meta, instanceId string) AuditPlanMetaV1 {
res := AuditPlanMetaV1{
Type: meta.Type,
Expand All @@ -88,6 +103,24 @@ func ConvertAuditPlanMetaWithInstanceIdToRes(meta auditplan.Meta, instanceId str
}
res.Params = paramsRes
}
// 高优先级参数
if len(meta.HighPriorityParams) > 0 {
highPriorityparamsRes := make([]HighPriorityCondition, 0, len(meta.HighPriorityParams))
for _, hpc := range meta.HighPriorityParams {
highPriorityparamsRes = append(highPriorityparamsRes, HighPriorityCondition{
Key: hpc.Key,
Desc: hpc.Desc,
Type: string(hpc.Type),
Value: hpc.Value,
EnumsValues: hpc.Enums,
Operator: Operator{
string(hpc.Operator.Value),
hpc.Operator.EnumsValue,
},
})
}
res.HighPriorityConditions = highPriorityparamsRes
}
return res
}

Expand Down
103 changes: 88 additions & 15 deletions sqle/api/controller/v1/instance_audit_plan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,16 @@ type CreateInstanceAuditPlanReqV1 struct {
}

type AuditPlan struct {
RuleTemplateName string `json:"rule_template_name" from:"rule_template_name" example:"default_MySQL"`
Type string `json:"audit_plan_type" form:"audit_plan_type" example:"slow log"`
Params []AuditPlanParamReqV1 `json:"audit_plan_params" valid:"dive,required"`
RuleTemplateName string `json:"rule_template_name" from:"rule_template_name" example:"default_MySQL"`
Type string `json:"audit_plan_type" form:"audit_plan_type" example:"slow log"`
Params []AuditPlanParamReqV1 `json:"audit_plan_params" valid:"dive,required"`
HighPriorityConditions []HighPriorityConditionReq `json:"high_priority_conditions" valid:"dive,required"`
NeedMarkHighPrioritySQL bool `json:"need_mark_high_priority_sql"`
}
type HighPriorityConditionReq struct {
Key string `json:"key" form:"key" valid:"required"`
Value string `json:"value" form:"value" valid:"required"`
Operator string `json:"operator" form:"operator" default:">" enums:">,=,<" valid:"oneof=> = <"`
}

type CreatInstanceAuditPlanResV1 struct {
Expand All @@ -53,6 +60,31 @@ type CreatInstanceAuditPlanRes struct {
InstanceAuditPlanID string `json:"instance_audit_plan_id"`
}

func checkAndGenerateHighPriorityParams(auditPlanType, instanceType string, hpcParamsReq []HighPriorityConditionReq) (params.ParamsWithOperator, error) {
meta, err := auditplan.GetMeta(auditPlanType)
if err != nil {
return nil, err
}
if meta.InstanceType != auditplan.InstanceTypeAll && meta.InstanceType != instanceType {
return nil, fmt.Errorf("audit plan type %s not found", auditPlanType)
}
resetParams := make([]*params.ParamWithOperator, 0)
for _, hpcParam := range hpcParamsReq {
for _, p := range meta.HighPriorityParams {
if p.Key != hpcParam.Key {
continue
}
// set and valid param.
p.Value = hpcParam.Value
p.Operator.Value = params.OperatorValue(hpcParam.Operator)
resetParams = append(resetParams, p)
break
}

}
return resetParams, nil
}

// @Summary 添加实例扫描任务
// @Description create instance audit plan
// @Id createInstanceAuditPlanV1
Expand All @@ -64,7 +96,6 @@ type CreatInstanceAuditPlanRes struct {
// @Success 200 {object} v1.CreatInstanceAuditPlanResV1
// @router /v1/projects/{project_name}/instance_audit_plans [post]
func CreateInstanceAuditPlan(c echo.Context) error {

req := new(CreateInstanceAuditPlanReqV1)
if err := controller.BindAndValidateReq(c, req); err != nil {
return controller.JSONBaseErrorReq(c, err)
Expand Down Expand Up @@ -137,11 +168,18 @@ func CreateInstanceAuditPlan(c echo.Context) error {
return controller.JSONBaseErrorReq(c, errors.New(errors.DataConflict, err))
}

hpc, err := checkAndGenerateHighPriorityParams(auditPlan.Type, inst.DbType, auditPlan.HighPriorityConditions)
if err != nil {
return controller.JSONBaseErrorReq(c, errors.New(errors.DataConflict, err))
}

auditPlans = append(auditPlans, &model.AuditPlanV2{
Type: auditPlan.Type,
RuleTemplateName: ruleTemplateName,
Params: ps,
ActiveStatus: model.ActiveStatusNormal,
Type: auditPlan.Type,
RuleTemplateName: ruleTemplateName,
Params: ps,
HighPriorityParams: hpc,
NeedMarkHighPrioritySQL: auditPlan.NeedMarkHighPrioritySQL,
ActiveStatus: model.ActiveStatusNormal,
})
}

Expand Down Expand Up @@ -288,17 +326,25 @@ func UpdateInstanceAuditPlan(c echo.Context) error {
if err != nil {
return controller.JSONBaseErrorReq(c, errors.New(errors.DataConflict, err))
}
hpc, err := checkAndGenerateHighPriorityParams(auditPlan.Type, dbAuditPlans.DBType, auditPlan.HighPriorityConditions)
if err != nil {
return controller.JSONBaseErrorReq(c, errors.New(errors.DataConflict, err))
}
res := &model.AuditPlanV2{
Type: auditPlan.Type,
RuleTemplateName: ruleTemplateName,
Params: ps,
InstanceAuditPlanID: dbAuditPlans.ID,
Type: auditPlan.Type,
RuleTemplateName: ruleTemplateName,
Params: ps,
HighPriorityParams: hpc,
NeedMarkHighPrioritySQL: auditPlan.NeedMarkHighPrioritySQL,
InstanceAuditPlanID: dbAuditPlans.ID,
}

// if the data exists in the database, update the data; if it does not exist, insert the data.
if dbAuditPlan, ok := dbAuditPlansMap[auditPlan.Type]; ok {
dbAuditPlan.RuleTemplateName = res.RuleTemplateName
dbAuditPlan.Params = res.Params
dbAuditPlan.HighPriorityParams = res.HighPriorityParams
dbAuditPlan.NeedMarkHighPrioritySQL = res.NeedMarkHighPrioritySQL
result := dbAuditPlan
resultAuditPlans = append(resultAuditPlans, result)
} else {
Expand Down Expand Up @@ -549,9 +595,11 @@ type InstanceAuditPlanDetailResV1 struct {
}

type AuditPlanRes struct {
RuleTemplateName string `json:"rule_template_name" from:"rule_template_name" example:"default_MySQL"`
Type AuditPlanTypeResBase `json:"audit_plan_type" form:"audit_plan_type"`
Params []AuditPlanParamResV1 `json:"audit_plan_params" valid:"dive,required"`
RuleTemplateName string `json:"rule_template_name" from:"rule_template_name" example:"default_MySQL"`
Type AuditPlanTypeResBase `json:"audit_plan_type" form:"audit_plan_type"`
Params []AuditPlanParamResV1 `json:"audit_plan_params" valid:"dive,required"`
NeedMarkHighPrioritySQL bool `json:"need_mark_high_priority_sql"`
HighPriorityConditions []HighPriorityCondition `json:"high_priority_conditions"`
}

// @Summary 获取实例扫描任务详情
Expand Down Expand Up @@ -626,6 +674,31 @@ func ConvertAuditPlansToRes(auditPlans []*model.AuditPlanV2) ([]AuditPlanRes, er
resAuditPlan.Params = paramsRes
}

if v.HighPriorityParams != nil && len(v.HighPriorityParams) > 0 {
hppParamsRes := make([]HighPriorityCondition, len(v.HighPriorityParams))
for i, hpp := range v.HighPriorityParams {
for _, metaHpp := range meta.HighPriorityParams {
if metaHpp.Key != hpp.Key {
continue
}
highParamRes := HighPriorityCondition{
Key: metaHpp.Key,
Desc: metaHpp.Desc,
Value: hpp.Value,
Type: string(metaHpp.Type),
Operator: Operator{
Value: string(hpp.Operator.Value),
EnumsValue: metaHpp.Operator.EnumsValue,
},
}
hppParamsRes[i] = highParamRes
break
}
}
resAuditPlan.HighPriorityConditions = hppParamsRes
resAuditPlan.NeedMarkHighPrioritySQL = v.NeedMarkHighPrioritySQL
}

resAuditPlans = append(resAuditPlans, resAuditPlan)
}
return resAuditPlans, nil
Expand Down
2 changes: 1 addition & 1 deletion sqle/api/controller/v1/report_push_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func GetReportPushConfigList(c echo.Context) error {
PushFrequencyCron: reportPushConfig.PushFrequencyCron,
PushUserType: reportPushConfig.PushUserType,
PushUserList: reportPushConfig.PushUserList,
LastPushTime: reportPushConfig.LastPushTime,
LastPushTime: reportPushConfig.ReportPushConfigRecord.LastPushTime,
})
}
return c.JSON(http.StatusOK, GetReportPushConfigsListResV1{
Expand Down
6 changes: 3 additions & 3 deletions sqle/api/controller/v1/sql_manage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type GetSqlManageListReq struct {
FilterDbType *string `query:"filter_db_type" json:"filter_db_type,omitempty"`
FilterRuleName *string `query:"filter_rule_name" json:"filter_rule_name,omitempty"`
FilterBusiness *string `query:"filter_business" json:"filter_business,omitempty"`
FilterPriority *string `query:"filter_priority" json:"filter_priority,omitempty" enums:"hight,low"`
FilterPriority *string `query:"filter_priority" json:"filter_priority,omitempty" enums:"high,low"`
FuzzySearchEndpoint *string `query:"fuzzy_search_endpoint" json:"fuzzy_search_endpoint,omitempty"`
FuzzySearchSchemaName *string `query:"fuzzy_search_schema_name" json:"fuzzy_search_schema_name,omitempty"`
SortField *string `query:"sort_field" json:"sort_field,omitempty" valid:"omitempty,oneof=first_appear_timestamp last_receive_timestamp fp_count" enums:"first_appear_timestamp,last_receive_timestamp,fp_count"`
Expand Down Expand Up @@ -128,7 +128,7 @@ type ExportSqlManagesReq struct {
FilterStatus *string `query:"filter_status" json:"filter_status,omitempty"`
FilterDbType *string `query:"filter_db_type" json:"filter_db_type,omitempty"`
FilterRuleName *string `query:"filter_rule_name" json:"filter_rule_name,omitempty"`
FilterPriority *string `query:"filter_priority" json:"filter_priority,omitempty" enums:"hight,low"`
FilterPriority *string `query:"filter_priority" json:"filter_priority,omitempty" enums:"high,low"`
FuzzySearchEndpoint *string `query:"fuzzy_search_endpoint" json:"fuzzy_search_endpoint,omitempty"`
FuzzySearchSchemaName *string `query:"fuzzy_search_schema_name" json:"fuzzy_search_schema_name,omitempty"`
SortField *string `query:"sort_field" json:"sort_field,omitempty" valid:"omitempty,oneof=first_appear_timestamp last_receive_timestamp fp_count" enums:"first_appear_timestamp,last_receive_timestamp,fp_count"`
Expand All @@ -145,7 +145,7 @@ type ExportSqlManagesReq struct {
// @Param fuzzy_search_sql_fingerprint query string false "fuzzy search sql fingerprint"
// @Param filter_assignee query string false "assignee"
// @Param filter_business query string false "business"
// @Param filter_priority query string false "priority" Enums(hight,low)
// @Param filter_priority query string false "priority" Enums(high,low)
// @Param filter_instance_id query string false "instance id"
// @Param filter_source query string false "source" Enums(audit_plan,sql_audit_record)
// @Param filter_audit_level query string false "audit level" Enums(normal,notice,warn,error)
Expand Down
2 changes: 1 addition & 1 deletion sqle/api/controller/v2/sql_manage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ type SqlManage struct {
// @Param filter_rule_name query string false "rule name"
// @Param filter_db_type query string false "db type"
// @Param filter_business query string false "business"
// @Param filter_priority query string false "priority" Enums(hight,low)
// @Param filter_priority query string false "priority" Enums(high,low)
// @Param fuzzy_search_endpoint query string false "fuzzy search endpoint"
// @Param fuzzy_search_schema_name query string false "fuzzy search schema name"
// @Param sort_field query string false "sort field" Enums(first_appear_timestamp,last_receive_timestamp,fp_count)
Expand Down
14 changes: 14 additions & 0 deletions sqle/dms/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,20 @@ func GetPorjectUIDByName(ctx context.Context, projectName string, needActive ...
}
return project.ProjectUid, nil
}
func GetProjectByID(ProjectId string) (project dmsV1.ListProject, err error) {
ret, _, err := dmsobject.ListProjects(context.TODO(), GetDMSServerAddress(), dmsV1.ListProjectReq{
PageSize: 1,
PageIndex: 1,
FilterByUID: ProjectId,
})
if err != nil {
return project, err
}
if len(ret) > 0 && ret[0] != nil {
project = *ret[0]
}
return project, nil
}

func GetPorjectByName(ctx context.Context, projectName string) (project *dmsV1.ListProject, err error) {
ret, total, err := dmsobject.ListProjects(ctx, controller.GetDMSServerAddress(), dmsV1.ListProjectReq{
Expand Down
20 changes: 11 additions & 9 deletions sqle/model/instance_audit_plan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,14 @@ func (s *Storage) GetLatestAuditPlanRecordsV2() ([]*AuditPlanDetail, error) {
type AuditPlanV2 struct {
Model

InstanceAuditPlanID uint `json:"instance_audit_plan_id" gorm:"not null"`
Type string `json:"type" gorm:"type:varchar(255)"`
RuleTemplateName string `json:"rule_template_name" gorm:"type:varchar(255)"`
Params params.Params `json:"params" gorm:"type:varchar(1000)"`
ActiveStatus string `json:"active_status" gorm:"type:varchar(255)"`
LastCollectionTime *time.Time `json:"last_collection_time" gorm:"type:datetime(3)"`
InstanceAuditPlanID uint `json:"instance_audit_plan_id" gorm:"not null"`
Type string `json:"type" gorm:"type:varchar(255)"`
RuleTemplateName string `json:"rule_template_name" gorm:"type:varchar(255)"`
Params params.Params `json:"params" gorm:"type:varchar(1000)"`
HighPriorityParams params.ParamsWithOperator `json:"high_priority_params" gorm:"type:varchar(1000)"`
NeedMarkHighPrioritySQL bool `json:"need_mark_high_priority_sql"`
ActiveStatus string `json:"active_status" gorm:"type:varchar(255)"`
LastCollectionTime *time.Time `json:"last_collection_time" gorm:"type:datetime(3)"`

AuditPlanSQLs []*SQLManageRecord `gorm:"foreignKey:SourceId"`
}
Expand Down Expand Up @@ -406,10 +408,10 @@ func (s *Storage) RemoveSQLFromQueue(sql *SQLManageQueue) error {
}

func (s *Storage) UpdateManagerSQL(sql *SQLManageRecord) error {
const query = "INSERT INTO `sql_manage_records` (`sql_id`,`source`,`source_id`,`project_id`,`instance_id`,`schema_name`,`sql_fingerprint`, `sql_text`, `info`, `audit_level`, `audit_results`) " +
"VALUES (?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `source` = VALUES(source),`source_id` = VALUES(source_id),`project_id` = VALUES(project_id), `instance_id` = VALUES(instance_id), " +
const query = "INSERT INTO `sql_manage_records` (`sql_id`,`source`,`source_id`,`project_id`,`instance_id`,`schema_name`,`sql_fingerprint`, `sql_text`, `info`, `audit_level`, `audit_results`,`priority`) " +
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `source` = VALUES(source),`source_id` = VALUES(source_id),`project_id` = VALUES(project_id), `instance_id` = VALUES(instance_id), `priority` = VALUES(priority), " +
"`schema_name` = VALUES(`schema_name`), `sql_text` = VALUES(sql_text), `sql_fingerprint` = VALUES(sql_fingerprint), `info`= VALUES(info), `audit_level`= VALUES(audit_level), `audit_results`= VALUES(audit_results)"
return s.db.Exec(query, sql.SQLID, sql.Source, sql.SourceId, sql.ProjectId, sql.InstanceID, sql.SchemaName, sql.SqlFingerprint, sql.SqlText, sql.Info, sql.AuditLevel, sql.AuditResults).Error
return s.db.Exec(query, sql.SQLID, sql.Source, sql.SourceId, sql.ProjectId, sql.InstanceID, sql.SchemaName, sql.SqlFingerprint, sql.SqlText, sql.Info, sql.AuditLevel, sql.AuditResults, sql.Priority).Error
}

func (s *Storage) UpdateManagerSQLStatus(sql *SQLManageRecord) error {
Expand Down
Loading

0 comments on commit 0ea5602

Please sign in to comment.