Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

optimize: audit plan gen sql id #2765

Merged
merged 1 commit into from
Nov 19, 2024
Merged

optimize: audit plan gen sql id #2765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions sqle/server/auditplan/sql_info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ import (
type SQLV2 struct {
SQLId string
// from audit plan
Source string
SourceId string
ProjectId string
InstanceID string
Source string
SourceId string // 实例扫描任务id: instance audit plan id
ProjectId string
InstanceID string
AuditPlanId string // 扫描任务id: audit plan id

// from collect
SQLContent string
Expand All @@ -31,14 +32,14 @@ func (s *SQLV2) GenSQLId() {
Schema string
InstID string
Source string
ApID string
AuditPlanID string
}{
ProjectId: s.ProjectId,
Fingerprint: s.Fingerprint,
Schema: s.SchemaName,
InstID: s.InstanceID,
Source: s.Source,
ApID: s.SourceId,
AuditPlanID: s.AuditPlanId,
},
)
if err != nil {
Expand Down
13 changes: 7 additions & 6 deletions sqle/server/auditplan/task_type_mysql_audit_log_ali.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,13 @@ func (at *MySQLAuditLogAliTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPlan
cache := NewSQLV2Cache()
for _, sql := range slowSqls {
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
}
fp, err := util.Fingerprint(sql.sql, true)
if err != nil {
Expand Down
16 changes: 9 additions & 7 deletions sqle/server/auditplan/task_type_mysql_processlist.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package auditplan
import (
"context"
"fmt"
"github.com/actiontech/sqle/sqle/errors"
"strconv"
"time"

"github.com/actiontech/sqle/sqle/errors"

"github.com/actiontech/sqle/sqle/dms"
"github.com/actiontech/sqle/sqle/driver/mysql/executor"
"github.com/actiontech/sqle/sqle/driver/mysql/util"
Expand Down Expand Up @@ -110,12 +111,13 @@ func (at *MySQLProcessListTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPlan
for i := range res {
query := res[i]["info"].String
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: res[i]["db"].String,
SQLContent: query,
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: res[i]["db"].String,
SQLContent: query,
}
fp, err := util.Fingerprint(query, true)
if err != nil {
Expand Down
34 changes: 18 additions & 16 deletions sqle/server/auditplan/task_type_mysql_schema_meta.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/actiontech/sqle/sqle/errors"
"strconv"
"time"

"github.com/actiontech/sqle/sqle/errors"

"github.com/actiontech/sqle/sqle/dms"
"github.com/actiontech/sqle/sqle/driver/mysql/executor"
driverV2 "github.com/actiontech/sqle/sqle/driver/v2"
Expand Down Expand Up @@ -161,25 +162,25 @@ func (at *BaseSchemaMetaTaskV2) Metrics() []string {
func (at *BaseSchemaMetaTaskV2) genSQLId(sql *SQLV2) string {
md5Json, err := json.Marshal(
struct {
ProjectId string
Schema string
InstName string
Source string
ApID string
MetaName string
MetaType string
ProjectId string
Schema string
InstName string
Source string
AuditPlanId string
MetaName string
MetaType string
}{
ProjectId: sql.ProjectId,
Schema: sql.SchemaName,
InstName: sql.InstanceID,
Source: sql.Source,
ApID: sql.SourceId,
MetaName: sql.Info.Get(MetricNameMetaName).String(),
MetaType: sql.Info.Get(MetricNameMetaType).String(),
ProjectId: sql.ProjectId,
Schema: sql.SchemaName,
InstName: sql.InstanceID,
Source: sql.Source,
AuditPlanId: sql.AuditPlanId,
MetaName: sql.Info.Get(MetricNameMetaName).String(),
MetaType: sql.Info.Get(MetricNameMetaType).String(),
},
)
if err != nil { // todo: 处理错误
return utils.Md5String(fmt.Sprintf("%s:%s:%s:%s", sql.SourceId, sql.SchemaName, sql.Info.Get(MetricNameMetaName).String(), sql.Info.Get(MetricNameMetaType).String()))
return utils.Md5String(fmt.Sprintf("%s:%s:%s:%s", sql.AuditPlanId, sql.SchemaName, sql.Info.Get(MetricNameMetaName).String(), sql.Info.Get(MetricNameMetaType).String()))
} else {
return utils.Md5String(string(md5Json))
}
Expand Down Expand Up @@ -239,6 +240,7 @@ func (at *BaseSchemaMetaTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPlan,
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.SchemaName,
Expand Down
13 changes: 7 additions & 6 deletions sqle/server/auditplan/task_type_mysql_slowlog_ali.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,13 @@ func (at *MySQLSlowLogAliTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPlan,
cache := NewSQLV2Cache()
for _, sql := range slowSqls {
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
}
fp, err := util.Fingerprint(sql.sql, true)
if err != nil {
Expand Down
13 changes: 7 additions & 6 deletions sqle/server/auditplan/task_type_mysql_slowlog_baidu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,12 +150,13 @@ func (at *MySQLSlowLogBaiduTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPla
cache := NewSQLV2Cache()
for _, sql := range slowSqls {
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
}
fp, err := util.Fingerprint(sql.sql, true)
if err != nil {
Expand Down
13 changes: 7 additions & 6 deletions sqle/server/auditplan/task_type_mysql_slowlog_huawei.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,12 +169,13 @@ func (at *MySQLSlowLogHuaweiTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPl
cache := NewSQLV2Cache()
for _, sql := range slowSqls {
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: sql.schema,
SQLContent: sql.sql,
}
fp, err := util.Fingerprint(sql.sql, true)
if err != nil {
Expand Down
4 changes: 3 additions & 1 deletion sqle/server/auditplan/task_type_oracle_topsql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package auditplan
import (
"context"
"fmt"
"github.com/actiontech/sqle/sqle/errors"
"strconv"
"time"

"github.com/actiontech/sqle/sqle/errors"

"github.com/actiontech/sqle/sqle/dms"
"github.com/actiontech/sqle/sqle/locale"
"github.com/actiontech/sqle/sqle/model"
Expand Down Expand Up @@ -135,6 +136,7 @@ func (at *OracleTopSQLTaskV2) ExtractSQL(logger *logrus.Entry, ap *AuditPlan, pe
sqlV2 := &SQLV2{
Source: ap.Type,
SourceId: strconv.FormatUint(uint64(ap.InstanceAuditPlanId), 10),
AuditPlanId: strconv.FormatUint(uint64(ap.ID), 10),
ProjectId: ap.ProjectId,
InstanceID: ap.InstanceID,
SchemaName: "", // todo: top sql 未采集schema, 需要填充
Expand Down
Loading