Skip to content

Commit

Permalink
Merge pull request #1930 from actiontech/knowledge
Browse files Browse the repository at this point in the history
Knowledge
  • Loading branch information
sjjian authored Oct 19, 2023
2 parents aab1d16 + 66690d7 commit 80ca1f6
Show file tree
Hide file tree
Showing 5 changed files with 386 additions and 16 deletions.
1 change: 1 addition & 0 deletions sqle/api/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,7 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config config.SqleConfi
// sql audit
v1Router.POST("/sql_audit", v1.DirectAudit)
v1Router.POST("/audit_files", v1.DirectAuditFiles)
v2Router.POST("/audit_files", v2.DirectAuditFiles)
v1Router.GET("/sql_analysis", v1.DirectGetSQLAnalysis)

// enterprise customized apis
Expand Down
131 changes: 126 additions & 5 deletions sqle/api/controller/v2/sql_audit.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package v2

import (
e "errors"
"fmt"
"net/http"
"strings"

parser "github.com/actiontech/mybatis-mapper-2-sql"
"github.com/actiontech/sqle/sqle/api/controller"
v1 "github.com/actiontech/sqle/sqle/api/controller/v1"
"github.com/actiontech/sqle/sqle/errors"
"github.com/actiontech/sqle/sqle/log"
"github.com/actiontech/sqle/sqle/model"
"github.com/actiontech/sqle/sqle/server"
Expand All @@ -28,10 +32,10 @@ type AuditResDataV2 struct {
}

type AuditSQLResV2 struct {
Number uint `json:"number"`
ExecSQL string `json:"exec_sql"`
AuditResult []*AuditResult `json:"audit_result"`
AuditLevel string `json:"audit_level"`
Number uint `json:"number"`
ExecSQL string `json:"exec_sql"`
AuditResult []AuditResult `json:"audit_result"`
AuditLevel string `json:"audit_level"`
}

type DirectAuditResV2 struct {
Expand Down Expand Up @@ -94,7 +98,7 @@ func convertTaskResultToAuditResV2(task *model.Task) *AuditResDataV2 {
results[i] = AuditSQLResV2{
Number: sql.Number,
ExecSQL: sql.Content,
AuditResult: ar,
AuditResult: convertAuditResultToAuditResV2(sql.AuditResults),
AuditLevel: sql.AuditLevel,
}

Expand All @@ -106,3 +110,120 @@ func convertTaskResultToAuditResV2(task *model.Task) *AuditResDataV2 {
SQLResults: results,
}
}

type DirectAuditFileReqV2 struct {
InstanceType string `json:"instance_type" form:"instance_type" example:"MySQL" valid:"required"`
// 调用方不应该关心SQL是否被完美的拆分成独立的条目, 拆分SQL由SQLE实现
// 每个数组元素是一个文件内容
FileContents []string `json:"file_contents" form:"file_contents" example:"select * from t1; select * from t2;"`
SQLType string `json:"sql_type" form:"sql_type" example:"sql" enums:"sql,mybatis," valid:"omitempty,oneof=sql mybatis"`
ProjectName string `json:"project_name" form:"project_name" example:"project1" valid:"required"`
InstanceName *string `json:"instance_name" form:"instance_name" example:"instance1"`
SchemaName *string `json:"schema_name" form:"schema_name" example:"schema1"`
}

// @Summary 直接从文件内容提取SQL并审核,SQL文件暂时只支持一次解析一个文件
// @Description Direct audit sql from SQL files and MyBatis files
// @Id directAuditFilesV2
// @Tags sql_audit
// @Security ApiKeyAuth
// @Param req body v2.DirectAuditFileReqV2 true "files that should be audited"
// @Success 200 {object} v2.DirectAuditResV2
// @router /v2/audit_files [post]
func DirectAuditFiles(c echo.Context) error {
req := new(DirectAuditFileReqV2)
err := controller.BindAndValidateReq(c, req)
if err != nil {
return err
}

user := controller.GetUserName(c)
s := model.GetStorage()
if yes, err := s.IsProjectMember(user, req.ProjectName); err != nil {
return controller.JSONBaseErrorReq(c, fmt.Errorf("check privilege failed: %v", err))
} else if !yes {
return controller.JSONBaseErrorReq(c, errors.New(errors.ErrAccessDeniedError, e.New("you are not the project member")))
}

if len(req.FileContents) <= 0 {
return controller.JSONBaseErrorReq(c, e.New("file_contents is required"))
}

sqls := ""
if req.SQLType == v1.SQLTypeMyBatis {
ss, err := parser.ParseXMLs(req.FileContents, false)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
sqls = strings.Join(ss, ";")
} else {
// sql文件暂时只支持一次解析一个文件
sqls = req.FileContents[0]
}

l := log.NewEntry().WithField("api", "[post]/v2/audit_files")

var instance *model.Instance
var exist bool
if req.InstanceName != nil {
instance, exist, err = s.GetInstanceByNameAndProjectName(*req.InstanceName, req.ProjectName)
if err != nil {
return controller.JSONBaseErrorReq(c, err)
}
if !exist {
return controller.JSONBaseErrorReq(c, v1.ErrInstanceNotExist)
}
}

var schemaName string
if req.SchemaName != nil {
schemaName = *req.SchemaName
}

var task *model.Task
if instance != nil && schemaName != "" {
task, err = server.DirectAuditByInstance(l, sqls, schemaName, instance)
} else {
task, err = server.AuditSQLByDBType(l, sqls, req.InstanceType, nil, "")
}
if err != nil {
l.Errorf("audit sqls failed: %v", err)
return controller.JSONBaseErrorReq(c, v1.ErrDirectAudit)
}

return c.JSON(http.StatusOK, DirectAuditResV2{
BaseRes: controller.BaseRes{},
Data: convertFileAuditTaskResultToAuditResV2(task),
})
}

func convertFileAuditTaskResultToAuditResV2(task *model.Task) *AuditResDataV2 {
results := make([]AuditSQLResV2, len(task.ExecuteSQLs))
for i, sql := range task.ExecuteSQLs {
results[i] = AuditSQLResV2{
Number: sql.Number,
ExecSQL: sql.Content,
AuditResult: convertAuditResultToAuditResV2(sql.AuditResults),
AuditLevel: sql.AuditLevel,
}

}
return &AuditResDataV2{
AuditLevel: task.AuditLevel,
Score: task.Score,
PassRate: task.PassRate,
SQLResults: results,
}
}

func convertAuditResultToAuditResV2(auditResults model.AuditResults) []AuditResult {
ar := make([]AuditResult, len(auditResults))
for i := range auditResults {
ar[i] = AuditResult{
Level: auditResults[i].Level,
Message: auditResults[i].Message,
RuleName: auditResults[i].RuleName,
}
}
return ar
}
100 changes: 96 additions & 4 deletions sqle/docs/docs.go
Original file line number Diff line number Diff line change
Expand Up @@ -8775,6 +8775,40 @@ var doc = `{
}
}
},
"/v2/audit_files": {
"post": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "Direct audit sql from SQL files and MyBatis files",
"tags": [
"sql_audit"
],
"summary": "直接从文件内容提取SQL并审核,SQL文件暂时只支持一次解析一个文件",
"operationId": "directAuditFilesV2",
"parameters": [
{
"description": "files that should be audited",
"name": "req",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/v2.DirectAuditFileReqV2"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/v2.DirectAuditResV2"
}
}
}
}
},
"/v2/configurations/drivers": {
"get": {
"security": [
Expand Down Expand Up @@ -9898,6 +9932,26 @@ var doc = `{
}
}
},
"model.AuditResult": {
"type": "object",
"properties": {
"level": {
"type": "string"
},
"message": {
"type": "string"
},
"rule_name": {
"type": "string"
}
}
},
"model.AuditResults": {
"type": "array",
"items": {
"$ref": "#/definitions/model.AuditResult"
}
},
"v1.AuditPlanCount": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -16400,10 +16454,8 @@ var doc = `{
"type": "string"
},
"audit_result": {
"type": "array",
"items": {
"$ref": "#/definitions/v2.AuditResult"
}
"type": "object",
"$ref": "#/definitions/model.AuditResults"
},
"exec_sql": {
"type": "string"
Expand Down Expand Up @@ -16564,6 +16616,46 @@ var doc = `{
}
}
},
"v2.DirectAuditFileReqV2": {
"type": "object",
"properties": {
"file_contents": {
"description": "调用方不应该关心SQL是否被完美的拆分成独立的条目, 拆分SQL由SQLE实现\n每个数组元素是一个文件内容",
"type": "array",
"items": {
"type": "string"
},
"example": [
"select * from t1; select * from t2;"
]
},
"instance_name": {
"type": "string",
"example": "instance1"
},
"instance_type": {
"type": "string",
"example": "MySQL"
},
"project_name": {
"type": "string",
"example": "project1"
},
"schema_name": {
"type": "string",
"example": "schema1"
},
"sql_type": {
"type": "string",
"enum": [
"sql",
"mybatis",
""
],
"example": "sql"
}
}
},
"v2.DirectAuditReqV2": {
"type": "object",
"properties": {
Expand Down
Loading

0 comments on commit 80ca1f6

Please sign in to comment.