Merge branch 'feature/formatter_adding_variables'

app/apinto/plugin.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"github.com/eolinker/apinto/drivers/plugins/app"
+	body_record_truncation "github.com/eolinker/apinto/drivers/plugins/body-record-truncation"
 	"github.com/eolinker/apinto/drivers/plugins/cors"
 	data_transform "github.com/eolinker/apinto/drivers/plugins/data-transform"
 	dubbo2_proxy_rewrite "github.com/eolinker/apinto/drivers/plugins/dubbo2-proxy-rewrite"
@@ -10,7 +11,9 @@ import (
 	"github.com/eolinker/apinto/drivers/plugins/gzip"
 	params_check "github.com/eolinker/apinto/drivers/plugins/params-check"
 	"github.com/eolinker/apinto/drivers/plugins/prometheus"
+	request_file_parse "github.com/eolinker/apinto/drivers/plugins/request-file-parse"
 	request_interception "github.com/eolinker/apinto/drivers/plugins/request-interception"
+	response_file_parse "github.com/eolinker/apinto/drivers/plugins/response-file-parse"
 	response_filter "github.com/eolinker/apinto/drivers/plugins/response-filter"
 	response_rewrite_v2 "github.com/eolinker/apinto/drivers/plugins/response-rewrite_v2"
 
@@ -71,12 +74,14 @@ func pluginRegister(extenderRegister eosc.IExtenderDriverRegister) {
 	params_check.Register(extenderRegister)
 	data_transform.Register(extenderRegister)
 	request_interception.Register(extenderRegister)
+	request_file_parse.Register(extenderRegister)
 
 	// 响应处理插件
 	response_rewrite.Register(extenderRegister)
 	response_rewrite_v2.Register(extenderRegister)
 	response_filter.Register(extenderRegister)
 	gzip.Register(extenderRegister)
+	response_file_parse.Register(extenderRegister)
 
 	// 安全相关插件
 	ip_restriction.Register(extenderRegister)
@@ -90,6 +95,7 @@ func pluginRegister(extenderRegister eosc.IExtenderDriverRegister) {
 	prometheus.Register(extenderRegister)
 	monitor.Register(extenderRegister)
 	proxy_mirror.Register(extenderRegister)
+	body_record_truncation.Register(extenderRegister)
 
 	// 计数插件
 	counter.Register(extenderRegister)

drivers/output/fileoutput/config.go

@@ -5,11 +5,17 @@ import (
 )
 
 type Config struct {
-	Scopes    []string             `json:"scopes" label:"作用域"`
-	File      string               `json:"file" yaml:"file" label:"文件名称"`
-	Dir       string               `json:"dir" yaml:"dir" label:"文件存放目录"`
-	Period    string               `json:"period" yaml:"period" enum:"hour,day" label:"日志分割周期"`
-	Expire    int                  `json:"expire" yaml:"expire" label:"日志保存时间" description:"单位：天" default:"7" minimum:"1"`
-	Type      string               `json:"type" yaml:"type" enum:"json,line" label:"输出格式"`
+	Scopes []string `json:"scopes" label:"作用域"`
+	File   string   `json:"file" yaml:"file" label:"文件名称"`
+	Dir    string   `json:"dir" yaml:"dir" label:"文件存放目录"`
+	Period string   `json:"period" yaml:"period" enum:"hour,day" label:"日志分割周期"`
+	Expire int      `json:"expire" yaml:"expire" label:"日志保存时间" description:"单位：天" default:"7" minimum:"1"`
+	Type   string   `json:"type" yaml:"type" enum:"json,line" label:"输出格式"`
+	//BodyConfig BodyConfig           `json:"body_config" yaml:"body_config" label:"请求体/响应体配置" description:"请求体/响应体配置" switch:"type===json"`
 	Formatter eosc.FormatterConfig `json:"formatter" yaml:"formatter" label:"格式化配置"`
 }
+
+//type BodyConfig struct {
+//	BodySize int    `json:"body_size" label:"请求体/响应体截取长度" description:"单位：M" default:"10" minimum:"0"`
+//	BodyCode string `json:"body_code" label:"请求体/响应体编码" enum:"latin,utf8,gbk" default:"utf8"`
+//}

drivers/output/fileoutput/write.go

@@ -2,11 +2,12 @@ package fileoutput
 
 import (
 	"fmt"
+	"net/http"
+
 	"github.com/eolinker/eosc"
 	"github.com/eolinker/eosc/formatter"
 	"github.com/eolinker/eosc/log/filelog"
 	"github.com/eolinker/eosc/router"
-	"net/http"
 
 	"time"
 )
@@ -15,7 +16,6 @@ type FileWriter struct {
 	formatter eosc.IFormatter
 	transport *filelog.FileWriterByPeriod
 	//id        string
-
 	fileHandler http.Handler
 }
 
@@ -37,7 +37,10 @@ func (a *FileWriter) reset(cfg *Config, name string) (err error) {
 	if !has {
 		return errorFormatterType
 	}
-
+	//var extendCfg []byte
+	//if cfg.Type == "json" {
+	//	extendCfg, _ = json.Marshal(cfg.BodyConfig)
+	//}
 	fm, err := factory.Create(cfg.Formatter)
 	if err != nil {
 		return err

drivers/output/httpoutput/http.go

@@ -73,6 +73,7 @@ func create(config *Config) (formatter.ITransport, eosc.IFormatter, error) {
 	if !has {
 		return nil, nil, errFormatterType
 	}
+
 	fm, err := factory.Create(config.Formatter)
 	if err != nil {
 		return nil, nil, err

drivers/output/kafka/producer.go

@@ -32,6 +32,7 @@ func (o *tProducer) reset(cfg *ProducerConfig) (err error) {
 	if !has {
 		return errorFormatterType
 	}
+
 	o.formatter, err = factory.Create(cfg.Formatter)
 
 	if o.producer != nil {

drivers/output/nsq/write.go

@@ -54,6 +54,7 @@ func (n *Writer) reset(config *Config) error {
 	if !has {
 		return errFormatterType
 	}
+
 	fm, err := factory.Create(config.Formatter)
 	if err != nil {
 		return err

drivers/output/prometheus/driver.go

@@ -2,14 +2,15 @@ package prometheus
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/eolinker/apinto/drivers"
 	scope_manager "github.com/eolinker/apinto/scope-manager"
 	"github.com/eolinker/apinto/utils"
 	"github.com/eolinker/eosc"
 	"github.com/eolinker/eosc/router"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"strings"
 )
 
 func Check(v *Config, workers map[eosc.RequireId]eosc.IWorker) error {
@@ -164,7 +165,6 @@ func Create(id, name string, cfg *Config, workers map[eosc.RequireId]eosc.IWorke
 		}
 		metrics[metric.Metric] = m
 	}
-
 	//注册路由
 	p.registry = registry
 	p.handler = promhttp.InstrumentMetricHandler(

drivers/plugins/body-check/body-check.go

@@ -30,7 +30,8 @@ func (b *BodyCheck) DoHttpFilter(ctx http_service.IHttpContext, next eocontext.I
 		if err != nil {
 			return err
 		}
-		bodySize := len([]rune(string(body)))
+		// 计算body大小
+		bodySize := len(body)
 		if !b.isEmpty && bodySize < 1 {
 			ctx.Response().SetStatus(400, "400")
 			ctx.Response().SetBody([]byte("Body is required"))

drivers/plugins/body-record-truncation/config.go

@@ -0,0 +1,5 @@
+package body_record_truncation
+
+type Config struct {
+	BodySize int64 `json:"body_size" label:"截断大小"`
+}

drivers/plugins/body-record-truncation/executor.go

@@ -0,0 +1,73 @@
+package body_record_truncation
+
+import (
+	"github.com/eolinker/apinto/drivers"
+	"github.com/eolinker/eosc"
+	"github.com/eolinker/eosc/eocontext"
+	http_service "github.com/eolinker/eosc/eocontext/http-context"
+)
+
+var ()
+
+var _ http_service.HttpFilter = (*executor)(nil)
+var _ eocontext.IFilter = (*executor)(nil)
+var _ eosc.IWorker = (*executor)(nil)
+
+type executor struct {
+	drivers.WorkerBase
+	bodySize int64
+}
+
+func (e *executor) DoFilter(ctx eocontext.EoContext, next eocontext.IChain) (err error) {
+	return http_service.DoHttpFilter(e, ctx, next)
+}
+
+func (e *executor) DoHttpFilter(ctx http_service.IHttpContext, next eocontext.IChain) (err error) {
+	if ctx.Request().Method() == "POST" || ctx.Request().Method() == "PUT" || ctx.Request().Method() == "PATCH" {
+		if e.bodySize != 0 && int64(ctx.Request().ContentLength()) > e.bodySize {
+			// 当请求体大小大于限制时，截断请求体
+			entry := ctx.GetEntry()
+			body := entry.Read("ctx_request_body")
+			v, _ := body.(string)
+			ctx.SetLabel("request_body", v[:e.bodySize])
+			ctx.WithValue("request_body_complete", 0)
+		} else {
+			ctx.WithValue("request_body_complete", 1)
+		}
+	}
+	if next != nil {
+		err = next.DoChain(ctx)
+	}
+	if e.bodySize != 0 && int64(ctx.Response().ContentLength()) > e.bodySize {
+		// 当响应体大小大于限制时，截断响应体
+		entry := ctx.GetEntry()
+		body := entry.Read("ctx_response_body")
+		v, _ := body.(string)
+		ctx.SetLabel("response_body", v[:e.bodySize])
+		ctx.WithValue("response_body_complete", 0)
+	} else {
+		ctx.WithValue("response_body_complete", 1)
+	}
+	return err
+}
+
+func (e *executor) Destroy() {
+	return
+}
+
+func (e *executor) Start() error {
+	return nil
+}
+
+func (e *executor) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWorker) error {
+	return nil
+}
+
+func (e *executor) Stop() error {
+	e.Destroy()
+	return nil
+}
+
+func (e *executor) CheckSkill(skill string) bool {
+	return http_service.FilterSkillName == skill
+}

drivers/plugins/body-record-truncation/factory.go

@@ -0,0 +1,26 @@
+package body_record_truncation
+
+import (
+	"github.com/eolinker/apinto/drivers"
+	"github.com/eolinker/eosc"
+)
+
+const (
+	Name = "body-record-truncation"
+)
+
+func Register(register eosc.IExtenderDriverRegister) {
+	register.RegisterExtenderDriver(Name, NewFactory())
+}
+
+func NewFactory() eosc.IExtenderDriverFactory {
+	return drivers.NewFactory[Config](Create)
+}
+
+func Create(id, name string, conf *Config, workers map[eosc.RequireId]eosc.IWorker) (eosc.IWorker, error) {
+
+	return &executor{
+		WorkerBase: drivers.Worker(id, name),
+		bodySize:   conf.BodySize << 20,
+	}, nil
+}

drivers/plugins/gRPC-to-http/complete.go

@@ -126,7 +126,8 @@ func (h *complete) Complete(org eocontext.EoContext) error {
 			request.URI().SetHost(targetHost)
 		}
 		response := fasthttp.AcquireResponse()
-		lastErr = fasthttp_client.ProxyTimeout(scheme, node, request, response, timeOut)
+
+		_, lastErr = fasthttp_client.ProxyTimeout(scheme, node, request, response, timeOut)
 		if lastErr == nil {
 			return newGRPCResponse(ctx, response, methodDesc)
 		}

drivers/plugins/request-file-parse/config.go

@@ -0,0 +1,8 @@
+package request_file_parse
+
+type Config struct {
+	FileKey       string   `json:"file_key" label:"文件Key"`
+	FileSuffix    []string `json:"file_suffix" label:"文件有效后缀列表"`
+	LargeWarn     int64    `json:"large_warn" label:"文件大小警告阈值"`
+	LargeWarnText string   `json:"large_warn_text" label:"文件大小警告标签值"`
+}