From 3c6893b081aa629a6497795268d9e0eb9672003f Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Mon, 5 Jun 2023 11:23:43 +0800 Subject: [PATCH 01/31] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20mod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 66 ++++++++++------------------------------------------------ 1 file changed, 11 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 1cf1654b..0aee34f9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 - github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/consul/api v1.9.1 github.com/influxdata/influxdb-client-go/v2 v2.12.1 github.com/jhump/protoreflect v1.14.1 @@ -25,34 +24,18 @@ require ( github.com/urfave/cli/v2 v2.23.4 github.com/valyala/fasthttp v1.42.0 golang.org/x/crypto v0.7.0 - golang.org/x/net v0.8.0 google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 ) require ( - cloud.google.com/go v0.65.0 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.1 // indirect github.com/RoaringBitmap/roaring v0.7.1 // indirect github.com/Workiva/go-datastructures v1.0.52 // indirect - github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 // indirect - github.com/alibaba/sentinel-golang v1.0.4 // indirect - github.com/aliyun/alibaba-cloud-sdk-go v1.61.1704 // indirect github.com/apache/dubbo-getty v1.4.8 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect - github.com/buger/jsonparser v1.1.1 // indirect - github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect - github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect - github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect github.com/creasty/defaults v1.5.2 // indirect - github.com/dubbogo/go-zookeeper v1.0.4-0.20211212162352-f9d2183d89d5 // indirect - github.com/dubbogo/grpc-go v1.42.9 // indirect github.com/dubbogo/triple v1.1.8 // indirect - github.com/emicklei/go-restful/v3 v3.7.4 // indirect - github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect - github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect - github.com/go-co-op/gocron v1.9.0 // indirect github.com/go-kit/log v0.1.0 // indirect github.com/go-logfmt/logfmt v0.5.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -61,16 +44,9 @@ require ( github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.11.0 // indirect - github.com/go-resty/resty/v2 v2.7.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/hashicorp/vault/sdk v0.3.0 // indirect github.com/jinzhu/copier v0.3.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/knadh/koanf v1.4.1 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -79,37 +55,18 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/nacos-group/nacos-sdk-go v1.1.1 // indirect - github.com/nacos-group/nacos-sdk-go/v2 v2.1.2 // indirect github.com/natefinch/lumberjack v2.0.0+incompatible // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml v1.7.0 // indirect - github.com/polarismesh/polaris-go v1.1.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/statsd_exporter v0.21.0 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect github.com/shirou/gopsutil/v3 v3.22.2 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.2.2 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.1 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect - github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect - github.com/uber/jaeger-lib v2.4.1+incompatible // indirect - github.com/ugorji/go/codec v1.2.6 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - github.com/zouyx/agollo/v3 v3.4.5 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - google.golang.org/appengine v1.6.6 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + golang.org/x/net v0.8.0 // indirect ) require ( @@ -130,10 +87,9 @@ require ( github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/fatih/color v1.9.0 // indirect - github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 github.com/google/btree v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect @@ -177,13 +133,13 @@ require ( github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/v2 v2.305.5 // indirect - go.etcd.io/etcd/client/v3 v3.5.5 // indirect - go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/raft/v3 v3.5.5 // indirect - go.etcd.io/etcd/server/v3 v3.5.5 // indirect + go.etcd.io/etcd/api/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/v2 v2.305.7 // indirect + go.etcd.io/etcd/client/v3 v3.5.7 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.7 // indirect + go.etcd.io/etcd/raft/v3 v3.5.7 // indirect + go.etcd.io/etcd/server/v3 v3.5.7 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0 // indirect go.opentelemetry.io/otel v1.7.0 // indirect go.opentelemetry.io/otel/trace v1.7.0 // indirect @@ -198,4 +154,4 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect ) -//replace github.com/eolinker/eosc => ../eosc +replace github.com/eolinker/eosc => ../eosc From 12213b7c7920df2b2e2f485161332f7a0860a7a1 Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Mon, 5 Jun 2023 15:47:49 +0800 Subject: [PATCH 02/31] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E4=BD=BF=E7=94=A8eosc=E7=9A=84=E6=A0=87=E5=87=86=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/fileoutput/write.go | 13 ++++++------ output/file-transport/file.go | 1 + output/file-transport/transporter.go | 30 ---------------------------- 3 files changed, 8 insertions(+), 36 deletions(-) delete mode 100644 output/file-transport/transporter.go diff --git a/drivers/output/fileoutput/write.go b/drivers/output/fileoutput/write.go index f1f4659a..64e4456c 100644 --- a/drivers/output/fileoutput/write.go +++ b/drivers/output/fileoutput/write.go @@ -1,14 +1,15 @@ package fileoutput import ( - file_transport "github.com/eolinker/apinto/output/file-transport" "github.com/eolinker/eosc" "github.com/eolinker/eosc/formatter" + "github.com/eolinker/eosc/log/filelog" + "time" ) type FileWriter struct { formatter eosc.IFormatter - transport *file_transport.FileWriterByPeriod + transport *filelog.FileWriterByPeriod //id string } @@ -37,14 +38,14 @@ func (a *FileWriter) reset(cfg *Config) (err error) { } transport := a.transport - c := &file_transport.Config{ + c := &filelog.Config{ Dir: cfg.Dir, File: cfg.File, - Expire: cfg.Expire, - Period: file_transport.ParsePeriod(cfg.Period), + Expire: time.Duration(cfg.Expire) * 24 * time.Hour, + Period: filelog.ParsePeriod(cfg.Period), } if transport == nil { - transport = file_transport.NewFileWriteByPeriod(c) + transport = filelog.NewFileWriteByPeriod(c) } else { transport.Reset(c) } diff --git a/output/file-transport/file.go b/output/file-transport/file.go index bf93e088..823227e7 100644 --- a/output/file-transport/file.go +++ b/output/file-transport/file.go @@ -43,6 +43,7 @@ func (w *FileController) dropHistory() { _ = os.Remove(f) } } + } } } diff --git a/output/file-transport/transporter.go b/output/file-transport/transporter.go deleted file mode 100644 index b3570c84..00000000 --- a/output/file-transport/transporter.go +++ /dev/null @@ -1,30 +0,0 @@ -package file_transport - -import "github.com/eolinker/eosc/formatter" - -//Transporter filelog-Transporter结构 -type Transporter struct { - writer *FileWriterByPeriod -} - -func (t *Transporter) Write(bytes []byte) error { - _, err := t.writer.Write(bytes) - return err -} - -//Close 关闭 -func (t *Transporter) Close() error { - t.writer.Close() - return nil -} - -//NewtTransporter 创建file-Transporter -func NewtTransporter(cfg *Config) formatter.ITransport { - - fileWriterByPeriod := NewFileWriteByPeriod(cfg) - - return &Transporter{ - writer: fileWriterByPeriod, - } - -} From b5177689e0befa223792a53d02cf7ee2f012cc16 Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Mon, 5 Jun 2023 15:48:17 +0800 Subject: [PATCH 03/31] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=9A=84=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- output/file-transport/config.go | 16 --- output/file-transport/file.go | 49 ------- output/file-transport/period.go | 71 ---------- output/file-transport/writer.go | 241 -------------------------------- 4 files changed, 377 deletions(-) delete mode 100644 output/file-transport/config.go delete mode 100644 output/file-transport/file.go delete mode 100644 output/file-transport/period.go delete mode 100644 output/file-transport/writer.go diff --git a/output/file-transport/config.go b/output/file-transport/config.go deleted file mode 100644 index c494cfef..00000000 --- a/output/file-transport/config.go +++ /dev/null @@ -1,16 +0,0 @@ -package file_transport - -//Config filelog-Transporter所需配置 -type Config struct { - Dir string - File string - Expire int - Period LogPeriod -} - -func (c *Config) IsUpdate(cfg *Config) bool { - if cfg.File != c.File || cfg.Dir != c.Dir || cfg.Period != c.Period || cfg.Expire != c.Expire { - return true - } - return false -} diff --git a/output/file-transport/file.go b/output/file-transport/file.go deleted file mode 100644 index 823227e7..00000000 --- a/output/file-transport/file.go +++ /dev/null @@ -1,49 +0,0 @@ -package file_transport - -import ( - "fmt" - "os" - "path/filepath" - "time" -) - -type FileController struct { - expire time.Duration - dir string - file string - period LogPeriod -} - -func (w *FileController) timeTag(t time.Time) string { - - tag := t.Format(w.period.FormatLayout()) - - return filepath.Join(w.dir, fmt.Sprintf("%s-%s.log", w.file, tag)) -} -func (w *FileController) fileName() string { - return filepath.Join(w.dir, fmt.Sprintf("%s.log", w.file)) -} -func (w *FileController) history(history string) { - - path := w.fileName() - os.Rename(path, history) - -} - -func (w *FileController) dropHistory() { - - expireTime := time.Now().Add(-w.expire) - pathPatten := filepath.Join(w.dir, fmt.Sprintf("%s-*", w.file)) - files, err := filepath.Glob(pathPatten) - if err == nil { - for _, f := range files { - if info, e := os.Stat(f); e == nil { - - if expireTime.After(info.ModTime()) { - _ = os.Remove(f) - } - } - - } - } -} diff --git a/output/file-transport/period.go b/output/file-transport/period.go deleted file mode 100644 index 61c872cb..00000000 --- a/output/file-transport/period.go +++ /dev/null @@ -1,71 +0,0 @@ -package file_transport - -import ( - "strings" -) - -//LogPeriod 日志周期 -type LogPeriod interface { - String() string - FormatLayout() string -} - -//LogPeriodType 日志周期类型 -type LogPeriodType int - -//ParsePeriod 解析周期 -func ParsePeriod(v string) LogPeriod { - switch strings.ToLower(v) { - //case "month": - // return PeriodMonth, nil - case "day": - return PeriodDay - case "hour": - return PeriodHour - default: - return PeriodDay - } - - //return nil, fmt.Errorf("not a valid period: %q", v) -} -func (period LogPeriodType) String() string { - switch period { - //case PeriodMonth: - // return "month" - case PeriodDay: - return "day" - case PeriodHour: - return "hour" - default: - return "unknown" - } -} - -const ( - //PeriodMonth 月 - PeriodMonth LogPeriodType = iota - //PeriodDay 日 - PeriodDay - //PeriodHour 时 - PeriodHour -) - -//FormatLayout 格式化 -func (period LogPeriodType) FormatLayout() string { - switch period { - case PeriodHour: - { - return "2006-01-02-15" - } - case PeriodDay: - { - return "2006-01-02" - } - case PeriodMonth: - { - return "2006-01" - } - default: - return "2006-01-02-15" - } -} diff --git a/output/file-transport/writer.go b/output/file-transport/writer.go deleted file mode 100644 index 21415e88..00000000 --- a/output/file-transport/writer.go +++ /dev/null @@ -1,241 +0,0 @@ -package file_transport - -import ( - "bufio" - "bytes" - "context" - "github.com/eolinker/eosc/log" - "os" - "sync" - "time" -) - -// MaxBuffer buffer最大值 -const MaxBuffer = 1024 * 500 - -var ( - bufferPool = &sync.Pool{ - New: func() interface{} { - return new(bytes.Buffer) - }, - } -) - -// FileWriterByPeriod 文件周期写入 -type FileWriterByPeriod struct { - wC chan *bytes.Buffer - - enable bool - cancelFunc context.CancelFunc - locker sync.RWMutex - wg sync.WaitGroup - resetChan chan FileController -} - -// NewFileWriteByPeriod 获取新的FileWriterByPeriod -func NewFileWriteByPeriod(cfg *Config) *FileWriterByPeriod { - w := &FileWriterByPeriod{ - locker: sync.RWMutex{}, - wg: sync.WaitGroup{}, - enable: false, - resetChan: make(chan FileController), - } - - w.Open(&FileController{ - dir: cfg.Dir, - file: cfg.File, - period: cfg.Period, - expire: time.Duration(cfg.Expire) * 24 * time.Hour, - }) - return w -} - -func (w *FileWriterByPeriod) Reset(cfg *Config) { - w.resetChan <- FileController{ - dir: cfg.Dir, - file: cfg.File, - period: cfg.Period, - expire: time.Duration(cfg.Expire) * 24 * time.Hour, - } -} - -// Open 打开 -func (w *FileWriterByPeriod) Open(config *FileController) { - w.locker.Lock() - defer w.locker.Unlock() - - if w.enable { - return - } - - ctx, cancel := context.WithCancel(context.Background()) - w.cancelFunc = cancel - w.wC = make(chan *bytes.Buffer, 100) - - w.enable = true - go func() { - w.wg.Add(1) - w.do(ctx, config) - w.wg.Done() - }() -} - -// Close 关闭 -func (w *FileWriterByPeriod) Close() { - - isClose := false - w.locker.Lock() - if !w.enable { - w.locker.Unlock() - return - } - - if w.cancelFunc != nil { - isClose = true - w.cancelFunc() - w.cancelFunc = nil - } - w.enable = false - w.locker.Unlock() - if isClose { - w.wg.Wait() - } -} -func (w *FileWriterByPeriod) isEnable() bool { - w.locker.Lock() - defer w.locker.Unlock() - return w.enable -} -func (w *FileWriterByPeriod) Write(p []byte) (n int, err error) { - - l := len(p) - - if l == 0 { - return - } - if !w.isEnable() { - return l, nil - } - buffer := bufferPool.Get().(*bytes.Buffer) - buffer.Reset() - buffer.Write(p) - if p[l-1] != '\n' { - buffer.WriteByte('\n') - } - w.wC <- buffer - return l, nil -} - -func (w *FileWriterByPeriod) do(ctx context.Context, config *FileController) { - fileController := *config - fileController.initFile() - f, lastTag, e := fileController.openFile() - if e != nil { - log.Errorf("open log file:%s\n", e.Error()) - return - } - - buf := bufio.NewWriter(f) - t := time.NewTicker(time.Second * 5) - defer t.Stop() - tFlush := time.NewTimer(time.Second) - - resetFunc := func(controller FileController) { - if lastTag != fileController.timeTag(time.Now()) { - if buf.Buffered() > 0 { - buf.Flush() - tFlush.Reset(time.Second) - } - f.Close() - fileController.history(lastTag) - fnew, tag, err := fileController.openFile() - if err != nil { - return - } - lastTag = tag - f = fnew - buf.Reset(f) - - go fileController.dropHistory() - } - } - - for { - select { - case <-ctx.Done(): - { - for len(w.wC) > 0 { - p := <-w.wC - buf.Write(p.Bytes()) - bufferPool.Put(p) - } - buf.Flush() - f.Close() - t.Stop() - //w.wg.Done() - return - } - - case <-t.C: - { - - resetFunc(fileController) - - } - case <-tFlush.C: - { - if buf.Buffered() > 0 { - buf.Flush() - } - tFlush.Reset(time.Second) - } - case p := <-w.wC: - { - buf.Write(p.Bytes()) - bufferPool.Put(p) - if buf.Buffered() > MaxBuffer { - buf.Flush() - } - tFlush.Reset(time.Second) - } - case controller, ok := <-w.resetChan: - { - if ok { - resetFunc(controller) - fileController = controller - } - } - } - } -} - -func (w *FileController) initFile() { - err := os.MkdirAll(w.dir, 0666) - if err != nil { - log.Error(err) - } - path := w.fileName() - nowHistoryName := w.timeTag(time.Now()) - if info, e := os.Stat(path); e == nil { - - timeTag := w.timeTag(info.ModTime()) - if timeTag != nowHistoryName { - w.history(timeTag) - } - } - - w.dropHistory() - -} - -func (w *FileController) openFile() (*os.File, string, error) { - path := w.fileName() - nowTag := w.timeTag(time.Now()) - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - - if err != nil { - return nil, "", err - } - return f, nowTag, err - -} From b65556c342300e4b1f448f8ac77485b56eafe871 Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Tue, 6 Jun 2023 18:54:21 +0800 Subject: [PATCH 04/31] =?UTF-8?q?=E5=AF=B9=E6=8E=A5log=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/fileoutput/write.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/output/fileoutput/write.go b/drivers/output/fileoutput/write.go index 64e4456c..e5781daf 100644 --- a/drivers/output/fileoutput/write.go +++ b/drivers/output/fileoutput/write.go @@ -38,7 +38,7 @@ func (a *FileWriter) reset(cfg *Config) (err error) { } transport := a.transport - c := &filelog.Config{ + c := filelog.Config{ Dir: cfg.Dir, File: cfg.File, Expire: time.Duration(cfg.Expire) * 24 * time.Hour, From bda61573431231f877f30dd86e896b5b01c620df Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Wed, 7 Jun 2023 18:28:08 +0800 Subject: [PATCH 05/31] =?UTF-8?q?=E5=AF=B9=E6=8E=A5access=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/fileoutput/output.go | 24 +++++++++++++++++++++--- drivers/output/fileoutput/write.go | 9 ++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/drivers/output/fileoutput/output.go b/drivers/output/fileoutput/output.go index 50df8db3..cd0f33e5 100644 --- a/drivers/output/fileoutput/output.go +++ b/drivers/output/fileoutput/output.go @@ -1,7 +1,10 @@ package fileoutput import ( + "fmt" scope_manager "github.com/eolinker/apinto/scope-manager" + "github.com/eolinker/eosc/router" + "net/http" "reflect" "github.com/eolinker/apinto/drivers" @@ -19,6 +22,14 @@ type FileOutput struct { isRunning bool } +func (a *FileOutput) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if a.writer == nil || a.writer.fileHandler == nil { + w.WriteHeader(http.StatusNotFound) + return + } + a.writer.fileHandler.ServeHTTP(w, r) +} + func (a *FileOutput) Output(entry eosc.IEntry) error { w := a.writer if w != nil { @@ -45,7 +56,7 @@ func (a *FileOutput) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWo w = new(FileWriter) } - err = w.reset(cfg) + err = w.reset(cfg, a.Name()) if err != nil { return err } @@ -58,7 +69,9 @@ func (a *FileOutput) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWo func (a *FileOutput) Stop() error { scope_manager.Del(a.Id()) + router.DeletePath(a.Id()) a.isRunning = false + w := a.writer if w != nil { err := w.stop() @@ -74,13 +87,18 @@ func (a *FileOutput) Start() error { if w == nil { w = new(FileWriter) } - - err := w.reset(a.config) + err := router.SetPath(a.Id(), fmt.Sprintf("/log/access-log/%s", a.Name()), a) if err != nil { return err } + err = w.reset(a.config, a.Name()) + if err != nil { + return err + } + a.writer = w scope_manager.Set(a.Id(), a, a.config.Scopes...) + return nil } diff --git a/drivers/output/fileoutput/write.go b/drivers/output/fileoutput/write.go index e5781daf..fbb14326 100644 --- a/drivers/output/fileoutput/write.go +++ b/drivers/output/fileoutput/write.go @@ -1,9 +1,13 @@ package fileoutput import ( + "fmt" "github.com/eolinker/eosc" "github.com/eolinker/eosc/formatter" "github.com/eolinker/eosc/log/filelog" + "github.com/eolinker/eosc/router" + "net/http" + "time" ) @@ -11,6 +15,8 @@ type FileWriter struct { formatter eosc.IFormatter transport *filelog.FileWriterByPeriod //id string + + fileHandler http.Handler } func (a *FileWriter) output(entry eosc.IEntry) error { @@ -25,7 +31,7 @@ func (a *FileWriter) output(entry eosc.IEntry) error { return nil } -func (a *FileWriter) reset(cfg *Config) (err error) { +func (a *FileWriter) reset(cfg *Config, name string) (err error) { factory, has := formatter.GetFormatterFactory(cfg.Type) if !has { @@ -46,6 +52,7 @@ func (a *FileWriter) reset(cfg *Config) (err error) { } if transport == nil { transport = filelog.NewFileWriteByPeriod(c) + a.fileHandler = transport.ServeHTTP(fmt.Sprintf("/%s/log/access/%s", router.RouterPrefix, name)) } else { transport.Reset(c) } From 82056d136088f2c571044ee4f5e9a46f9e2096f7 Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Fri, 9 Jun 2023 15:05:56 +0800 Subject: [PATCH 06/31] =?UTF-8?q?prometheus=20output=E5=8E=BB=E6=8E=89metr?= =?UTF-8?q?ics=5Fpath=E7=9A=84=E9=85=8D=E7=BD=AE,=E6=94=B9=E4=B8=BA/apinto?= =?UTF-8?q?/prometheus/metrics/{worker=5Fname}=E7=9A=84=E5=BD=A2=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/prometheus/check.go | 10 ---------- drivers/output/prometheus/config.go | 1 - drivers/output/prometheus/driver.go | 7 ++----- drivers/output/prometheus/output.go | 9 --------- 4 files changed, 2 insertions(+), 25 deletions(-) diff --git a/drivers/output/prometheus/check.go b/drivers/output/prometheus/check.go index 5ab3ee95..377c7002 100644 --- a/drivers/output/prometheus/check.go +++ b/drivers/output/prometheus/check.go @@ -2,7 +2,6 @@ package prometheus import ( "sort" - "strings" ) // checkScopesChange 检查scopes配置是否有改变 @@ -27,15 +26,6 @@ func checkScopesChange(oldScopes, newScopes []string) bool { return false } -func checkPathChange(oldPath, newPath string) bool { - oldPath = "/" + strings.Trim(oldPath, "/") - newPath = "/" + strings.Trim(newPath, "/") - if oldPath != newPath { - return true - } - return false -} - func checkMetricConfigChange(oldMC, newMC *MetricConfig) bool { if oldMC.Collector != newMC.Collector { return true diff --git a/drivers/output/prometheus/config.go b/drivers/output/prometheus/config.go index 46a48d6c..ac6bebbd 100644 --- a/drivers/output/prometheus/config.go +++ b/drivers/output/prometheus/config.go @@ -2,7 +2,6 @@ package prometheus type Config struct { Scopes []string `json:"scopes" label:"作用域"` - Path string `json:"path" yaml:"path" required:"true" label:"Metrics路径"` Metrics []*MetricConfig `json:"metrics" yaml:"metrics" required:"true" label:"指标列表"` } diff --git a/drivers/output/prometheus/driver.go b/drivers/output/prometheus/driver.go index 013f2ac4..df93789b 100644 --- a/drivers/output/prometheus/driver.go +++ b/drivers/output/prometheus/driver.go @@ -20,10 +20,6 @@ func Check(v *Config, workers map[eosc.RequireId]eosc.IWorker) error { func doCheck(promConf *Config) (map[string]*metricInfoCfg, error) { metricLabels := make(map[string]*metricInfoCfg, len(promConf.Metrics)) - if match := utils.CheckUrlPath(promConf.Path); !match { - return nil, fmt.Errorf(errorPathFormat, promConf.Path) - } - if len(promConf.Metrics) == 0 { return nil, errorNullMetrics } @@ -174,7 +170,8 @@ func Create(id, name string, cfg *Config, workers map[eosc.RequireId]eosc.IWorke p.registry, promhttp.HandlerFor(p.registry, promhttp.HandlerOpts{}), ) - err = router.SetPath(p.Id(), p.config.Path, p) + //路径为 metrics路径为 /apinto/prometheus/metrics/{worker_name} 前面的/apinto/在router.SetPath里做拼接 + err = router.SetPath(p.Id(), fmt.Sprintf("/prometheus/metrics/%s", name), p) if err != nil { return nil, fmt.Errorf("create output %s fail: %w", p.Id(), err) } diff --git a/drivers/output/prometheus/output.go b/drivers/output/prometheus/output.go index 08c72993..b2ad7de5 100644 --- a/drivers/output/prometheus/output.go +++ b/drivers/output/prometheus/output.go @@ -109,15 +109,6 @@ func (p *PromOutput) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWo ) } - //若path有变，更新router - if checkPathChange(p.config.Path, cfg.Path) { - //重新设置路由 - err = router.SetPath(p.Id(), cfg.Path, p) - if err != nil { - return fmt.Errorf("reset output %s fail: %w", p.Id(), err) - } - } - if isMetricsUpdate { p.registry = newRegistry p.handler = handler From adb36a83f82306dec507aaf6144928bf1bf381e7 Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Fri, 9 Jun 2023 15:55:15 +0800 Subject: [PATCH 07/31] =?UTF-8?q?metrics=E8=B7=AF=E5=BE=84=E6=94=B9?= =?UTF-8?q?=E4=B8=BA/apinto/metrics/prometheus/{worker=5Fname}?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/prometheus/driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/output/prometheus/driver.go b/drivers/output/prometheus/driver.go index df93789b..55a38783 100644 --- a/drivers/output/prometheus/driver.go +++ b/drivers/output/prometheus/driver.go @@ -170,8 +170,8 @@ func Create(id, name string, cfg *Config, workers map[eosc.RequireId]eosc.IWorke p.registry, promhttp.HandlerFor(p.registry, promhttp.HandlerOpts{}), ) - //路径为 metrics路径为 /apinto/prometheus/metrics/{worker_name} 前面的/apinto/在router.SetPath里做拼接 - err = router.SetPath(p.Id(), fmt.Sprintf("/prometheus/metrics/%s", name), p) + //metrics路径为 /apinto/metrics/prometheus/{worker_name} 前面的/apinto/在router.SetPath里做拼接 + err = router.SetPath(p.Id(), fmt.Sprintf("/metrics/prometheus/%s", name), p) if err != nil { return nil, fmt.Errorf("create output %s fail: %w", p.Id(), err) } From 7870b1111ce9a676fda368910c81976c6c886eba Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Mon, 12 Jun 2023 10:37:13 +0800 Subject: [PATCH 08/31] =?UTF-8?q?=E4=BF=AE=E6=AD=A3access-log=E7=9A=84name?= =?UTF-8?q?space?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/fileoutput/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/output/fileoutput/output.go b/drivers/output/fileoutput/output.go index cd0f33e5..4dfe040a 100644 --- a/drivers/output/fileoutput/output.go +++ b/drivers/output/fileoutput/output.go @@ -87,7 +87,7 @@ func (a *FileOutput) Start() error { if w == nil { w = new(FileWriter) } - err := router.SetPath(a.Id(), fmt.Sprintf("/log/access-log/%s", a.Name()), a) + err := router.SetPath(a.Id(), fmt.Sprintf("/log/access/%s/", a.Name()), a) if err != nil { return err } From c29f032321b7f89647e66c6f4cfd8ed6c4b8452f Mon Sep 17 00:00:00 2001 From: huangmengzhu Date: Thu, 15 Jun 2023 11:20:00 +0800 Subject: [PATCH 09/31] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E5=99=A8=E6=B3=A8=E5=86=8C=E7=9A=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=8E=A5=E5=8F=A3path=E5=A4=9A=E4=BA=86/=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/output/fileoutput/write.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/output/fileoutput/write.go b/drivers/output/fileoutput/write.go index fbb14326..3980cce0 100644 --- a/drivers/output/fileoutput/write.go +++ b/drivers/output/fileoutput/write.go @@ -52,7 +52,7 @@ func (a *FileWriter) reset(cfg *Config, name string) (err error) { } if transport == nil { transport = filelog.NewFileWriteByPeriod(c) - a.fileHandler = transport.ServeHTTP(fmt.Sprintf("/%s/log/access/%s", router.RouterPrefix, name)) + a.fileHandler = transport.ServeHTTP(fmt.Sprintf("/%slog/access/%s", router.RouterPrefix, name)) } else { transport.Reset(c) } From 0e0450e6dccc104752ed8e18ea59811bd04794bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 17:38:08 +0800 Subject: [PATCH 10/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 56 ++++++++++++++------------------------------------ 1 file changed, 15 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69f904f6..1c2404cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,16 +3,17 @@ stages: # - check - build - deploy - - publish +# - publish variables: - PATH: /usr/local/sonar-scanner/sonar-scanner-4.4.0.2170-linux/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/data/golang/go/bin/:/root/go/bin - GOROOT: /data/golang/go + PATH: /opt/go/go/bin/:/opt/node/node/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin + GOROOT: /opt/go/go GOPROXY: https://goproxy.cn - APP: apinto - VERSION: $CI_COMMIT_SHORT_SHA SAVE_DIR: /opt/apinto - APP_PRE: ${APP}-${VERSION} + VERSION: $CI_COMMIT_SHORT_SHA + APP: apinto + APP_PRE: ${APP}_${VERSION} + BUILD_DIR: apserver-build default: before_script: @@ -30,25 +31,14 @@ merge-informer: # 飞书回调 curl -X POST -H "Content-Type: application/json" \ -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"项目：${CI_PROJECT_NAME}\\n提交人：${GITLAB_USER_NAME}\\n提交信息：${CI_MERGE_REQUEST_TITLE}\\n合并分支信息：${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} -> ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}\\n差异性地址：${DIFF_URL}\\n请及时review代码\"}}" \ https://open.feishu.cn/open-apis/bot/v2/hook/1c334752-2874-41a1-8f1b-3060f2d46b6c -# checker: -# stage: check -# rules: -# - if: $CI_COMMIT_BRANCH=="develop" -# script: -# - set +e -# - go mod tidy -# - go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.data -timeout=1m ./... -# - golangci-lint run --timeout=3m --out-format checkstyle --issues-exit-code 0 ./... > report.xml -# - sonar-scanner + builder: stage: build rules: - - if: $CI_COMMIT_BRANCH=="develop" - # when: on_success - - if: $CI_COMMIT_BRANCH=="test" - # when: on_success + - if: $CI_COMMIT_BRANCH=="main" script: - - sed -i '/replace github.com\/eolinker\/eosc => */d' go.mod +# - sed -i '/replace github.com\/eolinker\/eosc => */d' go.mod + - cd ../eosc && git pull - go mod tidy - cd build/cmd && ./package.sh $VERSION cache: @@ -57,20 +47,14 @@ builder: deployer: stage: deploy rules: - - if: $CI_COMMIT_BRANCH=="develop" + - if: $CI_COMMIT_BRANCH=="main" variables: - DEPLOY_SERVER: root@172.18.189.43 DEPLOY_DESC: "DEV 环境" - - if: $CI_COMMIT_BRANCH=="test" - variables: - DEPLOY_SERVER: root@172.18.65.60 - DEPLOY_DESC: "TEST 环境" dependencies: - builder script: - - set -e - - scp out/${APP_PRE}.linux.x64.tar.gz ${DEPLOY_SERVER}:${SAVE_DIR} - - ssh ${DEPLOY_SERVER} "set -e; cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP}; ./install.sh upgrade;cd ${SAVE_DIR}; ./clean.sh ${APP_PRE}" + - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} + - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP}; ./install.sh upgrade;cd ${SAVE_DIR}; ./clean.sh ${APP_PRE} - | curl -X POST -H "Content-Type: application/json" \ -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"项目：apinto\\n环境：${DEPLOY_DESC}\\n更新部署完成.\"}}" \ @@ -78,14 +62,4 @@ deployer: when: on_success cache: paths: - - out/ -publisher: - stage: publish - only: - - tags - script: - - sed -i '/replace github.com\/eolinker\/eosc => */d' go.mod - - go mod tidy - - GOVERSION=$(go version) EoscVersion=$(sed -n 's/.*eosc v/v/p' go.mod) goreleaser release --skip-validate --rm-dist --skip-publish - - mkdir -p /data/pkg/apinto/${CI_COMMIT_TAG} - - cp -if dist/*.tar.gz /data/pkg/apinto/${CI_COMMIT_TAG} \ No newline at end of file + - out/ \ No newline at end of file From fe2ab32f80cf52f47af8345793b56b060d733c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 17:59:19 +0800 Subject: [PATCH 11/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c2404cb..3670ef0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ builder: - if: $CI_COMMIT_BRANCH=="main" script: # - sed -i '/replace github.com\/eolinker\/eosc => */d' go.mod - - cd ../eosc && git pull + - cd ../eosc && git pull && cd .. - go mod tidy - cd build/cmd && ./package.sh $VERSION cache: From 169cb6751d4534714cc1f375ab32e1e9d105c124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 18:01:35 +0800 Subject: [PATCH 12/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3670ef0f..92322f5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ builder: - if: $CI_COMMIT_BRANCH=="main" script: # - sed -i '/replace github.com\/eolinker\/eosc => */d' go.mod - - cd ../eosc && git pull && cd .. + - cd ../eosc && git pull && cd ../apinto - go mod tidy - cd build/cmd && ./package.sh $VERSION cache: From 49bd9e18f3fa1cf7c75f01ccb52f8949b8d1e715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 18:08:55 +0800 Subject: [PATCH 13/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92322f5f..b1584258 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ variables: SAVE_DIR: /opt/apinto VERSION: $CI_COMMIT_SHORT_SHA APP: apinto - APP_PRE: ${APP}_${VERSION} + APP_PRE: ${APP}-${VERSION} BUILD_DIR: apserver-build default: From fd0fb2fb6efb6aacca5ff60c67def0c4a0f76fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 18:15:06 +0800 Subject: [PATCH 14/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1584258..7379948c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,11 @@ deployer: - builder script: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP}; ./install.sh upgrade;cd ${SAVE_DIR}; ./clean.sh ${APP_PRE} + - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} + - ./apinto stop + - sleep 10s + - ./apinto start + - cd ${SAVE_DIR}; ./clean.sh ${APP_PRE} - | curl -X POST -H "Content-Type: application/json" \ -d "{\"msg_type\":\"text\",\"content\":{\"text\":\"项目：apinto\\n环境：${DEPLOY_DESC}\\n更新部署完成.\"}}" \ From 33fda347e11557b46042383fbb10aa569d926107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 18:25:40 +0800 Subject: [PATCH 15/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7379948c..b526de34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,7 @@ deployer: script: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} + - pwd - ./apinto stop - sleep 10s - ./apinto start From 9c5d1d95d13bb1daa2e28ca2c9cb5f90f6d62a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 18:30:14 +0800 Subject: [PATCH 16/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b526de34..f97d7cd1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ deployer: script: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} - - pwd + - ln -s ${APP_PRE}/apinto.yml ./ && ln -s ${APP_PRE}/config.yml ./ && ln -snf ${APP_PRE}/tmp ./ - ./apinto stop - sleep 10s - ./apinto start From 8bbf60ed35ae5792ab2ab5a7a3b3e53691c9d2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Fri, 16 Jun 2023 18:32:54 +0800 Subject: [PATCH 17/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f97d7cd1..9d9cf181 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ deployer: script: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} - - ln -s ${APP_PRE}/apinto.yml ./ && ln -s ${APP_PRE}/config.yml ./ && ln -snf ${APP_PRE}/tmp ./ + - ln -s ${SAVE_DIR}/apinto.yml ./ && ln -s ${SAVE_DIR}/config.yml ./ && ln -s ${SAVE_DIR}/tmp ./ - ./apinto stop - sleep 10s - ./apinto start From 926ab424cd8050458e3c2d7354b0934bd33e1e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 01:38:58 +0800 Subject: [PATCH 18/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d9cf181..747ec50b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,7 +55,7 @@ deployer: script: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} - - ln -s ${SAVE_DIR}/apinto.yml ./ && ln -s ${SAVE_DIR}/config.yml ./ && ln -s ${SAVE_DIR}/tmp ./ + - ln -snF ${SAVE_DIR}/etc/* ./ - ./apinto stop - sleep 10s - ./apinto start From 044aee54b6357eaed0bb7eb5f0daef114d260ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:08:14 +0800 Subject: [PATCH 19/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 747ec50b..b8a03cb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,6 +59,7 @@ deployer: - ./apinto stop - sleep 10s - ./apinto start + - ps -ef | grep apinto - cd ${SAVE_DIR}; ./clean.sh ${APP_PRE} - | curl -X POST -H "Content-Type: application/json" \ From efbbf9dfd3469be8b313c1db670941b43fbf1985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:14:44 +0800 Subject: [PATCH 20/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8a03cb4..6fad1b7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,6 +59,7 @@ deployer: - ./apinto stop - sleep 10s - ./apinto start + - sleep 10s - ps -ef | grep apinto - cd ${SAVE_DIR}; ./clean.sh ${APP_PRE} - | From dbcbe1292b3e36bc39c1cc478cf2a62825e3da1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:16:47 +0800 Subject: [PATCH 21/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fad1b7f..2ab1eb85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,6 +50,7 @@ deployer: - if: $CI_COMMIT_BRANCH=="main" variables: DEPLOY_DESC: "DEV 环境" + APINTO_DEBUG: "true" dependencies: - builder script: @@ -59,7 +60,6 @@ deployer: - ./apinto stop - sleep 10s - ./apinto start - - sleep 10s - ps -ef | grep apinto - cd ${SAVE_DIR}; ./clean.sh ${APP_PRE} - | From 65ef2c0b5563d2b0e3b2690fbd53715a2ea4ca83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:19:47 +0800 Subject: [PATCH 22/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ab1eb85..b8a03cb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,6 @@ deployer: - if: $CI_COMMIT_BRANCH=="main" variables: DEPLOY_DESC: "DEV 环境" - APINTO_DEBUG: "true" dependencies: - builder script: From e1ef047b0b793bcd26382659ecee128768c295f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:21:46 +0800 Subject: [PATCH 23/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8a03cb4..2ab1eb85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,6 +50,7 @@ deployer: - if: $CI_COMMIT_BRANCH=="main" variables: DEPLOY_DESC: "DEV 环境" + APINTO_DEBUG: "true" dependencies: - builder script: From b8436d8e62b9fadfffdc4f72605ed0d8b6b4a2ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:24:44 +0800 Subject: [PATCH 24/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ab1eb85..1c1acc29 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,7 +56,7 @@ deployer: script: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} - - ln -snF ${SAVE_DIR}/etc/* ./ + - ln -snF ${SAVE_DIR}/etc/tmp ./ && cp ${SAVE_DIR}/etc/apinto.yml ./ && cp ${SAVE_DIR}/etc/config.yml ./ - ./apinto stop - sleep 10s - ./apinto start From 737aae918af3796e878ab61b4acd26eb13e8200c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:26:15 +0800 Subject: [PATCH 25/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c1acc29..c5f4d66e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,6 +57,7 @@ deployer: - cp out/${APP_PRE}.linux.x64.tar.gz ${SAVE_DIR} - cd ${SAVE_DIR};cd ${SAVE_DIR}; mkdir -p ${APP_PRE};tar -zxvf ${APP_PRE}.linux.x64.tar.gz -C ${APP_PRE};cd ${APP_PRE}/${APP} - ln -snF ${SAVE_DIR}/etc/tmp ./ && cp ${SAVE_DIR}/etc/apinto.yml ./ && cp ${SAVE_DIR}/etc/config.yml ./ + - pwd - ./apinto stop - sleep 10s - ./apinto start From 90d280f1199a8e878d8d066537764f005541e5e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=81=A5?= Date: Mon, 19 Jun 2023 02:29:51 +0800 Subject: [PATCH 26/31] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c5f4d66e..f68de119 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,6 @@ deployer: - if: $CI_COMMIT_BRANCH=="main" variables: DEPLOY_DESC: "DEV 环境" - APINTO_DEBUG: "true" dependencies: - builder script: From fd26e784d0b90e2cb1e9d670888a13f80a490d5b Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Mon, 26 Jun 2023 14:27:52 +0800 Subject: [PATCH 27/31] =?UTF-8?q?=E4=BF=AE=E5=A4=8DbaseNode=E7=9A=84id?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E5=88=9D=E5=A7=8B=E5=8C=96=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- discovery/node.go | 4 ++-- discovery/nodes.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/discovery/node.go b/discovery/node.go index 965c4a17..cdcd12ac 100644 --- a/discovery/node.go +++ b/discovery/node.go @@ -54,8 +54,8 @@ func (n *_BaseNode) Last() time.Time { } -func newBaseNode(ip string, port int, statusChecker _INodeStatusCheck) *_BaseNode { - return &_BaseNode{ip: ip, port: port, status: Running, statusChecker: statusChecker} +func newBaseNode(id string, ip string, port int, statusChecker _INodeStatusCheck) *_BaseNode { + return &_BaseNode{id: id, ip: ip, port: port, status: Running, statusChecker: statusChecker} } func (n *_BaseNode) ID() string { diff --git a/discovery/nodes.go b/discovery/nodes.go index e2e4f9bd..3eaddb87 100644 --- a/discovery/nodes.go +++ b/discovery/nodes.go @@ -29,7 +29,7 @@ func (ac *appContainer) Get(ip string, port int) INode { return node } - ac.nodes.Set(id, newBaseNode(ip, port, ac)) + ac.nodes.Set(id, newBaseNode(id, ip, port, ac)) node, _ = ac.nodes.Get(id) return node } From 6a17e111c29c8264b224d45bb24ee81c4351e980 Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Mon, 26 Jun 2023 15:55:20 +0800 Subject: [PATCH 28/31] =?UTF-8?q?client=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/discovery/consul/clients.go | 8 +-- drivers/discovery/nacos/client.go | 84 +++++++++++++++++++++++---- drivers/discovery/nacos/config.go | 17 +----- drivers/discovery/nacos/driver.go | 13 +++-- drivers/discovery/nacos/nacos.go | 6 +- drivers/discovery/nacos/nacos_test.go | 12 ++-- 6 files changed, 99 insertions(+), 41 deletions(-) diff --git a/drivers/discovery/consul/clients.go b/drivers/discovery/consul/clients.go index fe9254cf..72decf81 100644 --- a/drivers/discovery/consul/clients.go +++ b/drivers/discovery/consul/clients.go @@ -18,11 +18,11 @@ func newClients(addrs []string, param map[string]string) *consulClients { clients := make([]*api.Client, 0, len(addrs)) defaultConfig := api.DefaultConfig() - if _, has := param["token"]; has { - defaultConfig.Token = param["token"] + if v, has := param["token"]; has { + defaultConfig.Token = v } - if _, has := param["namespace"]; has { - defaultConfig.Namespace = param["namespace"] + if v, has := param["namespace"]; has { + defaultConfig.Namespace = v } for _, addr := range addrs { diff --git a/drivers/discovery/nacos/client.go b/drivers/discovery/nacos/client.go index ecf8dd73..1123c8b9 100644 --- a/drivers/discovery/nacos/client.go +++ b/drivers/discovery/nacos/client.go @@ -3,9 +3,12 @@ package nacos import ( "encoding/json" "fmt" + "github.com/nacos-group/nacos-sdk-go/v2/clients" + "github.com/nacos-group/nacos-sdk-go/v2/clients/naming_client" + "github.com/nacos-group/nacos-sdk-go/v2/common/constant" + "github.com/nacos-group/nacos-sdk-go/v2/vo" "io" "net/http" - "net/url" "strconv" "strings" @@ -15,19 +18,80 @@ import ( ) type client struct { - address []string - params url.Values + namingClient naming_client.INamingClient + group string + clusters []string } -func newClient(address []string, params url.Values) *client { - adds := make([]string, 0, len(address)) - for _, a := range address { - if !strings.HasPrefix(a, "http://") && !strings.HasPrefix(a, "https://") { - a = fmt.Sprintf("%s://%s", defaultScheme, a) +func newClient(name string, address []string, params map[string]string) (*client, error) { + clientConfig := &constant.ClientConfig{ + LogDir: fmt.Sprintf("/var/log/apinto/nacos/%s", name), + CacheDir: fmt.Sprintf("/var/cache/apinto/nacos/%s", name), + LogLevel: "error", + } + //获取namespaceId, username,password + if v, has := params["namespaceId"]; has { + clientConfig.NamespaceId = v + } + if v, has := params["username"]; has { + clientConfig.Username = v + } + if v, has := params["password"]; has { + clientConfig.Password = v + } + + serverConfigs := make([]constant.ServerConfig, 0, len(address)) + var ( + scheme, ipAddr string + port uint64 + err error + ) + for _, addr := range address { + schemeIdx := strings.Index(addr, "://") + if schemeIdx < 0 { + scheme = defaultScheme + } else { + scheme = addr[:schemeIdx] + addr = addr[schemeIdx+3:] + } + portIdx := strings.Index(addr, ":") + if portIdx < 0 { + ipAddr = addr + port = 0 + } else { + ipAddr = addr[:portIdx] + portStr := addr[portIdx+1:] + port, err = strconv.ParseUint(portStr, 10, 64) + if err != nil { + return nil, err + } } - adds = append(adds, a) + serverConfigs = append(serverConfigs, constant.ServerConfig{ + Scheme: scheme, + IpAddr: ipAddr, + Port: port, + }) } - return &client{adds, params} + + namingClient, err := clients.NewNamingClient(vo.NacosClientParam{ + ClientConfig: clientConfig, + ServerConfigs: serverConfigs, + }) + if err != nil { + return nil, err + } + + c := &client{namingClient: namingClient} + //获取group, clusters + if v, has := params["group"]; has { + c.group = v + } + if v, has := params["clusters"]; has { + clusters := strings.Split(strings.TrimSpace(v), ",") + c.clusters = clusters + } + + return c, nil } // GetNodeList 从nacos接入地址中获取对应服务的节点列表 diff --git a/drivers/discovery/nacos/config.go b/drivers/discovery/nacos/config.go index 50f5ed35..1c342156 100644 --- a/drivers/discovery/nacos/config.go +++ b/drivers/discovery/nacos/config.go @@ -1,27 +1,14 @@ package nacos -import ( - "net/url" -) - const defaultScheme = "http" -//Config nacos驱动配置 +// Config nacos驱动配置 type Config struct { Config AccessConfig `json:"config" label:"配置信息"` } -//AccessConfig 接入地址配置 +// AccessConfig 接入地址配置 type AccessConfig struct { Address []string `json:"address" label:"nacos地址"` Params map[string]string `json:"params" label:"参数"` } - -func (c *Config) getParams() url.Values { - p := url.Values{} - p.Set("healthyOnly", "true") - for k, v := range c.Config.Params { - p.Set(k, v) - } - return p -} diff --git a/drivers/discovery/nacos/driver.go b/drivers/discovery/nacos/driver.go index 678ee644..eca8f3b0 100644 --- a/drivers/discovery/nacos/driver.go +++ b/drivers/discovery/nacos/driver.go @@ -1,6 +1,7 @@ package nacos import ( + "fmt" "github.com/eolinker/apinto/drivers" "sync" @@ -14,13 +15,15 @@ const ( // Create 创建nacos驱动实例 func Create(id, name string, cfg *Config, workers map[eosc.RequireId]eosc.IWorker) (eosc.IWorker, error) { - + c, err := newClient("", cfg.Config.Address, cfg.Config.Params) + if err != nil { + return nil, fmt.Errorf("create nacos client fail. err: %w", err) + } return &nacos{ WorkerBase: drivers.Worker(id, name), - client: newClient(cfg.Config.Address, cfg.getParams()), - - services: discovery.NewAppContainer(), - locker: sync.RWMutex{}, + client: c, + services: discovery.NewAppContainer(), + locker: sync.RWMutex{}, }, nil } diff --git a/drivers/discovery/nacos/nacos.go b/drivers/discovery/nacos/nacos.go index dd3a8ad1..a8abc89b 100644 --- a/drivers/discovery/nacos/nacos.go +++ b/drivers/discovery/nacos/nacos.go @@ -90,7 +90,11 @@ func (n *nacos) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWorker) if !ok { return fmt.Errorf("need %s,now %s", config.TypeNameOf((*Config)(nil)), config.TypeNameOf(conf)) } - n.client = newClient(cfg.Config.Address, cfg.getParams()) + nClient, err := newClient("", cfg.Config.Address, cfg.Config.Params) + if err != nil { + return fmt.Errorf("create nacos client fail. err: %w", err) + } + n.client = nClient return nil } diff --git a/drivers/discovery/nacos/nacos_test.go b/drivers/discovery/nacos/nacos_test.go index bd2c953d..8b971fdd 100644 --- a/drivers/discovery/nacos/nacos_test.go +++ b/drivers/discovery/nacos/nacos_test.go @@ -19,10 +19,10 @@ func TestGetApp(t *testing.T) { }, }, } + c, _ := newClient("asd", cfg.Config.Address, cfg.Config.Params) n := &nacos{ - client: newClient(cfg.Config.Address, cfg.getParams()), - nodes: discovery.NewNodesData(), - services: discovery.NewServices(), + client: c, + services: discovery.NewAppContainer(), locker: sync.RWMutex{}, } app, err := n.GetApp(serviceName) @@ -32,9 +32,9 @@ func TestGetApp(t *testing.T) { for _, node := range app.Nodes() { t.Log(node.ID()) } - ns, bo := n.nodes.Get(serviceName) - if bo { - t.Log(len(ns)) + ns, err := n.GetApp(serviceName) + if err == nil { + t.Log(len(ns.Nodes())) } else { t.Error("nodes error") } From f43619155c026ebe121ae052da573080682199ec Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Mon, 26 Jun 2023 16:09:03 +0800 Subject: [PATCH 29/31] =?UTF-8?q?nacos=E6=8E=A5=E5=85=A5nacos-go-sdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/discovery/nacos/client.go | 81 ++++++++++--------------------- drivers/discovery/nacos/nacos.go | 4 -- 2 files changed, 26 insertions(+), 59 deletions(-) diff --git a/drivers/discovery/nacos/client.go b/drivers/discovery/nacos/client.go index 1123c8b9..1b04a916 100644 --- a/drivers/discovery/nacos/client.go +++ b/drivers/discovery/nacos/client.go @@ -1,19 +1,14 @@ package nacos import ( - "encoding/json" "fmt" "github.com/nacos-group/nacos-sdk-go/v2/clients" "github.com/nacos-group/nacos-sdk-go/v2/clients/naming_client" "github.com/nacos-group/nacos-sdk-go/v2/common/constant" "github.com/nacos-group/nacos-sdk-go/v2/vo" - "io" - "net/http" "strconv" "strings" - "github.com/eolinker/eosc/log" - "github.com/eolinker/apinto/discovery" ) @@ -94,67 +89,43 @@ func newClient(name string, address []string, params map[string]string) (*client return c, nil } -// GetNodeList 从nacos接入地址中获取对应服务的节点列表 +// GetNodeList 从nacosClient获取对应服务的节点列表 func (c *client) GetNodeList(serviceName string) ([]discovery.NodeInfo, error) { nodes := make([]discovery.NodeInfo, 0) set := make(map[string]struct{}) - for _, addr := range c.address { - ins, err := c.GetInstanceList(addr, serviceName) - if err != nil { - log.Info("nacos get node instance list error:", err) - continue + instances, err := c.namingClient.SelectInstances(vo.SelectInstancesParam{ + ServiceName: serviceName, + Clusters: c.clusters, + GroupName: c.group, + HealthyOnly: true, + }) + if err != nil { + return nil, err + } + + for _, ins := range instances { + label := map[string]string{ + "weight": strconv.FormatFloat(ins.Weight, 'f', -1, 64), } + //ins的instanceID可能为空 + instanceID := fmt.Sprintf("%s:%d", ins.Ip, ins.Port) + if _, exist := set[instanceID]; !exist { + set[instanceID] = struct{}{} - for _, host := range ins.Hosts { - label := map[string]string{ - "valid": strconv.FormatBool(host.Valid), - "marked": strconv.FormatBool(host.Marked), - "weight": strconv.FormatFloat(host.Weight, 'f', -1, 64), - } - if _, exist := set[host.InstanceID]; !exist { - set[host.InstanceID] = struct{}{} - nodes = append(nodes, discovery.NodeInfo{ - Ip: host.IP, - Port: host.Port, - Labels: label, - }) + for k, v := range ins.Metadata { + label[k] = v } + nodes = append(nodes, discovery.NodeInfo{ + Ip: ins.Ip, + Port: int(ins.Port), + Labels: label, + }) } } + if len(nodes) == 0 { return nil, discovery.ErrDiscoveryDown } return nodes, nil } - -// GetInstanceList 获取目标地址指定服务名的实例列表 -func (c *client) GetInstanceList(addr string, serviceName string) (*Instance, error) { - addr = addr + instancePath - paramsURL := c.params - paramsURL.Set("serviceName", serviceName) - req, err := http.NewRequest("GET", addr, nil) - if err != nil { - return nil, err - } - req.URL.RawQuery = paramsURL.Encode() - response, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - // 解析响应数据 - rawResponseData, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - err = response.Body.Close() - if err != nil { - return nil, err - } - var instance = &Instance{} - err = json.Unmarshal(rawResponseData, instance) - if err != nil { - return nil, err - } - return instance, nil -} diff --git a/drivers/discovery/nacos/nacos.go b/drivers/discovery/nacos/nacos.go index a8abc89b..fa0095b7 100644 --- a/drivers/discovery/nacos/nacos.go +++ b/drivers/discovery/nacos/nacos.go @@ -15,10 +15,6 @@ import ( "github.com/eolinker/eosc" ) -const ( - instancePath = "/nacos/v1/ns/instance/list" -) - var _ discovery.IDiscovery = (*nacos)(nil) type nacos struct { From a6146f685247c28f313d9e4f2c300f6bb43c1bcd Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Mon, 26 Jun 2023 17:48:49 +0800 Subject: [PATCH 30/31] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/discovery/nacos/client.go | 3 --- drivers/discovery/nacos/driver.go | 2 +- drivers/discovery/nacos/nacos.go | 4 +--- go.mod | 8 ++++++++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/discovery/nacos/client.go b/drivers/discovery/nacos/client.go index 1b04a916..9caee217 100644 --- a/drivers/discovery/nacos/client.go +++ b/drivers/discovery/nacos/client.go @@ -124,8 +124,5 @@ func (c *client) GetNodeList(serviceName string) ([]discovery.NodeInfo, error) { } } - if len(nodes) == 0 { - return nil, discovery.ErrDiscoveryDown - } return nodes, nil } diff --git a/drivers/discovery/nacos/driver.go b/drivers/discovery/nacos/driver.go index eca8f3b0..823e20e5 100644 --- a/drivers/discovery/nacos/driver.go +++ b/drivers/discovery/nacos/driver.go @@ -15,7 +15,7 @@ const ( // Create 创建nacos驱动实例 func Create(id, name string, cfg *Config, workers map[eosc.RequireId]eosc.IWorker) (eosc.IWorker, error) { - c, err := newClient("", cfg.Config.Address, cfg.Config.Params) + c, err := newClient(name, cfg.Config.Address, cfg.Config.Params) if err != nil { return nil, fmt.Errorf("create nacos client fail. err: %w", err) } diff --git a/drivers/discovery/nacos/nacos.go b/drivers/discovery/nacos/nacos.go index fa0095b7..0e84508a 100644 --- a/drivers/discovery/nacos/nacos.go +++ b/drivers/discovery/nacos/nacos.go @@ -64,7 +64,6 @@ func (n *nacos) Start() error { res, err := n.client.GetNodeList(serviceName) if err != nil { log.Warnf("nacos %s:%w for service %s", n.Name(), discovery.ErrDiscoveryDown, serviceName) - continue } //更新目标服务的节点列表 n.locker.Lock() @@ -86,7 +85,7 @@ func (n *nacos) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWorker) if !ok { return fmt.Errorf("need %s,now %s", config.TypeNameOf((*Config)(nil)), config.TypeNameOf(conf)) } - nClient, err := newClient("", cfg.Config.Address, cfg.Config.Params) + nClient, err := newClient(n.Name(), cfg.Config.Address, cfg.Config.Params) if err != nil { return fmt.Errorf("create nacos client fail. err: %w", err) } @@ -118,7 +117,6 @@ func (n *nacos) GetApp(serviceName string) (discovery.IApp, error) { ns, err := n.client.GetNodeList(serviceName) if err != nil { log.Errorf("%s get %s node list error: %v", driverName, serviceName, err) - } app = n.services.Set(serviceName, ns) diff --git a/go.mod b/go.mod index fbff9144..09f176cd 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/hashicorp/consul/api v1.9.1 github.com/influxdata/influxdb-client-go/v2 v2.12.1 github.com/jhump/protoreflect v1.14.1 + github.com/nacos-group/nacos-sdk-go/v2 v2.1.2 github.com/nsqio/go-nsq v1.1.0 github.com/ohler55/ojg v1.12.9 github.com/pkg/sftp v1.13.4 @@ -32,8 +33,10 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.1 // indirect github.com/RoaringBitmap/roaring v0.7.1 // indirect github.com/Workiva/go-datastructures v1.0.52 // indirect + github.com/aliyun/alibaba-cloud-sdk-go v1.61.1704 // indirect github.com/apache/dubbo-getty v1.4.8 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/creasty/defaults v1.5.2 // indirect github.com/dubbogo/triple v1.1.8 // indirect github.com/go-kit/log v0.1.0 // indirect @@ -46,7 +49,9 @@ require ( github.com/go-playground/validator/v10 v10.11.0 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/jinzhu/copier v0.3.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/knadh/koanf v1.4.1 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -67,6 +72,9 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.8.0 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect ) require ( From 64a0becebe7ecc131bbd883de40a542222c00b0a Mon Sep 17 00:00:00 2001 From: chenjiekun Date: Mon, 26 Jun 2023 18:05:42 +0800 Subject: [PATCH 31/31] =?UTF-8?q?=E4=BF=AE=E5=A4=8Deureka=EF=BC=8Cconsul?= =?UTF-8?q?=EF=BC=8C=E5=BD=93=E6=9C=8D=E5=8A=A1=E5=8F=91=E7=8E=B0=E4=B8=8D?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E6=9F=90=E6=9C=8D=E5=8A=A1=E6=97=B6=E4=BE=9D?= =?UTF-8?q?=E6=97=A7=E4=BD=BF=E7=94=A8=E6=97=A7=E5=AE=9E=E4=BE=8B=E7=9A=84?= =?UTF-8?q?bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/discovery/consul/clients.go | 14 ++++++-------- drivers/discovery/consul/consul.go | 3 --- drivers/discovery/eureka/client.go | 9 ++------- drivers/discovery/eureka/eureka.go | 3 --- drivers/discovery/nacos/nacos.go | 8 ++------ 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/drivers/discovery/consul/clients.go b/drivers/discovery/consul/clients.go index 72decf81..add92f15 100644 --- a/drivers/discovery/consul/clients.go +++ b/drivers/discovery/consul/clients.go @@ -53,8 +53,9 @@ func (c *consulClients) getNodes(service string) ([]discovery.NodeInfo, error) { nodeList := make([]discovery.NodeInfo, 0, 2) nodeIDSet := make(map[string]struct{}) for _, client := range c.clients { - clientNodes := getNodesFromClient(client, service) - if len(clientNodes) == 0 { + clientNodes, err := getNodesFromClient(client, service) + if err != nil { + log.Warnf("consul client down for service %s", service) continue } for _, n := range clientNodes { @@ -64,19 +65,16 @@ func (c *consulClients) getNodes(service string) ([]discovery.NodeInfo, error) { nodeIDSet[n.id] = struct{}{} } } - if len(nodeList) == 0 { - return nil, discovery.ErrDiscoveryDown - } return nodeList, nil } // getNodesFromClient 从连接的客户端返回健康的节点信息 -func getNodesFromClient(client *api.Client, service string) []*consulNodeInfo { +func getNodesFromClient(client *api.Client, service string) ([]*consulNodeInfo, error) { queryOptions := &api.QueryOptions{} serviceEntryArr, _, err := client.Health().Service(service, "", true, queryOptions) if err != nil { - return nil + return nil, err } nodes := make([]*consulNodeInfo, 0, len(serviceEntryArr)) @@ -91,5 +89,5 @@ func getNodesFromClient(client *api.Client, service string) []*consulNodeInfo { }) } - return nodes + return nodes, nil } diff --git a/drivers/discovery/consul/consul.go b/drivers/discovery/consul/consul.go index 5f851c08..7760fe38 100644 --- a/drivers/discovery/consul/consul.go +++ b/drivers/discovery/consul/consul.go @@ -47,7 +47,6 @@ func (c *consul) Start() error { nodeSet, err := c.clients.getNodes(serviceName) if err != nil { log.Warnf("consul %s:%s for service %s", c.Name(), discovery.ErrDiscoveryDown, serviceName) - continue } //更新目标服务的节点列表 c.services.Set(serviceName, nodeSet) @@ -83,8 +82,6 @@ func (c *consul) Stop() error { // GetApp 获取服务发现中目标服务的app func (c *consul) GetApp(serviceName string) (discovery.IApp, error) { - var err error - var has bool c.locker.RLock() app, has := c.services.GetApp(serviceName) c.locker.RUnlock() diff --git a/drivers/discovery/eureka/client.go b/drivers/discovery/eureka/client.go index 9e517345..3a33c270 100644 --- a/drivers/discovery/eureka/client.go +++ b/drivers/discovery/eureka/client.go @@ -22,16 +22,14 @@ func newClient(address []string, params url.Values) *client { // GetNodeList 从eureka接入地址中获取对应服务的节点列表 func (c *client) GetNodeList(serviceName string) ([]discovery.NodeInfo, error) { - isOk := false nodes := make([]discovery.NodeInfo, 0, 5) sets := make(map[string]struct{}) for _, addr := range c.address { app, err := c.GetApplication(addr, serviceName) if err != nil { - log.Info("eureka get node instance list error:", err) + log.Warnf("eureka get node instance list fail. err: %w", err) continue } - isOk = true for _, ins := range app.Instances { if ins.Status != eurekaStatusUp { continue @@ -54,12 +52,9 @@ func (c *client) GetNodeList(serviceName string) ([]discovery.NodeInfo, error) { } nodes = append(nodes, node) } - } } - if !isOk { - return nil, discovery.ErrDiscoveryDown - } + return nodes, nil } diff --git a/drivers/discovery/eureka/eureka.go b/drivers/discovery/eureka/eureka.go index f3137e01..6637ba63 100644 --- a/drivers/discovery/eureka/eureka.go +++ b/drivers/discovery/eureka/eureka.go @@ -73,12 +73,9 @@ func (e *eureka) Start() error { res, err := e.client.GetNodeList(serviceName) if err != nil { log.Warnf("eureka %s:%w for service %s", e.Name(), discovery.ErrDiscoveryDown, serviceName) - continue } //更新目标服务的节点列表 - e.locker.Lock() e.services.Set(serviceName, res) - e.locker.Unlock() } } } diff --git a/drivers/discovery/nacos/nacos.go b/drivers/discovery/nacos/nacos.go index 0e84508a..9d0f8953 100644 --- a/drivers/discovery/nacos/nacos.go +++ b/drivers/discovery/nacos/nacos.go @@ -66,10 +66,7 @@ func (n *nacos) Start() error { log.Warnf("nacos %s:%w for service %s", n.Name(), discovery.ErrDiscoveryDown, serviceName) } //更新目标服务的节点列表 - n.locker.Lock() n.services.Set(serviceName, res) - n.locker.Unlock() - } } } @@ -109,6 +106,7 @@ func (n *nacos) GetApp(serviceName string) (discovery.IApp, error) { } n.locker.Lock() + defer n.locker.Unlock() app, ok = n.services.GetApp(serviceName) if ok { return app.Agent(), nil @@ -116,12 +114,10 @@ func (n *nacos) GetApp(serviceName string) (discovery.IApp, error) { ns, err := n.client.GetNodeList(serviceName) if err != nil { - log.Errorf("%s get %s node list error: %v", driverName, serviceName, err) + log.Warnf("%s get %s node list error: %v", driverName, serviceName, err) } app = n.services.Set(serviceName, ns) - n.locker.Unlock() - return app.Agent(), nil }