diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..94a41f2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.idea +vendor + +config*.yaml +!config.example.yaml + +kucoin-market-for-linux +kucoin-market-for-mac +kucoin-market-for-windows.exe +kucoin_market + +runtime/* + +demo/* diff --git a/.env b/.env deleted file mode 100644 index c15f584..0000000 --- a/.env +++ /dev/null @@ -1,16 +0,0 @@ -# API_SKIP_VERIFY_TLS=1 - -API_BASE_URI=https://api.kucoin.com - -# If open order book true otherwise false -ENABLE_ORDER_BOOK=true - -# If open event watcher true otherwise false -ENABLE_EVENT_WATCHER=true - -# Password for RPS calls. Pass the same when calling -RPC_TOKEN=market-token - -REDIS_HOST=127.0.0.1:6379 -REDIS_PASSWORD= -REDIS_DB= \ No newline at end of file diff --git a/.env-sandbox b/.env-sandbox deleted file mode 100644 index 0dcccc1..0000000 --- a/.env-sandbox +++ /dev/null @@ -1,16 +0,0 @@ -# API_SKIP_VERIFY_TLS=1 - -API_BASE_URI=https://openapi-sandbox.kucoin.com - -# If open order book true otherwise false -ENABLE_ORDER_BOOK=true - -# If open event watcher true otherwise false -ENABLE_EVENT_WATCHER=true - -# Password for RPS calls. Pass the same when calling -RPC_TOKEN=market-token - -REDIS_HOST=127.0.0.1:6379 -REDIS_PASSWORD= -REDIS_DB= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2bee759..370b218 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ .idea vendor -kucoin_market -.env.local -__pycache__ -go.sum + +config*.yaml +!config.example.yaml + kucoin-market-for-linux kucoin-market-for-mac -kucoin-market-for-windows.exe \ No newline at end of file +kucoin-market-for-windows.exe +kucoin_market diff --git a/Dockerfile b/Dockerfile index 8f859c1..4375f58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN export GO111MODULE=on \ COPY . /go/src/github.com/Kucoin/kucoin-level3-sdk RUN cd /go/src/github.com/Kucoin/kucoin-level3-sdk \ - && CGO_ENABLED=0 go build -ldflags '-s -w' -o /go/bin/kucoin_market kucoin_market.go + && CGO_ENABLED=0 go build -ldflags '-s -w' -o /go/bin/kucoin_market cmd/main/market.go FROM debian:stretch @@ -22,7 +22,4 @@ VOLUME /app EXPOSE 9090 -COPY docker-entrypoint.sh /usr/local/bin/ -ENTRYPOINT ["docker-entrypoint.sh"] - -CMD ["kucoin_market", "-c", "/app/.env", "-symbol", "BTC-USDT", "-p", "9090", "-rpckey", "BTC-USDT"] +CMD ["kucoin_market", "start", "-c", "config.yaml"] diff --git a/README.md b/README.md index f3e8093..5bae843 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,53 @@ -# Kucoin Level3 market +# Kucoin Level3 Market -## guide +## Guide [中文文档](README_CN.md) ## Installation -1. install dependencies +1. build ``` -go get github.com/JetBlink/orderbook -go get github.com/go-redis/redis -go get github.com/gorilla/websocket -go get github.com/joho/godotenv -go get github.com/Kucoin/kucoin-go-sdk -go get github.com/shopspring/decimal -``` - -2. build - -``` -CGO_ENABLED=0 go build -ldflags '-s -w' -o kucoin_market kucoin_market.go +CGO_ENABLED=0 go build -ldflags '-s -w' -o kucoin_market cmd/main/market.go ``` or you can download the latest available [release](https://github.com/Kucoin/kucoin-level3-sdk/releases) ## Usage -1. [vim .env](.env): - ``` - # API_SKIP_VERIFY_TLS=1 - - API_BASE_URI=https://api.kucoin.com +1. [vim config.yaml](config.example.yaml): + ``` + app_debug: true - # If open order book true otherwise false - ENABLE_ORDER_BOOK=true + symbol: KCS-USDT + #symbol: XBTUSDM - # If open event watcher true otherwise false - ENABLE_EVENT_WATCHER=true + app: + name: market + log_file: "./runtime/log/market.log" - # Password for RPS calls. Pass the same when calling - RPC_TOKEN=market-token + api_server: + network: tcp + address: 0.0.0.0:9090 + token: your-rpc-token - REDIS_HOST=127.0.0.1:6379 - REDIS_PASSWORD= - REDIS_DB= + market.kucoin_v2: + url: "https://api.kucoin.com" + type: "spot" + # url: "https://api-futures.kucoin.com" + # type: "future" + + redis: + addr: 127.0.0.1:6379 + password: "" + db: 0 ``` 1. Run Command： ``` - ./kucoin_market -c .env -symbol BTC-USDT -p 9090 -rpckey BTC-USDT + ./kucoin_market start -c config.yaml ``` - ## Docker Usage @@ -61,29 +57,50 @@ or you can download the latest available [release](https://github.com/Kucoin/kuc docker build -t kucoin_market . ``` -1. [vim .env](.env) +1. [vim config.yaml](config.example.yaml): + ``` + app_debug: true + + symbol: KCS-USDT + #symbol: XBTUSDM + + app: + name: market + log_file: "./runtime/log/market.log" + + api_server: + network: tcp + address: 0.0.0.0:9090 + token: your-rpc-token + + market.kucoin_v2: + url: "https://api.kucoin.com" + type: "spot" + # url: "https://api-futures.kucoin.com" + # type: "future" + + redis: + addr: 127.0.0.1:6379 + password: "" + db: 0 + ``` 1. Run ``` - docker run --rm -it -v $(pwd)/.env:/app/.env --net=host kucoin_market + docker run --rm -it -v $(pwd)/config.yaml:/app/config.yaml --net=host kucoin_market ``` ## RPC Method -> endpoint : 127.0.0.1:9090 +> default endpoint : 127.0.0.1:9090 > the sdk rpc is based on golang jsonrpc 1.0 over tcp. see:[python jsonrpc client demo](./demo/python-demo/level3/rpc.py) * Get Part Order Book ``` - {"method": "Server.GetPartOrderBook", "params": [{"token": "your-rpc-token", "number": 1}], "id": 0} - ``` - -* Get Full Order Book - ``` - {"method": "Server.GetOrderBook", "params": [{"token": "your-rpc-token"}], "id": 0} + {"method": "Server.GetOrderBook", "params": [{"token": "your-rpc-token", "number": 1}], "id": 0} ``` * Add Event ClientOids To Channels @@ -91,10 +108,6 @@ see:[python jsonrpc client demo](./demo/python-demo/level3/rpc.py) {"method": "Server.AddEventClientOidsToChannels", "params": [{"token": "your-rpc-token", "data": {"clientOid": ["channel-1", "channel-2"]}}], "id": 0} ``` -* Add Event OrderIds To Channels - ``` - {"method": "Server.AddEventOrderIdsToChannels", "params": [{"token": "your-rpc-token", "data": {"orderId": ["channel-1", "channel-2"]}}], "id": 0} - ``` ## Python-Demo > the demo including orderbook display @@ -102,6 +115,6 @@ see:[python jsonrpc client demo](./demo/python-demo/level3/rpc.py) see:[python use_level3 demo](./demo/python-demo/order_book_demo.py) - Run order_book.py ``` - command: python order_book.py + command: python3 order_book_demo.py describe: display orderbook ``` diff --git a/README_CN.md b/README_CN.md index 21d4205..f7b4857 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,72 +1,106 @@ -# Kucoin Level3 market +# Kucoin Level3 Market -## 入门文档 - [英文文档](README.md) +## 文档 + [English Document](README.md) ## 安装 -1. install dependencies +1. 编译 ``` -go get github.com/JetBlink/orderbook -go get github.com/go-redis/redis -go get github.com/gorilla/websocket -go get github.com/joho/godotenv -go get github.com/Kucoin/kucoin-go-sdk -go get github.com/shopspring/decimal +CGO_ENABLED=0 go build -ldflags '-s -w' -o kucoin_market cmd/main/market.go ``` -2. build - -``` -CGO_ENABLED=0 go build -ldflags '-s -w' -o kucoin_market kucoin_market.go -``` - -或者直接下载已经编译完成的二进制文件 +或者直接下载已经编译完成的[二进制文件](https://github.com/Kucoin/kucoin-level3-sdk/releases) ## 用法 -1. [vim .env](.env): - ``` - # API_SKIP_VERIFY_TLS=1 - - API_BASE_URI=https://api.kucoin.com +1. [vim config.yaml](config.example.yaml): + ``` + app_debug: true - # If open order book true otherwise false - ENABLE_ORDER_BOOK=true + symbol: KCS-USDT + #symbol: XBTUSDM - # If open event watcher true otherwise false - ENABLE_EVENT_WATCHER=true + app: + name: market + log_file: "./runtime/log/market.log" - # Password for RPS calls. Pass the same when calling - RPC_TOKEN=market-token + api_server: + network: tcp + address: 0.0.0.0:9090 + token: your-rpc-token - REDIS_HOST=127.0.0.1:6379 - REDIS_PASSWORD= - REDIS_DB= + market.kucoin_v2: + url: "https://api.kucoin.com" + type: "spot" + # url: "https://api-futures.kucoin.com" + # type: "future" + + redis: + addr: 127.0.0.1:6379 + password: "" + db: 0 ``` 1. 运行命令： ``` - ./kucoin_market -c .env -symbol BTC-USDT -p 9090 -rpckey BTC-USDT + ./kucoin_market start -c config.yaml ``` +## Docker 用法 + +1. 编译镜像 + + ``` + docker build -t kucoin_market . + ``` + +1. [vim config.yaml](config.example.yaml): + ``` + app_debug: true + + symbol: KCS-USDT + #symbol: XBTUSDM + + app: + name: market + log_file: "./runtime/log/market.log" + + api_server: + network: tcp + address: 0.0.0.0:9090 + token: your-rpc-token + + market.kucoin_v2: + url: "https://api.kucoin.com" + type: "spot" + # url: "https://api-futures.kucoin.com" + # type: "future" + + redis: + addr: 127.0.0.1:6379 + password: "" + db: 0 + ``` + +1. 运行 + + ``` + docker run --rm -it -v $(pwd)/config.yaml:/app/config.yaml --net=host kucoin_market + ``` + ## RPC Method -> endpoint : 127.0.0.1:9090 +> default endpoint : 127.0.0.1:9090 > the sdk rpc is based on golang jsonrpc 1.0 over tcp. see:[python jsonrpc client demo](./demo/python-demo/level3/rpc.py) * Get Part Order Book ``` - {"method": "Server.GetPartOrderBook", "params": [{"token": "your-rpc-token", "number": 1}], "id": 0} - ``` - -* Get Full Order Book - ``` - {"method": "Server.GetOrderBook", "params": [{"token": "your-rpc-token"}], "id": 0} + {"method": "Server.GetOrderBook", "params": [{"token": "your-rpc-token", "number": 1}], "id": 0} ``` * Add Event ClientOids To Channels @@ -74,17 +108,13 @@ see:[python jsonrpc client demo](./demo/python-demo/level3/rpc.py) {"method": "Server.AddEventClientOidsToChannels", "params": [{"token": "your-rpc-token", "data": {"clientOid": ["channel-1", "channel-2"]}}], "id": 0} ``` -* Add Event OrderIds To Channels - ``` - {"method": "Server.AddEventOrderIdsToChannels", "params": [{"token": "your-rpc-token", "data": {"orderId": ["channel-1", "channel-2"]}}], "id": 0} - ``` ## Python-Demo -> python的demo包含了一个本地orderbook的展示 -see:[python use_level3 demo](./demo/python-demo/order_book_demo.py) +> python的demo包含了一个本地orderbook的展示 +see:[python use_level3 demo](./demo/python-demo/order_book_demo.py) - Run order_book.py ``` - command: python order_book.py + command: python3 order_book_demo.py describe: display orderbook - ``` \ No newline at end of file + ``` diff --git a/api/api.go b/api/api.go deleted file mode 100644 index eebbb81..0000000 --- a/api/api.go +++ /dev/null @@ -1,102 +0,0 @@ -package api - -import ( - "encoding/json" - "fmt" - "net" - "net/rpc" - "net/rpc/jsonrpc" - - "github.com/Kucoin/kucoin-level3-sdk/builder" - "github.com/Kucoin/kucoin-level3-sdk/events" - "github.com/Kucoin/kucoin-level3-sdk/utils/log" -) - -//Server is api server -type Server struct { - level3Builder *builder.Builder - eventWatcher *events.Watcher - - apiPort string - token string -} - -//InitRpcServer init rpc server -func InitRpcServer(apiPort string, token string, level3Builder *builder.Builder, watcher *events.Watcher) { - if apiPort == "" || token == "" { - panic(fmt.Sprintf("missing configuration，apiPort: %s, token: %s", apiPort, token)) - } - - apiPort = ":" + apiPort - if err := rpc.Register(&Server{ - level3Builder: level3Builder, - eventWatcher: watcher, - - apiPort: apiPort, - token: token, - }); err != nil { - panic("api server run failed, error: %s" + err.Error()) - } - - log.Warn("start running rpc server, port: %s", apiPort) - - listener, err := net.Listen("tcp", apiPort) - if err != nil { - panic("api server run failed, error: %s" + err.Error()) - } - - for { - conn, err := listener.Accept() - if err != nil { - continue - } - - go jsonrpc.ServeConn(conn) - } -} - -//TokenMessage is token type message -type TokenMessage struct { - Token string `json:"token"` -} - -//Response is api response -type Response struct { - Code string `json:"code"` - Data interface{} `json:"data"` - Error string `json:"error"` -} - -func (s *Server) checkToken(token string) string { - if token != s.token { - return s.failure(TokenErrorCode, "error token") - } - - return "" -} - -func (s *Server) success(data interface{}) string { - response, _ := json.Marshal(&Response{ - Code: "0", - Data: data, - Error: "", - }) - - return string(response) -} - -const ( - ServerErrorCode = "10" - TokenErrorCode = "20" - TickerErrorCode = "30" -) - -func (s *Server) failure(code string, err string) string { - response, _ := json.Marshal(&Response{ - Code: code, - Data: "", - Error: err, - }) - - return string(response) -} diff --git a/api/event.go b/api/event.go deleted file mode 100644 index fabb192..0000000 --- a/api/event.go +++ /dev/null @@ -1,47 +0,0 @@ -package api - -type AddEventOrderIdsMessage struct { - Data map[string][]string `json:"data"` - TokenMessage -} - -type AddEventClientOidsMessage struct { - Data map[string][]string `json:"data"` - TokenMessage -} - -func (s *Server) AddEventOrderIdsToChannels(message *AddEventOrderIdsMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - if len(message.Data) == 0 { - *reply = s.failure(ServerErrorCode, "empty event data") - return nil - } - - s.eventWatcher.AddEventOrderIdsToChannels(message.Data) - - *reply = s.success("") - return nil -} - -// You must subscribe in advance according to the ClientOids subscription, -// or you will miss the receive message because of without the mapping relationship between message and orderId~ -func (s *Server) AddEventClientOidsToChannels(message *AddEventClientOidsMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - if len(message.Data) == 0 { - *reply = s.failure(ServerErrorCode, "empty event data") - return nil - } - - s.eventWatcher.AddEventClientOidsToChannels(message.Data) - - *reply = s.success("") - return nil -} diff --git a/api/monitor.go b/api/monitor.go deleted file mode 100644 index dc50c46..0000000 --- a/api/monitor.go +++ /dev/null @@ -1,45 +0,0 @@ -package api - -import ( - "encoding/json" - "time" -) - -func (s *Server) GetChanLen(message *TokenMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - ret := map[string]int{ - "level3Builder.Messages": len(s.level3Builder.Messages), - "eventWatcher.Messages": len(s.eventWatcher.Messages), - } - - if data, err := json.Marshal(ret); err == nil { - *reply = s.success(string(data)) - return nil - } - - *reply = s.failure(ServerErrorCode, "json failed") - return nil -} - -func (s *Server) Time(message *TokenMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - ret := map[string]int64{ - "time": time.Now().Unix(), - } - - if data, err := json.Marshal(ret); err == nil { - *reply = s.success(string(data)) - return nil - } - - *reply = s.failure(ServerErrorCode, "json failed") - return nil -} diff --git a/api/order_book.go b/api/order_book.go deleted file mode 100644 index 0718132..0000000 --- a/api/order_book.go +++ /dev/null @@ -1,55 +0,0 @@ -package api - -func (s *Server) GetOrderBook(message *TokenMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - data, err := s.level3Builder.SnapshotBytes() - if err != nil { - *reply = s.failure(TickerErrorCode, err.Error()) - return nil - } - - *reply = s.success(string(data)) - return nil -} - -type GetPartOrderBookMessage struct { - Number int `json:"number"` - TokenMessage -} - -func (s *Server) GetPartOrderBook(message *GetPartOrderBookMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - data, err := s.level3Builder.GetPartOrderBook(message.Number) - if err != nil { - *reply = s.failure(TickerErrorCode, err.Error()) - return nil - } - - *reply = s.success(string(data)) - return nil -} - - -func (s *Server) GetTicker(message *TokenMessage, reply *string) error { - if err := s.checkToken(message.Token); err != "" { - *reply = err - return nil - } - - data, err := s.level3Builder.GetTicker() - if err != nil { - *reply = s.failure(TickerErrorCode, err.Error()) - return nil - } - - *reply = s.success(string(data)) - return nil -} diff --git a/app/app.go b/app/app.go deleted file mode 100644 index 6fa1a4a..0000000 --- a/app/app.go +++ /dev/null @@ -1,146 +0,0 @@ -package app - -import ( - "encoding/json" - "os" - "strconv" - - "github.com/Kucoin/kucoin-go-sdk" - "github.com/Kucoin/kucoin-level3-sdk/api" - "github.com/Kucoin/kucoin-level3-sdk/builder" - "github.com/Kucoin/kucoin-level3-sdk/events" - "github.com/Kucoin/kucoin-level3-sdk/service" - "github.com/Kucoin/kucoin-level3-sdk/utils/log" -) - -type App struct { - apiService *kucoin.ApiService - symbol string - - enableOrderBook bool - level3Builder *builder.Builder - - enableEventWatcher bool - eventWatcher *events.Watcher - redisPool *service.Redis - - rpcPort string - rpcToken string -} - -func NewApp(symbol string, rpcPort string, rpcKey string) *App { - if symbol == "" { - panic("symbol is required") - } - - if rpcPort == "" { - panic("rpcPort is required") - } - - if rpcKey == "" { - panic("rpckey is required") - } - - apiService := kucoin.NewApiServiceFromEnv() - level3Builder := builder.NewBuilder(apiService, symbol) - - var redisHost = os.Getenv("REDIS_HOST") - var redisPassword = os.Getenv("REDIS_PASSWORD") - var redisDBEnv = os.Getenv("REDIS_DB") - var redisDB = 0 - if redisDBEnv != "" { - redisDB, _ = strconv.Atoi(redisDBEnv) - } - redisPool := service.NewRedis(redisHost, redisPassword, redisDB, rpcKey, symbol, rpcPort) - - eventWatcher := events.NewWatcher(redisPool) - - return &App{ - apiService: apiService, - symbol: symbol, - - enableOrderBook: os.Getenv("ENABLE_ORDER_BOOK") == "true", - level3Builder: level3Builder, - - enableEventWatcher: os.Getenv("ENABLE_EVENT_WATCHER") == "true", - redisPool: redisPool, - eventWatcher: eventWatcher, - - rpcPort: rpcPort, - rpcToken: os.Getenv("RPC_TOKEN"), - } -} - -func (app *App) Run() { - if app.enableOrderBook { - go app.level3Builder.ReloadOrderBook() - } - - if app.enableEventWatcher { - go app.eventWatcher.Run() - } - - //rpc server - go api.InitRpcServer(app.rpcPort, app.rpcToken, app.level3Builder, app.eventWatcher) - - app.websocket() -} - -func (app *App) writeMessage(msgRawData json.RawMessage) { - //log.Info("raw message : %s", kucoin.ToJsonString(msgRawData)) - if app.enableOrderBook { - app.level3Builder.Messages <- msgRawData - } - - if app.enableEventWatcher { - app.eventWatcher.Messages <- msgRawData - } - - const msgLenLimit = 50 - if len(app.level3Builder.Messages) > msgLenLimit || - len(app.eventWatcher.Messages) > msgLenLimit { - log.Error( - "msgLenLimit: app.level3Builder.Messages: %d, app.eventWatcher.Messages: %d, app.verify.Messages: %d", - len(app.level3Builder.Messages), - len(app.eventWatcher.Messages), - ) - } -} - -func (app *App) websocket() { - //todo recover dingTalk ? - apiService := app.apiService - - rsp, err := apiService.WebSocketPublicToken() - if err != nil { - panic(err) - } - - tk := &kucoin.WebSocketTokenModel{} - if err := rsp.ReadData(tk); err != nil { - panic(err) - } - - c := apiService.NewWebSocketClient(tk) - - mc, ec, err := c.Connect() - if err != nil { - panic(err) - } - - ch := kucoin.NewSubscribeMessage("/market/level3:"+app.symbol, false) - if err := c.Subscribe(ch); err != nil { - panic(err) - } - - for { - select { - case err := <-ec: - c.Stop() - panic(err) - - case msg := <-mc: - app.writeMessage(msg.RawData) - } - } -} diff --git a/build.sh b/build.sh index 74172a3..9102bfa 100755 --- a/build.sh +++ b/build.sh @@ -1,8 +1,9 @@ #!/bin/bash -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags '-s -w' -o kucoin-market-for-linux kucoin_market.go +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags '-s -w' -o kucoin-market-for-linux cmd/main/market.go -CGO_ENABLED=0 go build -ldflags '-s -w' -o kucoin-market-for-mac kucoin_market.go +CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags '-s -w' -o kucoin-market-for-mac cmd/main/market.go -CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags '-s -w' -o kucoin-market-for-windows.exe kucoin_market.go +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags '-s -w' -o kucoin-market-for-windows.exe cmd/main/market.go +echo "done" diff --git a/builder/builder.go b/builder/builder.go deleted file mode 100644 index cba09b7..0000000 --- a/builder/builder.go +++ /dev/null @@ -1,343 +0,0 @@ -package builder - -import ( - "encoding/json" - "errors" - "fmt" - "sync" - - "github.com/JetBlink/orderbook/base" - "github.com/JetBlink/orderbook/level3" - "github.com/Kucoin/kucoin-go-sdk" - "github.com/Kucoin/kucoin-level3-sdk/helper" - "github.com/Kucoin/kucoin-level3-sdk/level3stream" - "github.com/Kucoin/kucoin-level3-sdk/utils/log" - "github.com/shopspring/decimal" -) - -type Builder struct { - apiService *kucoin.ApiService - symbol string - lock *sync.RWMutex - Messages chan json.RawMessage - - fullOrderBook *level3.OrderBook - - latestMatch struct { - Price string `json:"price"` - Size string `json:"size"` - } -} - -func NewBuilder(apiService *kucoin.ApiService, symbol string) *Builder { - return &Builder{ - apiService: apiService, - symbol: symbol, - lock: &sync.RWMutex{}, - Messages: make(chan json.RawMessage, helper.MaxMsgChanLen*1024), - } -} - -func (b *Builder) resetOrderBook() { - b.lock.Lock() - b.fullOrderBook = level3.NewOrderBook() - b.lock.Unlock() -} - -func (b *Builder) ReloadOrderBook() { - //defer func() { - // if r := recover(); r != nil { - // log.Error("ReloadOrderBook panic : %v", r) - // b.ReloadOrderBook() - // } - //}() - - log.Warn("start running ReloadOrderBook, symbol: %s", b.symbol) - b.resetOrderBook() - - b.playback() - - for msg := range b.Messages { - l3Data, err := level3stream.NewStreamDataModel(msg) - if err != nil { - panic(err) - } - b.updateFromStream(l3Data) - } -} - -func (b *Builder) playback() { - log.Warn("prepare playback...") - - const tempMsgChanMaxLen = 10240 - tempMsgChan := make(chan *level3stream.StreamDataModel, tempMsgChanMaxLen) - firstSequence := "" - var fullOrderBook *DepthResponse - - for msg := range b.Messages { - l3Data, err := level3stream.NewStreamDataModel(msg) - if err != nil { - panic(err) - } - - tempMsgChan <- l3Data - - if firstSequence == "" { - firstSequence = l3Data.Sequence - log.Error("firstSequence: %s", firstSequence) - } - - if len(tempMsgChan) > 5 { - if fullOrderBook == nil { - log.Warn("start getting full level3 order book data, symbol: %s", b.symbol) - fullOrderBook, err = b.GetAtomicFullOrderBook() - if err != nil { - panic(err) - continue - } - log.Error("got full level3 order book data, Sequence: %s", fullOrderBook.Sequence) - } - - if len(tempMsgChan) > tempMsgChanMaxLen-5 { - panic("playback failed, tempMsgChan is too long, retry...") - } - - if fullOrderBook != nil && fullOrderBook.Sequence < firstSequence { - log.Error("full data Sequence %s is too small", fullOrderBook.Sequence) - fullOrderBook = nil - continue - } - - if fullOrderBook != nil && fullOrderBook.Sequence <= l3Data.Sequence { - log.Warn("sequence match, start playback, tempMsgChan: %d", len(tempMsgChan)) - - b.lock.Lock() - b.AddDepthToOrderBook(fullOrderBook) - b.lock.Unlock() - - n := len(tempMsgChan) - for i := 0; i < n; i++ { - b.updateFromStream(<-tempMsgChan) - } - - log.Warn("finish playback.") - break - } - } - } -} - -func (b *Builder) AddDepthToOrderBook(depth *DepthResponse) { - b.fullOrderBook.Sequence = helper.ParseUint64OrPanic(depth.Sequence) - - for index, elem := range depth.Asks { - order, err := level3.NewOrder(elem[0], base.AskSide, elem[1], elem[2], uint64(index), nil) - if err != nil { - panic(err) - } - - if err := b.fullOrderBook.AddOrder(order); err != nil { - panic(err) - } - } - - for index, elem := range depth.Bids { - order, err := level3.NewOrder(elem[0], base.BidSide, elem[1], elem[2], uint64(index), nil) - if err != nil { - panic(err) - } - - if err := b.fullOrderBook.AddOrder(order); err != nil { - panic(err) - } - } -} - -func (b *Builder) updateFromStream(msg *level3stream.StreamDataModel) { - //time.Now().UnixNano() - //log.Info("msg: %s", string(msg.GetRawMessage())) - - b.lock.Lock() - defer b.lock.Unlock() - - skip, err := b.updateSequence(msg) - if err != nil { - panic(err) - } - - if !skip { - b.updateOrderBook(msg) - } -} - -func (b *Builder) updateSequence(msg *level3stream.StreamDataModel) (bool, error) { - fullOrderBookSequenceValue := b.fullOrderBook.Sequence - msgSequenceValue := helper.ParseUint64OrPanic(msg.Sequence) - - if fullOrderBookSequenceValue+1 > msgSequenceValue { - return true, nil - } - - if fullOrderBookSequenceValue+1 != msgSequenceValue { - return false, errors.New(fmt.Sprintf( - "currentSequence: %d, msgSequence: %s, the sequence is not continuous, 当前chanLen: %d", - b.fullOrderBook.Sequence, - msg.Sequence, - len(b.Messages), - )) - } - - //更新 - //!!! sequence 需要更新，通过判断 sequence 是否自增来校验数据完整性，否则重放数据。 - b.fullOrderBook.Sequence = msgSequenceValue - - return false, nil -} - -func (b *Builder) updateOrderBook(msg *level3stream.StreamDataModel) { - //[3]string{"orderId", "price", "size"} - //var item = [3]string{msg.OrderId, msg.Price, msg.Size} - - side := "" - switch msg.Side { - case level3stream.SellSide: - side = base.AskSide - case level3stream.BuySide: - side = base.BidSide - default: - panic("error side: " + msg.Side) - } - - switch msg.Type { - case level3stream.MessageReceivedType: - case level3stream.MessageOpenType: - data := &level3stream.StreamDataOpenModel{} - if err := json.Unmarshal(msg.GetRawMessage(), data); err != nil { - panic(err) - } - - if data.Price == "" || data.Size == "0" { - return - } - - order, err := level3.NewOrder(data.OrderId, side, data.Price, data.Size, helper.ParseUint64OrPanic(data.Time), nil) - if err != nil { - log.Error(string(msg.GetRawMessage())) - panic(err) - } - if err := b.fullOrderBook.AddOrder(order); err != nil { - panic(err) - } - case level3stream.MessageDoneType: - data := &level3stream.StreamDataDoneModel{} - if err := json.Unmarshal(msg.GetRawMessage(), data); err != nil { - panic(err) - } - if err := b.fullOrderBook.RemoveByOrderId(data.OrderId); err != nil { - panic(err) - } - - case level3stream.MessageMatchType: - data := &level3stream.StreamDataMatchModel{} - if err := json.Unmarshal(msg.GetRawMessage(), data); err != nil { - panic(err) - } - sizeValue, err := decimal.NewFromString(data.Size) - if err != nil { - panic(err) - } - if err := b.fullOrderBook.MatchOrder(data.MakerOrderId, sizeValue); err != nil { - panic(err) - } - b.latestMatch.Price = data.Price - b.latestMatch.Size = data.Size - - case level3stream.MessageChangeType: - data := &level3stream.StreamDataChangeModel{} - if err := json.Unmarshal(msg.GetRawMessage(), data); err != nil { - panic(err) - } - sizeValue, err := decimal.NewFromString(data.NewSize) - if err != nil { - panic(err) - } - if err := b.fullOrderBook.ChangeOrder(data.OrderId, sizeValue); err != nil { - panic(err) - } - - default: - panic("error msg type: " + msg.Type) - } -} - -func (b *Builder) Snapshot() (*FullOrderBook, error) { - data, err := b.SnapshotBytes() - if err != nil { - return nil, err - } - - ret := &FullOrderBook{} - if err := json.Unmarshal(data, ret); err != nil { - return nil, err - } - - return ret, nil -} - -func (b *Builder) SnapshotBytes() ([]byte, error) { - b.lock.RLock() - data, err := json.Marshal(b.fullOrderBook) - b.lock.RUnlock() - if err != nil { - return nil, err - } - - return data, nil -} - -func (b *Builder) GetPartOrderBook(number int) ([]byte, error) { - defer func() { - if r := recover(); r != nil { - log.Error("GetPartOrderBook panic : %v", r) - } - }() - - b.lock.RLock() - defer b.lock.RUnlock() - - data, err := json.Marshal(map[string]interface{}{ - "sequence": b.fullOrderBook.Sequence, - base.AskSide: b.fullOrderBook.GetPartOrderBookBySide(base.AskSide, number), - base.BidSide: b.fullOrderBook.GetPartOrderBookBySide(base.BidSide, number), - }) - - if err != nil { - return nil, err - } - - return data, nil -} - -func (b *Builder) GetTicker() ([]byte, error) { - defer func() { - if r := recover(); r != nil { - log.Error("GetTicker panic : %v", r) - } - }() - - b.lock.RLock() - defer b.lock.RUnlock() - - data, err := json.Marshal(map[string]interface{}{ - "sequence": b.fullOrderBook.Sequence, - "match": b.latestMatch, - base.AskSide: b.fullOrderBook.GetPartOrderBookBySide(base.AskSide, 1), - base.BidSide: b.fullOrderBook.GetPartOrderBookBySide(base.BidSide, 1), - }) - - if err != nil { - return nil, err - } - - return data, nil -} diff --git a/builder/depth.go b/builder/depth.go deleted file mode 100644 index 6b46299..0000000 --- a/builder/depth.go +++ /dev/null @@ -1,33 +0,0 @@ -package builder - -import "errors" - -type DepthResponse struct { - Sequence string `json:"sequence"` - Asks [][3]string `json:"asks"` - Bids [][3]string `json:"bids"` -} - -type FullOrderBook struct { - Sequence uint64 `json:"sequence"` - Asks [][3]string `json:"asks"` - Bids [][3]string `json:"bids"` -} - -func (b *Builder) GetAtomicFullOrderBook() (*DepthResponse, error) { - rsp, err := b.apiService.AtomicFullOrderBook(b.symbol) - if err != nil { - return nil, err - } - - c := &DepthResponse{} - if err := rsp.ReadData(c); err != nil { - return nil, err - } - - if c.Sequence == "" { - return nil, errors.New("empty key sequence") - } - - return c, nil -} diff --git a/cmd/main/market.go b/cmd/main/market.go new file mode 100644 index 0000000..fbb83c1 --- /dev/null +++ b/cmd/main/market.go @@ -0,0 +1,7 @@ +package main + +import "github.com/Kucoin/kucoin-level3-sdk/cmd" + +func main() { + cmd.Execute() +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..97f4977 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,26 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "market", + Short: "The market app", + Long: "The market application", + Run: func(cmd *cobra.Command, args []string) { + if err := cmd.Help(); err != nil { + panic(err) + } + }, +} + +func init() { + rootCmd.AddCommand(startCmd) +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + panic(err) + } +} diff --git a/cmd/start.go b/cmd/start.go new file mode 100644 index 0000000..0c41f02 --- /dev/null +++ b/cmd/start.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "github.com/Kucoin/kucoin-level3-sdk/pkg/bootstrap" + "github.com/spf13/cobra" +) + +var startCmd = &cobra.Command{ + Use: "start", + Short: "Start app", + Long: "Start the market application", + Run: func(cmd *cobra.Command, args []string) { + cfgFile, err := cmd.Flags().GetString("config") + if err != nil { + panic(err) + } + + bootstrap.Run(cfgFile, cmd.Flags()) + }, +} + +func init() { + startCmd.Flags().StringP("config", "c", "config.yaml", "app config file") + startCmd.Flags().StringP("symbol", "s", "", "symbol") +} diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..360a39c --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,24 @@ +app_debug: true + +symbol: KCS-USDT +#symbol: XBTUSDM + +app: + name: market + log_file: "./runtime/log/market.log" + +market.kucoin_v2: + url: "https://api.kucoin.com" + type: "spot" + # url: "https://api-futures.kucoin.com" + # type: "future" + +api_server: + network: tcp + address: 0.0.0.0:9090 + token: your-rpc-token + +redis: + addr: 127.0.0.1:6379 + password: "" + db: 0 diff --git a/demo/python-demo/__pycache__/config.cpython-37.pyc b/demo/python-demo/__pycache__/config.cpython-37.pyc new file mode 100644 index 0000000..10cc916 Binary files /dev/null and b/demo/python-demo/__pycache__/config.cpython-37.pyc differ diff --git a/demo/python-demo/config.py b/demo/python-demo/config.py index 8b70f8c..988442f 100644 --- a/demo/python-demo/config.py +++ b/demo/python-demo/config.py @@ -1,5 +1,5 @@ rpc_config = { 'host': '127.0.0.1', 'port': 9090, - 'token': 'market-token' + 'token': 'your-rpc-token' } diff --git a/demo/python-demo/level3/__init__.pyc b/demo/python-demo/level3/__init__.pyc new file mode 100644 index 0000000..8bb9300 Binary files /dev/null and b/demo/python-demo/level3/__init__.pyc differ diff --git a/demo/python-demo/level3/__pycache__/__init__.cpython-37.pyc b/demo/python-demo/level3/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..ba8c865 Binary files /dev/null and b/demo/python-demo/level3/__pycache__/__init__.cpython-37.pyc differ diff --git a/demo/python-demo/level3/__pycache__/rpc.cpython-37.pyc b/demo/python-demo/level3/__pycache__/rpc.cpython-37.pyc new file mode 100644 index 0000000..28993d8 Binary files /dev/null and b/demo/python-demo/level3/__pycache__/rpc.cpython-37.pyc differ diff --git a/demo/python-demo/level3/rpc.py b/demo/python-demo/level3/rpc.py index 74d1023..f429301 100644 --- a/demo/python-demo/level3/rpc.py +++ b/demo/python-demo/level3/rpc.py @@ -49,12 +49,12 @@ def execute(self, data: map) -> map: if response.get('error') is not None: raise Exception(response.get('error')) - result = json.loads(response.get('result')) + result = response.get('result') if result['code'] != '0': raise Exception("rpc execute fail: %s" % result['error']) - return result + return result['data'] def close(self): """ @@ -81,25 +81,12 @@ def call(self, method: str, **kwargs): return self.execute(data) - def get_ticker(self, number): - result = self.call("GetPartOrderBook", number=number) - ticker = json.loads(result['data']) - if ticker['sequence'] == 0: - raise Exception("rpc get ticker fail: sequence is null") - return ticker - - def get_all_ticker(self): - result = self.call("GetOrderBook") - ticker = json.loads(result['data']) - if ticker['sequence'] == 0: - raise Exception("rpc get all ticker fail: sequence is null") - return ticker - - def add_event_order_id(self, data, channel): - args = {} - for i in data: - args[i] = [channel] - return self.call("AddEventOrderIdsToChannels", data=args) + def get_order_book(self, number): + order_book = self.call("GetOrderBook", number=number) + if len(order_book['asks']) == 0 or len(order_book['bids']) == 0: + raise Exception("empty order book") + + return order_book def add_event_client_id(self, data, channel): args = {} diff --git a/demo/python-demo/order_book_demo.py b/demo/python-demo/order_book_demo.py index 6569338..3e588f5 100644 --- a/demo/python-demo/order_book_demo.py +++ b/demo/python-demo/order_book_demo.py @@ -15,46 +15,22 @@ else: raise Exception('unsupported system') - while True: - rpc = RPC(rpc_config['host'], rpc_config['port'], rpc_config['token']) - - data = rpc.get_ticker(100) - - asks = data['asks'] - bids = data['bids'] - - price_list = [{}, {}] + rpc = RPC(rpc_config['host'], rpc_config['port'], rpc_config['token']) - for ask in asks: - if ask[1] not in price_list[0].keys(): - price_list[0].update({ask[1]: Decimal(ask[2])}) - else: - price_list[0].update({ - ask[1]: price_list[0][ask[1]] + Decimal(ask[2]) - }) - if len(price_list[0]) >= 13: - price_list[0].pop(ask[1]) - break - - for bid in bids: - if bid[1] not in price_list[1].keys(): - price_list[1].update({bid[1]: Decimal(bid[2])}) - else: - price_list[1].update({ - bid[1]: price_list[1][bid[1]] + Decimal(bid[2]) - }) - if len(price_list[1]) >= 13: - price_list[1].pop(bid[1]) - break - - d1 = sorted(price_list[0].items(), key=lambda v: v[0], reverse=True) - d2 = sorted(price_list[1].items(), key=lambda v: v[0], reverse=True) + while True: + order_book = rpc.get_order_book(11) +# import sys, json +# print(json.dumps(order_book)) +# sys.exit(0) os.system(cmd) - for d in d1: + asks = order_book['asks'] + asks.reverse() + + for d in asks: print("{} => {}".format(d[0], d[1])) print("---Spread---") - for d in d2: + for d in order_book['bids']: print("{} => {}".format(d[0], d[1])) time.sleep(0.5) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index 2f90106..0000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -# first arg is `-f` or `--some-option` -if [ "${1#-}" != "$1" ]; then - set -- kucoin_market "$@" -fi - -exec "$@" diff --git a/events/l3_events.go b/events/l3_events.go deleted file mode 100644 index ec255d2..0000000 --- a/events/l3_events.go +++ /dev/null @@ -1,180 +0,0 @@ -package events - -import ( - "encoding/json" - "sync" - - "github.com/Kucoin/kucoin-level3-sdk/helper" - "github.com/Kucoin/kucoin-level3-sdk/level3stream" - "github.com/Kucoin/kucoin-level3-sdk/service" - "github.com/Kucoin/kucoin-level3-sdk/utils/log" -) - -type Watcher struct { - Messages chan json.RawMessage - redisPool *service.Redis - lock *sync.RWMutex - - orderIds map[string]map[string]bool - clientOids map[string]map[string]bool -} - -func NewWatcher(redisPool *service.Redis) *Watcher { - return &Watcher{ - Messages: make(chan json.RawMessage, helper.MaxMsgChanLen), - redisPool: redisPool, - lock: &sync.RWMutex{}, - - orderIds: make(map[string]map[string]bool), - clientOids: make(map[string]map[string]bool), - } -} - -//todo 以后增加 对 已经交易和完成订单的跟踪，方便本地维护订单的状态 -func (w *Watcher) Run() { - log.Warn("start running Watcher") - - for msg := range w.Messages { - if !w.existEventOrderIds() { - continue - } - - l3Data, err := level3stream.NewStreamDataModel(msg) - if err != nil { - panic(err) - } - - switch l3Data.Type { - case level3stream.MessageReceivedType: - data := &level3stream.StreamDataReceivedModel{} - if err := json.Unmarshal(l3Data.GetRawMessage(), data); err != nil { - panic(err) - } - - w.lock.RLock() - channelsMap, ok := w.clientOids[data.ClientOid] - channels := getMapKeys(channelsMap) - w.lock.RUnlock() - if ok { - w.removeEventClientOid(data.ClientOid) - w.AddEventOrderIdsToChannels(map[string][]string{ - data.OrderId: channels, - }) - } - - w.publish(data.OrderId, string(l3Data.GetRawMessage())) - - case level3stream.MessageOpenType: - data := &level3stream.StreamDataOpenModel{} - if err := json.Unmarshal(l3Data.GetRawMessage(), data); err != nil { - panic(err) - } - w.publish(data.OrderId, string(l3Data.GetRawMessage())) - - case level3stream.MessageMatchType: - data := &level3stream.StreamDataMatchModel{} - if err := json.Unmarshal(l3Data.GetRawMessage(), data); err != nil { - panic(err) - } - - w.publish(data.MakerOrderId, string(l3Data.GetRawMessage())) - w.publish(data.TakerOrderId, string(l3Data.GetRawMessage())) - - case level3stream.MessageDoneType: - data := &level3stream.StreamDataDoneModel{} - if err := json.Unmarshal(l3Data.GetRawMessage(), data); err != nil { - panic(err) - } - - w.publish(data.OrderId, string(l3Data.GetRawMessage())) - w.removeEventOrderId(data.OrderId) - - case level3stream.MessageChangeType: - data := &level3stream.StreamDataChangeModel{} - if err := json.Unmarshal(l3Data.GetRawMessage(), data); err != nil { - panic(err) - } - - w.publish(data.OrderId, string(l3Data.GetRawMessage())) - - default: - panic("error msg type: " + l3Data.Type) - } - } -} - -func getMapKeys(data map[string]bool) []string { - keys := make([]string, 0, len(data)) - for k := range data { - keys = append(keys, k) - } - - return keys -} - -func (w *Watcher) publish(orderId string, message string) { - w.lock.RLock() - channelsMap, ok := w.orderIds[orderId] - channels := getMapKeys(channelsMap) - w.lock.RUnlock() - - if ok { - for _, channel := range channels { - if err := w.redisPool.Publish(channel, message); err != nil { - log.Error("redis publish to %s, msg: %s, error: %s", channel, message, err.Error()) - return - } - } - } -} - -func (w *Watcher) existEventOrderIds() bool { - w.lock.RLock() - defer w.lock.RUnlock() - if len(w.orderIds) == 0 && len(w.clientOids) == 0 { - return false - } - - return true -} - -func (w *Watcher) AddEventOrderIdsToChannels(data map[string][]string) { - w.lock.Lock() - defer w.lock.Unlock() - - for orderId, channels := range data { - for _, channel := range channels { - if w.orderIds[orderId] == nil { - w.orderIds[orderId] = make(map[string]bool) - } - w.orderIds[orderId][channel] = true - } - } -} - -func (w *Watcher) AddEventClientOidsToChannels(data map[string][]string) { - w.lock.Lock() - for clientOid, channels := range data { - for _, channel := range channels { - if w.clientOids[clientOid] == nil { - w.clientOids[clientOid] = make(map[string]bool) - } - w.clientOids[clientOid][channel] = true - } - } - w.lock.Unlock() -} - -func (w *Watcher) removeEventOrderId(orderId string) { - w.lock.Lock() - defer w.lock.Unlock() - - delete(w.orderIds, orderId) -} - -func (w *Watcher) removeEventClientOid(clientOid string) { - w.lock.Lock() - defer w.lock.Unlock() - - delete(w.clientOids, clientOid) -} diff --git a/go.mod b/go.mod index 9cb62a6..15e6207 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,20 @@ module github.com/Kucoin/kucoin-level3-sdk -go 1.13 +go 1.15 require ( - github.com/JetBlink/orderbook v1.0.0 - github.com/Kucoin/kucoin-go-sdk v1.1.10 - github.com/go-redis/redis v6.15.6+incompatible - github.com/joho/godotenv v1.3.0 - github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 + github.com/go-playground/validator/v10 v10.2.0 + github.com/go-redis/redis v6.15.7+incompatible + github.com/gorilla/websocket v1.4.2 + github.com/mitchellh/mapstructure v1.2.2 + github.com/onsi/gomega v1.9.0 // indirect + github.com/pkg/errors v0.8.1 + github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc + github.com/spf13/cobra v1.0.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.6.3 + github.com/subosito/gotenv v1.2.0 + go.uber.org/zap v1.14.1 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..523d5ce --- /dev/null +++ b/go.sum @@ -0,0 +1,238 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= +github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= +github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/helper/helper.go b/helper/helper.go deleted file mode 100644 index e8c5c52..0000000 --- a/helper/helper.go +++ /dev/null @@ -1,16 +0,0 @@ -package helper - -import ( - "strconv" -) - -const MaxMsgChanLen = 1024 - -func ParseUint64OrPanic(item string) uint64 { - value, err := strconv.ParseUint(item, 10, 64) - if err != nil { - panic(err) - } - - return value -} diff --git a/helper/str/string.go b/helper/str/string.go deleted file mode 100644 index e887684..0000000 --- a/helper/str/string.go +++ /dev/null @@ -1,28 +0,0 @@ -package str - -import ( - "errors" - - "github.com/shopspring/decimal" -) - -func Diff(a string, b string) error { - if a == b { - return nil - } - - aF, err := decimal.NewFromString(a) - if err != nil { - return err - } - bF, err := decimal.NewFromString(b) - if err != nil { - return err - } - - if !aF.Equal(bF) { - return errors.New("not equal: " + a + " != " + b) - } - - return nil -} diff --git a/kucoin_market.go b/kucoin_market.go deleted file mode 100644 index 1a0ba5b..0000000 --- a/kucoin_market.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "flag" - "github.com/Kucoin/kucoin-level3-sdk/app" - - "github.com/Kucoin/kucoin-level3-sdk/utils/log" - "github.com/joho/godotenv" -) - -func main() { - envFile := flag.String("c", ".env", ".env file") - symbol := flag.String("symbol", "", "SYMBOL") - rpcPort := flag.String("p", "", "rpc port") - rpcKey := flag.String("rpckey", "", "market maker redis rpckey") - - flag.Parse() - - loadEnv(*envFile) - app.NewApp(*symbol, *rpcPort, *rpcKey).Run() -} - -func loadEnv(file string) { - err := godotenv.Load(file) - if err != nil { - log.Error("Error loading .env file: %s", file) - } -} diff --git a/level3stream/stream_data_model.go b/level3stream/stream_data_model.go deleted file mode 100644 index 68b4899..0000000 --- a/level3stream/stream_data_model.go +++ /dev/null @@ -1,93 +0,0 @@ -package level3stream - -import ( - "encoding/json" -) - -type StreamDataModel struct { - Sequence string `json:"sequence"` - Symbol string `json:"symbol"` - Type string `json:"type"` - Side string `json:"side"` - rawMessage json.RawMessage -} - -func NewStreamDataModel(msgData json.RawMessage) (*StreamDataModel, error) { - l3Data := &StreamDataModel{} - - if err := json.Unmarshal(msgData, l3Data); err != nil { - return nil, err - } - l3Data.rawMessage = msgData - - return l3Data, nil -} - -func (l3Data *StreamDataModel) GetRawMessage() json.RawMessage { - return l3Data.rawMessage -} - -const ( - BuySide = "buy" - SellSide = "sell" - - LimitOrderType = "limit" - MarketOrderType = "market" - - MessageDoneCanceled = "canceled" - MessageDoneFilled = "filled" - - MessageReceivedType = "received" - MessageOpenType = "open" - MessageDoneType = "done" - MessageMatchType = "match" - MessageChangeType = "change" -) - -type StreamDataReceivedModel struct { - OrderType string `json:"orderType"` - Side string `json:"side"` - //Size string `json:"size"` - Price string `json:"price"` - //Funds string `json:"funds"` - OrderId string `json:"orderId"` - Time string `json:"time"` - ClientOid string `json:"clientOid"` -} - -type StreamDataOpenModel struct { - Side string `json:"side"` - Size string `json:"size"` - OrderId string `json:"orderId"` - Price string `json:"price"` - Time string `json:"time"` - //RemainSize string `json:"remainSize"` -} - -type StreamDataDoneModel struct { - Side string `json:"side"` - Size string `json:"size"` - Reason string `json:"reason"` - OrderId string `json:"orderId"` - Price string `json:"price"` - Time string `json:"time"` -} - -type StreamDataMatchModel struct { - Side string `json:"side"` - Size string `json:"size"` - Price string `json:"price"` - TakerOrderId string `json:"takerOrderId"` - MakerOrderId string `json:"makerOrderId"` - Time string `json:"time"` - TradeId string `json:"tradeId"` -} - -type StreamDataChangeModel struct { - Side string `json:"side"` - NewSize string `json:"newSize"` - OldSize string `json:"oldSize"` - Price string `json:"price"` - OrderId string `json:"orderId"` - Time string `json:"time"` -} diff --git a/pkg/api/anycall.go b/pkg/api/anycall.go new file mode 100644 index 0000000..be5c5de --- /dev/null +++ b/pkg/api/anycall.go @@ -0,0 +1,28 @@ +package api + +import ( + "encoding/json" +) + +type AnyCallMessage struct { + TokenMessage + Method string `json:"method"` + Args json.RawMessage `json:"args"` +} + +func (s *Server) AnyCall(message *AnyCallMessage, reply *Response) error { + if errResp := s.checkToken(message.Token); errResp != nil { + *reply = *errResp + return nil + } + //log.Debug("AnyCall method: " + message.Method + ", args: " + string(message.Args)) + + data, err := s.app.AnyCall(message.Method, message.Args) + if err != nil { + *reply = s.failure(ServerErrorCode, err.Error()) + return nil + } + + *reply = s.success(data) + return nil +} diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..9e9e895 --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,101 @@ +package api + +import ( + "net" + "net/rpc" + "net/rpc/jsonrpc" + "os" + "strings" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/app" + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "go.uber.org/zap" +) + +//Server is api server +type Server struct { + app *app.App +} + +func fixMarketName(market string) string { + return strings.Split(market, "_v")[0] +} + +//InitRpcServer init rpc server +func InitRpcServer(app *app.App) { + apiAddress := cfg.AppConfig.ApiServer.Address + + if err := rpc.Register(&Server{ + app: app, + }); err != nil { + log.Panic("rpc Register error: " + err.Error()) + } + + log.Info("start running rpc server, listen: " + cfg.AppConfig.ApiServer.Network + "://" + apiAddress) + + if strings.HasPrefix(cfg.AppConfig.ApiServer.Network, "unix") { + err := os.Remove(apiAddress) + switch { + case os.IsNotExist(err), err == nil: + default: + log.Panic("remove socket failed", zap.Error(err)) + } + } + listener, err := net.Listen(cfg.AppConfig.ApiServer.Network, apiAddress) + if err != nil { + log.Panic("api server run failed, error: " + err.Error()) + } + defer func() { _ = listener.Close() }() + for { + conn, err := listener.Accept() + if err != nil { + continue + } + go jsonrpc.ServeConn(conn) + } +} + +//TokenMessage is token type message +type TokenMessage struct { + Token string `json:"token"` +} + +//Response is api response +type Response struct { + Code string `json:"code"` + Data interface{} `json:"data"` + Error string `json:"error"` +} + +func (s *Server) checkToken(token string) *Response { + if token != cfg.AppConfig.ApiServer.Token { + resp := s.failure(TokenErrorCode, "error rpc token") + return &resp + } + + return nil +} + +func (s *Server) success(data interface{}) Response { + return Response{ + Code: "0", + Data: data, + Error: "", + } +} + +const ( + ServerErrorCode = "10" + TokenErrorCode = "20" + TickerErrorCode = "30" + ConfNotFound = "40" +) + +func (s *Server) failure(code string, err string) Response { + return Response{ + Code: code, + Data: "", + Error: err, + } +} diff --git a/pkg/api/order_book.go b/pkg/api/order_book.go new file mode 100644 index 0000000..2bc8c2a --- /dev/null +++ b/pkg/api/order_book.go @@ -0,0 +1,16 @@ +package api + +type GetPartOrderBookMessage struct { + Number int `json:"number"` + TokenMessage +} + +func (s *Server) GetOrderBook(message *GetPartOrderBookMessage, reply *Response) error { + if errResp := s.checkToken(message.Token); errResp != nil { + *reply = *errResp + return nil + } + + *reply = s.success(s.app.PartOrderBook(message.Number)) + return nil +} diff --git a/pkg/api/order_watcher.go b/pkg/api/order_watcher.go new file mode 100644 index 0000000..ab323df --- /dev/null +++ b/pkg/api/order_watcher.go @@ -0,0 +1,26 @@ +package api + +type AddEventClientOidsMessage struct { + Data map[string][]string `json:"data"` + TokenMessage +} + +func (s *Server) AddEventClientOidsToChannels(message *AddEventClientOidsMessage, reply *Response) error { + if errResp := s.checkToken(message.Token); errResp != nil { + *reply = *errResp + return nil + } + + if len(message.Data) == 0 { + *reply = s.failure(ServerErrorCode, "empty event data") + return nil + } + + if err := s.app.AddEventClientOidsToChannels(message.Data); err != nil { + *reply = s.failure(ServerErrorCode, err.Error()) + return nil + } + + *reply = s.success("") + return nil +} diff --git a/pkg/app/app.go b/pkg/app/app.go new file mode 100644 index 0000000..8a21f93 --- /dev/null +++ b/pkg/app/app.go @@ -0,0 +1,39 @@ +package app + +import ( + "encoding/json" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "go.uber.org/zap" +) + +type App struct { + exchange exchanges.Exchange +} + +func NewApp() *App { + exchange, err := exchanges.Load(cfg.AppConfig.MarketName()) + if err != nil { + log.Panic(err.Error(), zap.Error(err)) + } + + app := &App{ + exchange: exchange, + } + + return app +} + +func (app *App) PartOrderBook(number int) *exchanges.OrderBook { + return app.exchange.GetPartOrderBook(number) +} + +func (app *App) AddEventClientOidsToChannels(data map[string][]string) error { + return app.exchange.AddEventClientOidsToChannels(data) +} + +func (app *App) AnyCall(method string, args json.RawMessage) (interface{}, error) { + return app.exchange.AnyCall(method, args) +} diff --git a/pkg/bootstrap/app.go b/pkg/bootstrap/app.go new file mode 100644 index 0000000..1614f15 --- /dev/null +++ b/pkg/bootstrap/app.go @@ -0,0 +1,63 @@ +package bootstrap + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/api" + "github.com/Kucoin/kucoin-level3-sdk/pkg/app" + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + _ "github.com/Kucoin/kucoin-level3-sdk/pkg/includes" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/redis" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/helper" + "github.com/gorilla/websocket" + "github.com/shopspring/decimal" + "github.com/spf13/pflag" + "go.uber.org/zap" +) + +func Run(cfgFile string, flagSet *pflag.FlagSet) { + //load cfg + configFile, err := cfg.LoadConfig(cfgFile, flagSet, map[string]string{ + "symbol": "symbol", + }) + if err != nil { + panic(err) + } + + //init logger + log.New(cfg.AppConfig.AppDebug) + defer log.Sync() + + log.Info("using cfg file: " + configFile) + log.Debug("cfg data: " + helper.ToJsonString(cfg.AppConfig)) + + log.Info("init redis connections") + redis.InitConnections() + + decimal.MarshalJSONWithoutQuotes = true + + // websocket.DefaultDialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + websocket.DefaultDialer.ReadBufferSize = 2048000 //2000 kb + + // run market + marketApp := app.NewApp() + + // rpc server + go api.InitRpcServer(marketApp) + + fmt.Println("market finished bootstrap") + // Wait for interrupt signal to gracefully shutdown the server with + // a timeout of 5 seconds. + quit := make(chan os.Signal) + // kill (no param) default send syscall.SIGTERM + // kill -2 is syscall.SIGINT + // kill -9 is syscall.SIGKILL but can't be catch, so don't need add it + //register for interupt (Ctrl+C) and SIGTERM (docker) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + sig := <-quit + log.Warn("app showdown!!!", zap.String("signal", sig.String())) +} diff --git a/pkg/cfg/app.go b/pkg/cfg/app.go new file mode 100644 index 0000000..eb86b46 --- /dev/null +++ b/pkg/cfg/app.go @@ -0,0 +1,55 @@ +package cfg + +import ( + "github.com/go-playground/validator/v10" +) + +type appConf struct { + AppDebug bool `mapstructure:"app_debug"` + + Symbol string `mapstructure:"symbol" validate:"required"` + + App App `mapstructure:"app" validate:"required,dive"` + + ApiServer ApiServer `mapstructure:"api_server" validate:"required,dive"` + + Redis Redis `mapstructure:"redis"` + + Market map[string]interface{} `mapstructure:"market" validate:"required,len=1"` +} + +type App struct { + Name string `mapstructure:"name" validate:"required"` + LogFile string `mapstructure:"log_file" validate:"required"` +} + +type ApiServer struct { + Network string `mapstructure:"network" validate:"required"` + Address string `mapstructure:"address" validate:"required"` + Token string `mapstructure:"token" validate:"required"` +} + +type Redis struct { + Addr string `mapstructure:"addr" validate:"required"` + Password string `mapstructure:"password"` + Db int `mapstructure:"db"` +} + +func (cfg appConf) MarketName() string { + for name := range cfg.Market { + return name + } + + panic("undefined market name") +} + +func (cfg *appConf) Unpack(output interface{}) error { + marketName := cfg.MarketName() + marketCfg := cfg.Market[marketName] + if err := mapStructureParse(marketCfg, output); err != nil { + return err + } + + validate := validator.New() + return validate.Struct(output) +} diff --git a/pkg/cfg/load.go b/pkg/cfg/load.go new file mode 100644 index 0000000..22a3116 --- /dev/null +++ b/pkg/cfg/load.go @@ -0,0 +1,89 @@ +package cfg + +import ( + "os" + "reflect" + + "github.com/go-playground/validator/v10" + "github.com/mitchellh/mapstructure" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/subosito/gotenv" +) + +var AppConfig = &appConf{} + +var defaultMapStructureDecoderConfig = []viper.DecoderConfigOption{ + func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + //config.TagName = "yaml" + config.DecodeHook = mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + func(f reflect.Kind, t reflect.Kind, data interface{}) (interface{}, error) { + if f != reflect.String || t != reflect.String { + return data, nil + } + return os.ExpandEnv(data.(string)), nil + }, + ) + }, +} + +func mapStructureParse(input interface{}, output interface{}) error { + config := &mapstructure.DecoderConfig{ + Metadata: nil, + Result: output, + WeaklyTypedInput: true, + } + for _, opt := range defaultMapStructureDecoderConfig { + opt(config) + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + + if err := decoder.Decode(input); err != nil { + return err + } + + return nil +} + +func unmarshalConfig(rawConfig map[string]interface{}, tmpInitCfg interface{}) error { + _ = gotenv.OverLoad(".env") + + if err := mapStructureParse(rawConfig, tmpInitCfg); err != nil { + return err + } + + validate := validator.New() + if err := validate.Struct(tmpInitCfg); err != nil { + return err + } + + return nil +} + +func LoadConfig(cfgFile string, flagSet *pflag.FlagSet, keys map[string]string) (string, error) { + v := viper.New() + v.SetConfigFile(cfgFile) + v.SetConfigType("yaml") + for key, name := range keys { + if err := v.BindPFlag(key, flagSet.Lookup(name)); err != nil { + return "", err + } + } + + if err := v.ReadInConfig(); err != nil { + return "", err + } + + err := unmarshalConfig(v.AllSettings(), AppConfig) + if err != nil { + panic(err) + } + return v.ConfigFileUsed(), nil +} diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go new file mode 100644 index 0000000..85ea664 --- /dev/null +++ b/pkg/consts/consts.go @@ -0,0 +1,5 @@ +package consts + +const ( + MaxMsgChanLen = 1024 * 10 +) diff --git a/pkg/exchanges/exchange.go b/pkg/exchanges/exchange.go new file mode 100644 index 0000000..b4d6509 --- /dev/null +++ b/pkg/exchanges/exchange.go @@ -0,0 +1,55 @@ +package exchanges + +import ( + "encoding/json" + "errors" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "go.uber.org/zap" +) + +type Exchange interface { + GetPartOrderBook(number int) *OrderBook + AddEventClientOidsToChannels(data map[string][]string) error + AnyCall(method string, args json.RawMessage) (interface{}, error) +} + +type OrderBook struct { + Asks interface{} `json:"asks"` + Bids interface{} `json:"bids"` + Time string `json:"time"` + Info interface{} `json:"info,omitempty"` +} + +type Level3OrderBook struct { + Asks [][3]string `json:"asks"` + Bids [][3]string `json:"bids"` + Info interface{} `json:"info,omitempty"` +} + +type BasicExchange struct { +} + +func (be *BasicExchange) GetPartOrderBook(number int) *OrderBook { + return nil +} + +func (be *BasicExchange) AddEventClientOidsToChannels(data map[string][]string) error { + return errors.New("unsupported rpc method: AddEventClientOidsToChannels") +} + +func (be *BasicExchange) AnyCall(method string, args json.RawMessage) (ret interface{}, err error) { + defer func() { + if r := recover(); r != nil { + log.Error("AnyCall panic", zap.Any("r", r)) + + ret = nil + err = errors.New("AnyCall panic") + } + }() + + switch method { + default: + return nil, errors.New("unsupported rpc method: " + method) + } +} diff --git a/pkg/exchanges/exchange_reg.go b/pkg/exchanges/exchange_reg.go new file mode 100644 index 0000000..db33efb --- /dev/null +++ b/pkg/exchanges/exchange_reg.go @@ -0,0 +1,32 @@ +package exchanges + +import ( + "fmt" +) + +var exchangeReg = map[string]Factory{} + +// Factory is used by output plugins to build an output instance +type Factory func() (Exchange, error) + +// RegisterType registers a new output type. +func RegisterType(name string, f Factory) { + if exchangeReg[name] != nil { + panic(fmt.Errorf("exchange type '%v' exists already", name)) + } + exchangeReg[name] = f +} + +// findFactory finds an output type its factory if available. +func findFactory(name string) Factory { + return exchangeReg[name] +} + +func Load(name string) (Exchange, error) { + factory := findFactory(name) + if factory == nil { + return nil, fmt.Errorf("exchange type %v undefined", name) + } + + return factory() +} diff --git a/pkg/exchanges/kucoin-v2/config.go b/pkg/exchanges/kucoin-v2/config.go new file mode 100644 index 0000000..49d05b4 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/config.go @@ -0,0 +1,10 @@ +package kucoin_v2 + +type Config struct { + URL string `mapstructure:"url" validate:"required"` + Type string `mapstructure:"type" validate:"required"` + //Verify bool `mapstructure:"verify"` + //VerifyDir string `mapstructure:"verify_dir" validate:"required_with=Verify"` +} + +var defaultConfig = Config{} diff --git a/pkg/exchanges/kucoin-v2/events/order_watcher.go b/pkg/exchanges/kucoin-v2/events/order_watcher.go new file mode 100644 index 0000000..6bfabc1 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/events/order_watcher.go @@ -0,0 +1,192 @@ +package events + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/consts" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/stream" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/redis" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/base" + "go.uber.org/zap" +) + +type OrderWatcher struct { + Messages chan *sdk.WebSocketDownstreamMessage + lock *sync.RWMutex + + orderIds map[string]map[string]bool //orderId => channel + clientOids map[string]map[string]bool //clientOid => channel +} + +func NewOrderWatcher() *OrderWatcher { + return &OrderWatcher{ + Messages: make(chan *sdk.WebSocketDownstreamMessage, consts.MaxMsgChanLen), + lock: &sync.RWMutex{}, + + orderIds: make(map[string]map[string]bool), + clientOids: make(map[string]map[string]bool), + } +} + +func (w *OrderWatcher) Run() { + log.Info("start running OrderWatcher") + + for msg := range w.Messages { + if !w.existEventOrderIds() { + continue + } + + l3Data, err := stream.NewStreamDataModel(msg) + if err != nil { + log.Panic("NewStreamDataModel err: " + err.Error()) + return + } + + publishedData := base.ToJsonString(msg) + switch l3Data.Type { + case stream.MessageReceivedType: + data := &stream.DataReceivedModel{} + if err := json.Unmarshal(l3Data.Data(), data); err != nil { + log.Panic("Unmarshal err", zap.Error(err)) + } + + w.migrationClientOidToOrderIds(data.ClientOid, data.OrderId) + + w.publish(data.OrderId, publishedData) + + case stream.MessageOpenType: + data := &stream.DataOpenModel{} + if err := json.Unmarshal(l3Data.Data(), data); err != nil { + log.Panic("Unmarshal err", zap.Error(err)) + } + + w.publish(data.OrderId, publishedData) + + case stream.MessageMatchType: + data := &stream.DataMatchModel{} + if err := json.Unmarshal(l3Data.Data(), data); err != nil { + log.Panic("Unmarshal err", zap.Error(err)) + } + + w.publish(data.MakerOrderId, publishedData) + w.publish(data.TakerOrderId, publishedData) + + case stream.MessageDoneType: + data := &stream.DataDoneModel{} + if err := json.Unmarshal(l3Data.Data(), data); err != nil { + log.Panic("Unmarshal err", zap.Error(err)) + } + + w.publish(data.OrderId, publishedData) + w.removeEventOrderId(data.OrderId) + + case stream.MessageUpdateType: + data := &stream.DataUpdateModel{} + if err := json.Unmarshal(l3Data.Data(), data); err != nil { + log.Panic("Unmarshal err", zap.Error(err)) + } + + w.publish(data.OrderId, publishedData) + + default: + log.Panic("error msg type: " + l3Data.Type) + } + } +} + +func (w *OrderWatcher) migrationClientOidToOrderIds(clientOid, orderId string) { + w.lock.RLock() + channelsMap, ok := w.clientOids[clientOid] + var channels []string + if ok { + channels = getMapKeys(channelsMap) + } + w.lock.RUnlock() + if ok { + w.removeEventClientOid(clientOid) + w.AddEventOrderIdsToChannels(map[string][]string{ + orderId: channels, + }) + } +} + +func getMapKeys(data map[string]bool) []string { + keys := make([]string, 0, len(data)) + for k := range data { + keys = append(keys, k) + } + + return keys +} + +func (w *OrderWatcher) publish(orderId string, message string) { + w.lock.RLock() + channelsMap, ok := w.orderIds[orderId] + channels := getMapKeys(channelsMap) + w.lock.RUnlock() + + if ok { + for _, channel := range channels { + redis.Publish("", channel, message) + } + } +} + +func (w *OrderWatcher) existEventOrderIds() bool { + w.lock.RLock() + defer w.lock.RUnlock() + if len(w.orderIds) == 0 && len(w.clientOids) == 0 { + return false + } + + return true +} + +func (w *OrderWatcher) AddEventOrderIdsToChannels(data map[string][]string) { + w.lock.Lock() + defer w.lock.Unlock() + + log.Info(fmt.Sprintf("AddEventOrderIdsToChannels, %#v", data)) + for orderId, channels := range data { + for _, channel := range channels { + if w.orderIds[orderId] == nil { + w.orderIds[orderId] = make(map[string]bool) + } + w.orderIds[orderId][channel] = true + } + } +} + +func (w *OrderWatcher) AddEventClientOidsToChannels(data map[string][]string) error { + w.lock.Lock() + log.Info(fmt.Sprintf("AddEventClientOidsToChannels, %#v", data)) + for clientOid, channels := range data { + for _, channel := range channels { + if w.clientOids[clientOid] == nil { + w.clientOids[clientOid] = make(map[string]bool) + } + w.clientOids[clientOid][channel] = true + } + } + w.lock.Unlock() + + return nil +} + +func (w *OrderWatcher) removeEventOrderId(orderId string) { + w.lock.Lock() + defer w.lock.Unlock() + + delete(w.orderIds, orderId) +} + +func (w *OrderWatcher) removeEventClientOid(clientOid string) { + w.lock.Lock() + defer w.lock.Unlock() + + delete(w.clientOids, clientOid) +} diff --git a/pkg/exchanges/kucoin-v2/exchange.go b/pkg/exchanges/kucoin-v2/exchange.go new file mode 100644 index 0000000..49d1903 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/exchange.go @@ -0,0 +1,143 @@ +package kucoin_v2 + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/events" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/orderbook" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/verify" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "go.uber.org/zap" +) + +type Exchange struct { + exchanges.BasicExchange + + apiService *sdk.Kucoin + ob *orderbook.Builder + ow *events.OrderWatcher + verify *verify.Verify +} + +func newExchange() *Exchange { + apiService := sdk.NewKucoin(defaultConfig.URL, defaultConfig.Type, "", "", "", false, 30*time.Second) + + build := orderbook.NewBuilder(apiService, cfg.AppConfig.Symbol) + var verifyObj *verify.Verify + //if defaultConfig.Verify { + // verifyObj = verify.NewVerify(build, 20, defaultConfig.VerifyDir, cfg.AppConfig.Symbol) + //} + ex := &Exchange{ + apiService: apiService, + ob: build, + ow: events.NewOrderWatcher(), + verify: verifyObj, + } + + //init ob + go ex.ob.ReloadOrderBook() + + go ex.ow.Run() + + //if defaultConfig.Verify { + // go ex.verify.Run() + //} + + go ex.websocket() + + return ex +} + +func (ex *Exchange) websocket() { + tk, err := ex.apiService.WebSocketPublicToken() + if err != nil { + log.Panic("WebSocketPublicToken err", zap.Error(err)) + } + + c := ex.apiService.NewWebSocketClient(tk) + + mc, ec, err := c.Connect() + if err != nil { + log.Panic("Connect panic: " + err.Error()) + } + topic := sdk.L3TopicPrefix(defaultConfig.Type) + cfg.AppConfig.Symbol + ch := sdk.NewSubscribeMessage(topic, false) + log.Info("subscribe: " + topic) + if err := c.Subscribe(ch); err != nil { + log.Panic("Subscribe panic: "+err.Error(), zap.Error(err)) + } + log.Info("Subscribe finish", zap.String("topic", topic)) + for { + select { + case err := <-ec: + c.Stop() // Stop subscribing the WebSocket feed + log.Panic("Connect panic: " + err.Error()) + + case msg := <-mc: + //log.Debug("receive message", zap.Any("data", msg)) + ex.dispatch(msg) + } + } +} + +func (ex *Exchange) dispatch(msgRawData *sdk.WebSocketDownstreamMessage) { + //log.Debug("raw message : " + base.ToJsonString(msgRawData)) + ex.ob.Messages <- msgRawData + ex.ow.Messages <- msgRawData + //if defaultConfig.Verify { + // ex.verify.Messages <- msgRawData + //} +} + +func (ex *Exchange) monitorChanLen() { + for { + const msgLenLimit = 50 + if len(ex.ob.Messages) > msgLenLimit { + log.Info(fmt.Sprintf( + "msgLenLimit: ex.ob.Messages: %d", + len(ex.ob.Messages), + )) + } + time.Sleep(time.Second) + } +} + +func (ex Exchange) GetPartOrderBook(number int) *exchanges.OrderBook { + return ex.ob.GetPartOrderBook(number) +} + +func (ex Exchange) AddEventClientOidsToChannels(data map[string][]string) error { + return ex.ow.AddEventClientOidsToChannels(data) +} + +func (ex Exchange) AnyCall(method string, args json.RawMessage) (ret interface{}, err error) { + defer func() { + if r := recover(); r != nil { + log.Error("AnyCall panic", zap.Any("r", r)) + + ret = nil + err = errors.New("AnyCall panic") + } + }() + + switch method { + case "GetL3PartOrderBook": + type AnyCallArgs struct { + Number int `json:"number"` + } + var anyCallArgs AnyCallArgs + if err := json.Unmarshal(args, &anyCallArgs); err != nil { + return nil, errors.New("unmarshal AnyCallArgs error: " + string(args)) + } + + return ex.ob.GetL3PartOrderBook(anyCallArgs.Number), nil + default: + return nil, errors.New("unsupported rpc method: " + method) + } +} diff --git a/pkg/exchanges/kucoin-v2/orderbook/depth.go b/pkg/exchanges/kucoin-v2/orderbook/depth.go new file mode 100644 index 0000000..182162b --- /dev/null +++ b/pkg/exchanges/kucoin-v2/orderbook/depth.go @@ -0,0 +1,23 @@ +package orderbook + +//[5]interface{}{"orderId", "price", "size", "ts"} +type DepthResponse struct { + Sequence uint64 `json:"sequence"` + //todo update + Asks [][4]interface{} `json:"asks"` //Sort price from low to high + Bids [][4]interface{} `json:"bids"` //Sort price from high to low +} + +func (b *Builder) GetAtomicFullOrderBook() (*DepthResponse, error) { + resp, err := b.apiService.AtomicFullOrderBook(b.symbol) + if err != nil { + return nil, err + } + + var fullOrderBook DepthResponse + if err := resp.ReadJson(&fullOrderBook); err != nil { + return nil, err + } + + return &fullOrderBook, nil +} diff --git a/pkg/exchanges/kucoin-v2/orderbook/orderbook.go b/pkg/exchanges/kucoin-v2/orderbook/orderbook.go new file mode 100644 index 0000000..bfb3fe3 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/orderbook/orderbook.go @@ -0,0 +1,378 @@ +package orderbook + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "sync" + "time" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/consts" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/stream" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/base" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/level3" + "github.com/shopspring/decimal" + "go.uber.org/zap" +) + +type Builder struct { + apiService *sdk.Kucoin + symbol string + lock *sync.RWMutex + Messages chan *sdk.WebSocketDownstreamMessage + + OrderBookTime uint64 + Sequence uint64 //Sequence || UpdateID + fullOrderBook *level3.OrderBook +} + +func NewBuilder(apiService *sdk.Kucoin, symbol string) *Builder { + return &Builder{ + apiService: apiService, + symbol: symbol, + lock: &sync.RWMutex{}, + Messages: make(chan *sdk.WebSocketDownstreamMessage, consts.MaxMsgChanLen), + } +} + +func (b *Builder) resetOrderBook() { + b.lock.Lock() + b.fullOrderBook = level3.NewOrderBook() + b.lock.Unlock() +} + +func (b *Builder) ReloadOrderBook() { + defer func() { + if r := recover(); r != nil { + log.Panic("ReloadOrderBook Panic", zap.Any("recover", r)) + } + }() + + log.Info("start running ReloadOrderBook, symbol: " + b.symbol) + b.resetOrderBook() + + b.playback() + + for msg := range b.Messages { + l3Data, err := stream.NewStreamDataModel(msg) + if err != nil { + log.Panic("NewStreamDataModel panic", zap.Error(err)) + } + b.updateFromStream(l3Data) + } +} + +func (b *Builder) playback() { + log.Info("prepare playback...") + + const tempMsgChanMaxLen = 10240 + tempMsgChan := make(chan *stream.DataModel, tempMsgChanMaxLen) + firstSequence := uint64(0) + var fullOrderBook *DepthResponse + + for msg := range b.Messages { + l3Data, err := stream.NewStreamDataModel(msg) + if err != nil { + log.Panic("NewStreamDataModel panic", zap.Error(err)) + } + tempMsgChan <- l3Data + + if firstSequence == 0 { + firstSequence = l3Data.Sequence + log.Info(fmt.Sprintf("firstSequence: %d", firstSequence)) + } + + if len(tempMsgChan) > 5 { + if fullOrderBook == nil { + log.Info("start GetAtomicFullOrderBook , symbol: " + b.symbol) + fullOrderBook, err = b.GetAtomicFullOrderBook() + if err != nil { + log.Panic("GetAtomicFullOrderBook panic", zap.Error(err)) + continue + } + log.Info(fmt.Sprintf("finish GetAtomicFullOrderBook, Sequence: %d", fullOrderBook.Sequence)) + } + + if len(tempMsgChan) > tempMsgChanMaxLen-5 { + log.Panic("playback failed, tempMsgChan is too long, retry...") + } + + if fullOrderBook != nil && fullOrderBook.Sequence < firstSequence { + log.Info(fmt.Sprintf("fullOrderBook Sequence is small, Sequence: %d", fullOrderBook.Sequence)) + fullOrderBook = nil + continue + } + + if fullOrderBook != nil && fullOrderBook.Sequence <= l3Data.Sequence { //string camp + log.Info("sequence match, start playback, tempMsgChan: " + strconv.Itoa(len(tempMsgChan))) + + b.lock.Lock() + b.AddDepthToOrderBook(fullOrderBook) + b.lock.Unlock() + + n := len(tempMsgChan) + for i := 0; i < n; i++ { + b.updateFromStream(<-tempMsgChan) + } + + log.Info("finish playback.") + break + } + } + } +} + +func newOrderWithElem(side string, elem [4]interface{}, info interface{}) (*level3.Order, error) { + timeInt, err := elem[3].(json.Number).Int64() + if err != nil { + return nil, err + } + return level3.NewOrder(elem[0].(string), side, elem[1].(string), elem[2].(string), uint64(timeInt), info) +} + +func (b *Builder) AddDepthToOrderBook(depth *DepthResponse) { + b.Sequence = depth.Sequence + b.OrderBookTime = uint64(time.Now().UnixNano()) + b.formatDepthToOrderBook(depth, b.fullOrderBook) +} + +func (b *Builder) formatDepthToOrderBook(depth *DepthResponse, fullOrderBook *level3.OrderBook) { + fullOrderBook.Sequence = depth.Sequence + + for _, elem := range depth.Asks { + order, err := newOrderWithElem(base.AskSide, elem, nil) + if err != nil { + log.Panic("NewOrder panic", zap.Error(err)) + } + + if err := fullOrderBook.AddOrder(order); err != nil { + log.Panic("AddOrder panic", zap.Error(err)) + } + } + + for _, elem := range depth.Bids { + order, err := newOrderWithElem(base.BidSide, elem, nil) + if err != nil { + log.Panic("NewOrder panic", zap.Error(err)) + } + + if err := fullOrderBook.AddOrder(order); err != nil { + log.Panic("AddOrder panic", zap.Error(err)) + } + } + + return +} + +func (b *Builder) updateFromStream(msg *stream.DataModel) { + b.lock.Lock() + defer b.lock.Unlock() + + skip, err := b.updateSequence(msg) + if err != nil { + log.Panic("updateSequence panic", zap.Error(err)) + } + + if !skip { + b.updateOrderBook(msg) + } +} + +func (b *Builder) updateSequence(msg *stream.DataModel) (bool, error) { + if b.Sequence+1 > msg.Sequence { + return true, nil + } + + if b.Sequence+1 != msg.Sequence { + return false, errors.New(fmt.Sprintf( + "currentSequence: %d, msgSequence: %d, the sequence is not continuous, current chanLen: %d", + b.Sequence, + msg.Sequence, + len(b.Messages), + )) + } + + b.Sequence = msg.Sequence + b.fullOrderBook.Sequence = msg.Sequence + return false, nil +} + +func (b *Builder) updateOrderBook(msg *stream.DataModel) { + //[3]string{"orderId", "price", "size"} + //var item = [3]string{msg.OrderId, msg.Price, msg.Size} + + switch msg.Type { + case stream.MessageReceivedType: + + case stream.MessageOpenType: + data := &stream.DataOpenModel{} + if err := json.Unmarshal(msg.Data(), data); err != nil { + log.Panic("Unmarshal panic", zap.Error(err)) + } + + if data.Price == "" || data.Size == "0" || data.Price == "0" || data.Size == "" { + return + } + + side := "" + switch data.Side { + case stream.SellSide: + side = base.AskSide + case stream.BuySide: + side = base.BidSide + default: + panic("error side: " + data.Side) + } + + order, err := level3.NewOrder(data.OrderId, side, data.Price, data.Size, data.Time, nil) + if err != nil { + log.Panic("NewOrder panic: "+err.Error(), zap.String("Data", string(msg.Data()))) + } + if err := b.fullOrderBook.AddOrder(order); err != nil { + log.Panic("AddOrder panic: " + err.Error()) + } + b.OrderBookTime = data.Time + + case stream.MessageDoneType: + data := &stream.DataDoneModel{} + if err := json.Unmarshal(msg.Data(), data); err != nil { + log.Panic("Unmarshal panic", zap.Error(err)) + } + + if err := b.fullOrderBook.RemoveByOrderId(data.OrderId); err != nil { + log.Panic("RemoveByOrderId panic: " + err.Error()) + } + b.OrderBookTime = data.Time + + case stream.MessageMatchType: + data := &stream.DataMatchModel{} + if err := json.Unmarshal(msg.Data(), data); err != nil { + log.Panic("Unmarshal panic: " + err.Error()) + } + size, err := decimal.NewFromString(data.RemainSize) + if err != nil { + log.Panic("MatchOrder panic: " + err.Error()) + } + if err := b.fullOrderBook.ChangeOrder(data.MakerOrderId, size); err != nil { + log.Panic("MatchOrder panic: " + err.Error()) + } + b.OrderBookTime = data.Time + + case stream.MessageUpdateType: + data := &stream.DataUpdateModel{} + if err := json.Unmarshal(msg.Data(), data); err != nil { + log.Panic("Unmarshal panic: " + err.Error()) + } + + size, err := decimal.NewFromString(data.Size) + if err != nil { + log.Panic("UpdateOrder panic: " + err.Error()) + } + if err := b.fullOrderBook.ChangeOrder(data.OrderId, size); err != nil { + log.Panic("UpdateOrder panic: " + err.Error()) + } + b.OrderBookTime = data.Time + + default: + log.Panic("error msg type: " + msg.Type) + } + + ask, bid := b.fullOrderBook.GetOrderBookTickerOrder() + if ask != nil && bid != nil && bid.Price.Cmp(ask.Price) >= 0 { + log.Panic("order book cross", zap.String("asks", ask.Price.String()), zap.String("bids", bid.Price.String())) + } +} + +//[3]string{"orderId", "price", "size"} +type FullOrderBook struct { + Sequence uint64 `json:"sequence"` + Asks [][3]string `json:"asks"` + Bids [][3]string `json:"bids"` +} + +func (b *Builder) DepthResponse2FullOrderBook(atomicFullOrderBook *DepthResponse) (*FullOrderBook, error) { + orderBook := level3.NewOrderBook() + b.formatDepthToOrderBook(atomicFullOrderBook, orderBook) + data, err := json.Marshal(orderBook) + if err != nil { + return nil, err + } + + ret := &FullOrderBook{} + if err := json.Unmarshal(data, ret); err != nil { + return nil, err + } + + return ret, nil +} + +func (b *Builder) Snapshot() (*FullOrderBook, error) { + data, err := b.SnapshotBytes() + if err != nil { + return nil, err + } + + ret := &FullOrderBook{} + if err := json.Unmarshal(data, ret); err != nil { + return nil, err + } + + return ret, nil +} + +func (b *Builder) SnapshotBytes() ([]byte, error) { + b.lock.RLock() + data, err := json.Marshal(b.fullOrderBook) + b.lock.RUnlock() + if err != nil { + return nil, err + } + + return data, nil +} + +func (b *Builder) GetPartOrderBook(number int) *exchanges.OrderBook { + defer func() { + if r := recover(); r != nil { + log.Error("GetPartOrderBook panic", zap.Any("r", r)) + } + }() + + b.lock.RLock() + defer b.lock.RUnlock() + + data := &exchanges.OrderBook{ + Asks: b.fullOrderBook.GetPartOrderBookBySide(base.AskSide, number), + Bids: b.fullOrderBook.GetPartOrderBookBySide(base.BidSide, number), + Info: map[string]interface{}{ + "time": b.OrderBookTime, + }, + } + + return data +} + +func (b *Builder) GetL3PartOrderBook(number int) *exchanges.Level3OrderBook { + defer func() { + if r := recover(); r != nil { + log.Error("GetPartOrderBook panic", zap.Any("r", r)) + } + }() + + b.lock.RLock() + defer b.lock.RUnlock() + + data := &exchanges.Level3OrderBook{ + Asks: b.fullOrderBook.GetL3PartOrderBookBySide(base.AskSide, number), + Bids: b.fullOrderBook.GetL3PartOrderBookBySide(base.BidSide, number), + Info: map[string]interface{}{ + "time": b.OrderBookTime, + }, + } + + return data +} diff --git a/pkg/exchanges/kucoin-v2/sdk/cmd/main.go b/pkg/exchanges/kucoin-v2/sdk/cmd/main.go new file mode 100644 index 0000000..396d804 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/cmd/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "time" + + kucoin "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk" +) + +func main() { + apiService := kucoin.NewKucoin("http://openapi-v2.kucoin.net", "spot", "", "", "", true, 30*time.Second) + + tk, err := apiService.WebSocketPublicToken() + tk, err = apiService.WebSocketPrivateToken() + if err != nil { + panic(err) + } + + c := apiService.NewWebSocketClient(tk) + + mc, ec, err := c.Connect() + if err != nil { + panic(err) + } + + symbol := "KCS-BTC" + ch := kucoin.NewSubscribeMessage("/spotMarket/level2Depth5:"+symbol, false) + ch = kucoin.NewSubscribeMessage("/spotMarket/level2Depth50:"+symbol, false) + //ch = kucoin.NewSubscribeMessage("/spotMarket/level3:"+symbol, false) + ch = kucoin.NewSubscribeMessage("/spotMarket/tradeOrders", true) + + //ch = kucoin.NewSubscribeMessage("/spotMarket/advancedOrders", true) + + //ch = kucoin.NewSubscribeMessage("/contractMarket/level2depth5:XBTUSDM", false) + //ch = kucoin.NewSubscribeMessage("/contractMarket/level2depth50:XBTUSDM", false) + //ch = kucoin.NewSubscribeMessage("/contractMarket/level3v2:XBTUSDM", false) + //ch = kucoin.NewSubscribeMessage("/contractMarket/tradeOrders", true) + + if err := c.Subscribe(ch); err != nil { + panic(err) + } + + for { + select { + case err := <-ec: + c.Stop() // Stop subscribing the WebSocket feed + panic(err) + + case <-mc: + //log.Println(base.ToJsonString(msg)) + } + } +} diff --git a/pkg/exchanges/kucoin-v2/sdk/helper.go b/pkg/exchanges/kucoin-v2/sdk/helper.go new file mode 100644 index 0000000..90cfd31 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/helper.go @@ -0,0 +1,20 @@ +package sdk + +import ( + "encoding/json" + "strconv" +) + +// IntToString converts int64 to string. +func IntToString(i int64) string { + return strconv.FormatInt(i, 10) +} + +// ToJsonString converts any value to JSON string. +func ToJsonString(v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + return "" + } + return string(b) +} diff --git a/pkg/exchanges/kucoin-v2/sdk/http_client/request.go b/pkg/exchanges/kucoin-v2/sdk/http_client/request.go new file mode 100644 index 0000000..3ce3821 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/http_client/request.go @@ -0,0 +1,95 @@ +package http_client + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "net/http" + "net/url" + "strings" + "time" +) + +type Client struct { + baseUrl string + skipVerifyTls bool + timeout time.Duration + signer *KcSigner +} + +func NewClient(baseUrl string, apiKey string, apiSecret string, apiPassphrase string, skipVerifyTls bool, timeout time.Duration) *Client { + var signer *KcSigner + if apiKey != "" { + signer = NewKcSigner(apiKey, apiSecret, apiPassphrase) + } + return &Client{ + baseUrl: strings.TrimRight(baseUrl, "/"), + skipVerifyTls: skipVerifyTls, + timeout: timeout, + signer: signer, + } +} + +func (c *Client) Request(method string, uri string, params map[string]string) (*Response, error) { + query := make(url.Values) + var body []byte + switch method { + case http.MethodGet, http.MethodDelete: + for key, value := range params { + query.Add(key, value) + } + default: + if params == nil { + break + } + b, err := json.Marshal(params) + if err != nil { + return nil, err + } + body = b + } + + if len(query) > 0 { + if strings.Contains(uri, "?") { + uri += "&" + query.Encode() + } else { + uri += "?" + query.Encode() + } + } + + req, err := http.NewRequest(method, c.baseUrl+uri, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "KuMex-Go-SDK/1.0") + + //sign + if c.signer != nil { + var b bytes.Buffer + b.WriteString(method) + b.WriteString(uri) + b.Write(body) + h := c.signer.Headers(b.String()) + //log.Println(base.ToJsonString(h)) + for k, v := range h { + //log.Printf("%s : %s", k, v) + req.Header.Set(k, v) + } + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: c.skipVerifyTls}, + }, + Timeout: c.timeout, + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + return NewResponse(req, resp), nil +} diff --git a/pkg/exchanges/kucoin-v2/sdk/http_client/response.go b/pkg/exchanges/kucoin-v2/sdk/http_client/response.go new file mode 100644 index 0000000..4e01035 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/http_client/response.go @@ -0,0 +1,86 @@ +package http_client + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +type Response struct { + request *http.Request + response *http.Response + body []byte +} + +type ResponseBody struct { + Code string `json:"code"` + RawData json.RawMessage `json:"data"` // delay parsing + Message string `json:"msg"` +} + +func NewResponse(request *http.Request, response *http.Response) *Response { + resp := &Response{request, response, nil} + resp.body, _ = resp.read() + + return resp +} + +func (resp *Response) success() bool { + if resp.response.StatusCode != http.StatusOK { + return false + } + + return true +} + +func (resp *Response) read() ([]byte, error) { + defer resp.response.Body.Close() + return ioutil.ReadAll(resp.response.Body) +} + +func (resp *Response) StatusCode() int { + return resp.response.StatusCode +} + +func (resp *Response) Body() []byte { + return resp.body +} + +func (resp *Response) ReadJson(v interface{}) error { + if !resp.success() { + return resp.error(fmt.Sprintf("http code is not %d", http.StatusOK)) + } + + var responseBody ResponseBody + if err := json.Unmarshal(resp.body, &responseBody); err != nil { + return resp.error("ResponseBody json unmarshal failure") + } + + const ApiSuccess = "200000" + + if responseBody.Code != ApiSuccess { + return resp.error("api code is not " + ApiSuccess) + } + + //log.Debug("http ReadJson, url: " + resp.request.URL.String() + ", response RawData: " + string(responseBody.RawData)) + decoder := json.NewDecoder(bytes.NewReader(responseBody.RawData)) + decoder.UseNumber() + if err := decoder.Decode(v); err != nil { + return resp.error(fmt.Sprintf("responseBody.RawData json unmarshal failure: %v", err)) + } + + return nil +} + +func (resp *Response) error(error string) error { + return fmt.Errorf( + "http request failure, error: %s\nstatus code: %d, %s %s, body:\n%s", + error, + resp.response.StatusCode, + resp.request.Method, + resp.request.URL.String(), + string(resp.body), + ) +} diff --git a/pkg/exchanges/kucoin-v2/sdk/http_client/signer.go b/pkg/exchanges/kucoin-v2/sdk/http_client/signer.go new file mode 100644 index 0000000..92634c5 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/http_client/signer.go @@ -0,0 +1,64 @@ +package http_client + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "strconv" + "time" +) + +// Signer interface contains Sign() method. +type Signer interface { + Sign(plain []byte) []byte +} + +// Sha256Signer is the sha256 Signer. +type Sha256Signer struct { + key []byte +} + +// Sign makes a signature by sha256. +func (ss *Sha256Signer) Sign(plain []byte) []byte { + hm := hmac.New(sha256.New, ss.key) + hm.Write(plain) + return hm.Sum(nil) +} + +// KcSigner is the implement of Signer for KuCoin. +type KcSigner struct { + Sha256Signer + apiKey string + apiSecret string + apiPassPhrase string +} + +// Sign makes a signature by sha256 with `apiKey` `apiSecret` `apiPassPhrase`. +func (ks *KcSigner) Sign(plain []byte) []byte { + s := ks.Sha256Signer.Sign(plain) + return []byte(base64.StdEncoding.EncodeToString(s)) +} + +// Headers returns a map of signature header. +func (ks *KcSigner) Headers(plain string) map[string]string { + t := strconv.FormatInt(time.Now().UnixNano()/1000000, 10) + p := []byte(t + plain) + s := string(ks.Sign(p)) + return map[string]string{ + "KC-API-KEY": ks.apiKey, + "KC-API-PASSPHRASE": ks.apiPassPhrase, + "KC-API-TIMESTAMP": t, + "KC-API-SIGN": s, + } +} + +// NewKcSigner creates a instance of KcSigner. +func NewKcSigner(key, secret, passPhrase string) *KcSigner { + ks := &KcSigner{ + apiKey: key, + apiSecret: secret, + apiPassPhrase: passPhrase, + } + ks.key = []byte(secret) + return ks +} diff --git a/pkg/exchanges/kucoin-v2/sdk/kucoin.go b/pkg/exchanges/kucoin-v2/sdk/kucoin.go new file mode 100644 index 0000000..ef1d65b --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/kucoin.go @@ -0,0 +1,64 @@ +package sdk + +import ( + "net/http" + "time" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk/http_client" +) + +const ( + urlSpotSnapshot = "/api/v2/market/orderbook/level3" + + urlFuturesSnapshot = "/api/v2/level3/snapshot" + + topicSpotL3Prefix = "/spotMarket/level3:" + + topicFutureL3Prefix = "/contractMarket/level3v2:" +) + +type Kucoin struct { + httpClient *http_client.Client + + typ string +} + +func NewKucoin(baseUrl, typ, apiKey, apiSecret, apiPassphrase string, skipVerifyTls bool, timeout time.Duration) *Kucoin { + client := http_client.NewClient(baseUrl, apiKey, apiSecret, apiPassphrase, skipVerifyTls, timeout) + kucoin := &Kucoin{ + httpClient: client, + + typ: typ, + } + return kucoin +} + +func L3TopicPrefix(typ string) string { + var ret string + switch typ { + case "spot": + ret = topicSpotL3Prefix + case "future": + ret = topicFutureL3Prefix + default: + log.Panic("market type error, must be spot or future") + } + + return ret +} + +func (kucoin *Kucoin) AtomicFullOrderBook(symbol string) (*http_client.Response, error) { + var url string + switch kucoin.typ { + case "spot": + url = urlSpotSnapshot + case "future": + url = urlFuturesSnapshot + default: + log.Panic("market type error, must be spot or future") + } + + return kucoin.httpClient.Request(http.MethodGet, url, map[string]string{"symbol": symbol}) +} diff --git a/pkg/exchanges/kucoin-v2/sdk/websocket.go b/pkg/exchanges/kucoin-v2/sdk/websocket.go new file mode 100644 index 0000000..3b1ad5b --- /dev/null +++ b/pkg/exchanges/kucoin-v2/sdk/websocket.go @@ -0,0 +1,365 @@ +package sdk + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/url" + "sync" + "time" + + "github.com/gorilla/websocket" + "github.com/pkg/errors" +) + +// A WebSocketTokenModel contains a token and some servers for WebSocket feed. +type WebSocketTokenModel struct { + Token string `json:"token"` + Servers WebSocketServersModel `json:"instanceServers"` +} + +// A WebSocketServerModel contains some servers for WebSocket feed. +type WebSocketServerModel struct { + PingInterval int64 `json:"pingInterval"` + Endpoint string `json:"endpoint"` + Protocol string `json:"protocol"` + Encrypt bool `json:"encrypt"` + PingTimeout int64 `json:"pingTimeout"` +} + +// A WebSocketServersModel is the set of *WebSocketServerModel. +type WebSocketServersModel []*WebSocketServerModel + +// RandomServer returns a server randomly. +func (s WebSocketServersModel) RandomServer() (*WebSocketServerModel, error) { + //log.Println(helper.ToJsonString(s)) + l := len(s) + if l == 0 { + return nil, errors.New("No available server ") + } + return s[rand.Intn(l)], nil +} + +// WebSocketPrivateToken returns the token for private channel. +func (kucoin *Kucoin) WebSocketPrivateToken() (*WebSocketTokenModel, error) { + resp, err := kucoin.httpClient.Request(http.MethodPost, "/api/v1/bullet-private", nil) + if err != nil { + return nil, err + } + + var token WebSocketTokenModel + if err := resp.ReadJson(&token); err != nil { + return nil, err + } + + return &token, nil +} + +// WebSocketPublicToken returns the token for public channel. +func (kucoin *Kucoin) WebSocketPublicToken() (*WebSocketTokenModel, error) { + resp, err := kucoin.httpClient.Request(http.MethodPost, "/api/v1/bullet-public", nil) + if err != nil { + return nil, err + } + + var token WebSocketTokenModel + if err := resp.ReadJson(&token); err != nil { + return nil, err + } + + return &token, nil +} + +// All message types of WebSocket. +const ( + WelcomeMessage = "welcome" + PingMessage = "ping" + PongMessage = "pong" + SubscribeMessage = "subscribe" + AckMessage = "ack" + UnsubscribeMessage = "unsubscribe" + ErrorMessage = "error" + Message = "message" +) + +// A WebSocketMessage represents a message between the WebSocket client and server. +type WebSocketMessage struct { + Id string `json:"id"` + Type string `json:"type"` +} + +// A WebSocketSubscribeMessage represents a message to subscribe the public/private channel. +type WebSocketSubscribeMessage struct { + *WebSocketMessage + Topic string `json:"topic"` + PrivateChannel bool `json:"privateChannel"` + Response bool `json:"response"` +} + +// NewPingMessage creates a ping message instance. +func NewPingMessage() *WebSocketMessage { + return &WebSocketMessage{ + Id: IntToString(time.Now().UnixNano()), + Type: PingMessage, + } +} + +// NewSubscribeMessage creates a subscribe message instance. +func NewSubscribeMessage(topic string, privateChannel bool) *WebSocketSubscribeMessage { + return &WebSocketSubscribeMessage{ + WebSocketMessage: &WebSocketMessage{ + Id: IntToString(time.Now().UnixNano()), + Type: SubscribeMessage, + }, + Topic: topic, + PrivateChannel: privateChannel, + Response: true, + } +} + +// A WebSocketUnsubscribeMessage represents a message to unsubscribe the public/private channel. +type WebSocketUnsubscribeMessage WebSocketSubscribeMessage + +// NewUnsubscribeMessage creates a unsubscribe message instance. +func NewUnsubscribeMessage(topic string, privateChannel bool) *WebSocketUnsubscribeMessage { + return &WebSocketUnsubscribeMessage{ + WebSocketMessage: &WebSocketMessage{ + Id: IntToString(time.Now().UnixNano()), + Type: UnsubscribeMessage, + }, + Topic: topic, + PrivateChannel: privateChannel, + Response: true, + } +} + +// A WebSocketDownstreamMessage represents a message from the WebSocket server to client. +type WebSocketDownstreamMessage struct { + *WebSocketMessage + //Sn string `json:"sn"` + Topic string `json:"topic"` + Subject string `json:"subject"` + RawData json.RawMessage `json:"data"` +} + +// ReadData read the data in channel. +func (m *WebSocketDownstreamMessage) ReadData(v interface{}) error { + return json.Unmarshal(m.RawData, v) +} + +// A WebSocketClient represents a connection to WebSocket server. +type WebSocketClient struct { + // Wait all goroutines quit + wg *sync.WaitGroup + // Stop subscribing channel + done chan struct{} + // Pong channel to check pong message + pongs chan string + // ACK channel to check pong message + acks chan string + // Error channel + errors chan error + // Downstream message channel + messages chan *WebSocketDownstreamMessage + conn *websocket.Conn + token *WebSocketTokenModel + server *WebSocketServerModel + enableHeartbeat bool + skipVerifyTls bool +} + +// NewWebSocketClient creates an instance of WebSocketClient. +func (kucoin *Kucoin) NewWebSocketClient(token *WebSocketTokenModel) *WebSocketClient { + wc := &WebSocketClient{ + wg: &sync.WaitGroup{}, + done: make(chan struct{}), + errors: make(chan error, 1), + pongs: make(chan string, 1), + acks: make(chan string, 1), + token: token, + messages: make(chan *WebSocketDownstreamMessage, 2048), + skipVerifyTls: true, + } + return wc +} + +// Connect connects the WebSocket server. +func (wc *WebSocketClient) Connect() (<-chan *WebSocketDownstreamMessage, <-chan error, error) { + // Find out a server + s, err := wc.token.Servers.RandomServer() + if err != nil { + return wc.messages, wc.errors, err + } + //log.Println(ToJsonString(s)) + wc.server = s + + // Concat ws url + q := url.Values{} + q.Add("connectId", IntToString(time.Now().UnixNano())) + q.Add("token", wc.token.Token) + u := fmt.Sprintf("%s?%s", s.Endpoint, q.Encode()) + + //websocket.DefaultDialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: wc.skipVerifyTls} + + // Connect ws server + websocket.DefaultDialer.ReadBufferSize = 2048000 //2000 kb + wc.conn, _, err = websocket.DefaultDialer.Dial(u, nil) + if err != nil { + return wc.messages, wc.errors, err + } + + // Must read the first welcome message + for { + m := &WebSocketDownstreamMessage{} + if err := wc.conn.ReadJSON(m); err != nil { + return wc.messages, wc.errors, err + } + if m.Type == ErrorMessage { + return wc.messages, wc.errors, errors.Errorf("Error message: %s", ToJsonString(m)) + } + if m.Type == WelcomeMessage { + break + } + } + + wc.wg.Add(2) + go wc.read() + go wc.keepHeartbeat() + + return wc.messages, wc.errors, nil +} + +func (wc *WebSocketClient) read() { + defer func() { + close(wc.pongs) + close(wc.messages) + wc.wg.Done() + }() + + for { + select { + case <-wc.done: + return + default: + _, message, err := wc.conn.ReadMessage() + if err != nil { + wc.errors <- err + return + } + //fmt.Println(string(message)) + + m := &WebSocketDownstreamMessage{} + //log.Println("before ReadJSON") + if err := json.Unmarshal(message, m); err != nil { + wc.errors <- err + return + } + //log.Println(helper.ToJsonString(m)) + //log.Println("after ReadJSON") + + switch m.Type { + case WelcomeMessage: + case PongMessage: + if wc.enableHeartbeat { + wc.pongs <- m.Id + } + case AckMessage: + // log.Printf("Subscribed: %s==%s? %s", channel.Id, m.Id, channel.Topic) + wc.acks <- m.Id + case ErrorMessage: + wc.errors <- errors.Errorf("Error message: %s", ToJsonString(m)) + return + case Message: + wc.messages <- m + default: + wc.errors <- errors.Errorf("Unknown message type: %s", m.Type) + } + } + } +} + +func (wc *WebSocketClient) keepHeartbeat() { + wc.enableHeartbeat = true + // New ticker to send ping message + pt := time.NewTicker(time.Duration(wc.server.PingInterval)*time.Millisecond - time.Millisecond*200) + defer wc.wg.Done() + defer pt.Stop() + + for { + select { + case <-wc.done: + return + case <-pt.C: + p := NewPingMessage() + m := ToJsonString(p) + if err := wc.conn.WriteMessage(websocket.TextMessage, []byte(m)); err != nil { + wc.errors <- err + return + } + + // log.Printf("Ping: %s", ToJsonString(p)) + // Waiting (with timeout) for the server to response pong message + // If timeout, close this connection + select { + case pid := <-wc.pongs: + if pid != p.Id { + wc.errors <- errors.Errorf("Invalid pong id %s, expect %s", pid, p.Id) + return + } + case <-time.After(time.Duration(wc.server.PingTimeout) * time.Millisecond): + wc.errors <- errors.Errorf("Wait pong message timeout in %d ms", wc.server.PingTimeout) + return + } + } + } +} + +// Subscribe subscribes the specified channel. +func (wc *WebSocketClient) Subscribe(channels ...*WebSocketSubscribeMessage) error { + for _, c := range channels { + m := ToJsonString(c) + if err := wc.conn.WriteMessage(websocket.TextMessage, []byte(m)); err != nil { + return err + } + //log.Printf("Subscribing: %s, %s", c.Id, c.Topic) + select { + case id := <-wc.acks: + //log.Printf("ack: %s=>%s", id, c.Id) + if id != c.Id { + return errors.Errorf("Invalid ack id %s, expect %s", id, c.Id) + } + case <-time.After(time.Second * 5): + return errors.Errorf("Wait ack message timeout in %d s", 5) + } + } + return nil +} + +// Unsubscribe unsubscribes the specified channel. +func (wc *WebSocketClient) Unsubscribe(channels ...*WebSocketUnsubscribeMessage) error { + for _, c := range channels { + m := ToJsonString(c) + if err := wc.conn.WriteMessage(websocket.TextMessage, []byte(m)); err != nil { + return err + } + //log.Printf("Unsubscribing: %s, %s", c.Id, c.Topic) + select { + case id := <-wc.acks: + //log.Printf("ack: %s=>%s", id, c.Id) + if id != c.Id { + return errors.Errorf("Invalid ack id %s, expect %s", id, c.Id) + } + case <-time.After(time.Second * 5): + return errors.Errorf("Wait ack message timeout in %d s", 5) + } + } + return nil +} + +// Stop stops subscribing the specified channel, all goroutines quit. +func (wc *WebSocketClient) Stop() { + close(wc.done) + _ = wc.conn.Close() + wc.wg.Wait() +} diff --git a/pkg/exchanges/kucoin-v2/setup.go b/pkg/exchanges/kucoin-v2/setup.go new file mode 100644 index 0000000..93842f4 --- /dev/null +++ b/pkg/exchanges/kucoin-v2/setup.go @@ -0,0 +1,35 @@ +package kucoin_v2 + +import ( + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/helper" + "go.uber.org/zap" +) + +const ( + KucoinName = "kucoin_v2" + KucoinFutureName = "kumex_v2" + PoloniexSwapName = "poloniex_swap_v2" +) + +func init() { + exchanges.RegisterType(KucoinName, setup(KucoinName)) + exchanges.RegisterType(KucoinFutureName, setup(KucoinFutureName)) + exchanges.RegisterType(PoloniexSwapName, setup(PoloniexSwapName)) +} + +func setup(name string) exchanges.Factory { + return func() (exchanges.Exchange, error) { + // parse config + if err := cfg.AppConfig.Unpack(&defaultConfig); err != nil { + log.Panic("Unpack panic", zap.Error(err)) + } + log.Debug("market config for " + name + ":" + helper.ToJsonString(defaultConfig)) + + ex := newExchange() + + return ex, nil + } +} diff --git a/pkg/exchanges/kucoin-v2/stream/data_model.go b/pkg/exchanges/kucoin-v2/stream/data_model.go new file mode 100644 index 0000000..932d60d --- /dev/null +++ b/pkg/exchanges/kucoin-v2/stream/data_model.go @@ -0,0 +1,88 @@ +package stream + +import ( + "encoding/json" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk" +) + +type SequenceModel struct { + Sequence uint64 `json:"sequence"` +} + +//Level 3 websocket stream +type DataModel struct { + Type string `json:"type"` + + Sequence uint64 `json:"sequence"` + + //Symbol string `json:"symbol"` + + rawData json.RawMessage +} + +func NewStreamDataModel(msgData *sdk.WebSocketDownstreamMessage) (*DataModel, error) { + data := &SequenceModel{} + if err := json.Unmarshal(msgData.RawData, data); err != nil { + return nil, err + } + + return &DataModel{ + Type: msgData.Subject, + Sequence: data.Sequence, + rawData: msgData.RawData, + }, nil +} + +func (l3Data *DataModel) Data() json.RawMessage { + return l3Data.rawData +} + +const ( + BuySide = "buy" + SellSide = "sell" + + MessageReceivedType = "received" + MessageOpenType = "open" + MessageDoneType = "done" + MessageMatchType = "match" + MessageUpdateType = "update" +) + +type DataReceivedModel struct { + OrderId string `json:"orderId"` + ClientOid string `json:"clientOid"` + Time uint64 `json:"ts"` +} + +type DataOpenModel struct { + Side string `json:"side"` + Size string `json:"size"` + Price string `json:"price"` + OrderId string `json:"orderId"` + OrderTime uint64 `json:"orderTime"` + Time uint64 `json:"ts"` +} + +type DataUpdateModel struct { + OrderId string `json:"orderId"` + Size string `json:"size"` + Time uint64 `json:"ts"` +} + +type DataMatchModel struct { + Side string `json:"side"` + Price string `json:"price"` + Size string `json:"size"` + RemainSize string `json:"remainSize"` + TakerOrderId string `json:"takerOrderId"` + MakerOrderId string `json:"makerOrderId"` + TradeId string `json:"tradeId"` + Time uint64 `json:"ts"` +} + +type DataDoneModel struct { + OrderId string `json:"orderId"` + Reason string `json:"reason"` + Time uint64 `json:"ts"` +} diff --git a/pkg/exchanges/kucoin-v2/verify/verify.go b/pkg/exchanges/kucoin-v2/verify/verify.go new file mode 100644 index 0000000..65f4dcc --- /dev/null +++ b/pkg/exchanges/kucoin-v2/verify/verify.go @@ -0,0 +1,290 @@ +package verify + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/orderbook" + "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2/sdk" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "github.com/shopspring/decimal" + "go.uber.org/zap" +) + +type Verify struct { + level3Builder *orderbook.Builder + Messages chan *sdk.WebSocketDownstreamMessage //方便记录两次快照之间的数据流，方便在快照校验失败后，重复快照 + verifyFrequency time.Duration //校验频率 + + verifyLogDirectory string + update *os.File + + uniqStr string + + nextVerifyTime time.Time + snapshot map[uint64]*orderbook.FullOrderBook + atomicFullOrderBook *orderbook.DepthResponse +} + +func NewVerify(level3Builder *orderbook.Builder, verifyFrequency uint64, verifyLogDirectory string, uniqStr string) *Verify { + verifyLogDirectory = strings.TrimRight(verifyLogDirectory, "/") + v := &Verify{ + level3Builder: level3Builder, + Messages: make(chan *sdk.WebSocketDownstreamMessage, 1024), + verifyFrequency: time.Duration(verifyFrequency) * time.Second, + verifyLogDirectory: strings.TrimRight(verifyLogDirectory, "/") + "/", + uniqStr: uniqStr, + } + + v.resetVerify(true) + + //第一次校验时间在启动后2分钟开始开启 + v.nextVerifyTime = time.Now().Add(10 * time.Second) + + return v +} + +//重置数据 +func (v *Verify) resetVerify(setNextVerifyTime bool) { + if setNextVerifyTime { + v.nextVerifyTime = time.Now().Add(v.verifyFrequency) + } else { //防止请求过多 + v.nextVerifyTime = time.Now().Add(time.Minute) + } + v.snapshot = map[uint64]*orderbook.FullOrderBook{} + v.atomicFullOrderBook = nil +} + +//func (v *Verify) Log() { +// for msg := range v.Messages { +// log.Debug("verify websocket", zap.Any("type", msg.Subject), zap.Any("Data", string(msg.RawData))) +// } +//} + +func (v *Verify) Run() { + defer v.update.Close() + + v.checkLogDirExists() + + log.Info("start running Verify, verifyLogDirectory: "+v.verifyLogDirectory, zap.Any("verifyFrequency", v.verifyFrequency)) + + const snapshotMaxLen = 200 + const atomicStartLen = 10 + + for msg := range v.Messages { + v.writeWsMsg(msg) + if time.Now().After(v.nextVerifyTime) { + //获取快照 + snapshot, err := v.level3Builder.Snapshot() + if err != nil { + panic(err) + } + + if snapshot.Sequence == 0 { + continue + } + + if _, ok := v.snapshot[snapshot.Sequence]; ok { + //跳过已经存在的快照 + continue + } + v.snapshot[snapshot.Sequence] = snapshot + + //log.Error("准备校验%d", len(v.snapshot)) + + //获取全量 + if len(v.snapshot) == atomicStartLen { + //log.Error("获取全量...") + + go func() { + var err error + log.Info("verify 获取全量") + + v.atomicFullOrderBook, err = v.level3Builder.GetAtomicFullOrderBook() + if err != nil { + log.Warn("获取全量失败放弃本次校验", zap.Error(err)) + v.resetVerify(false) + } + }() + } + + if v.atomicFullOrderBook != nil { + //开始校验 !!! + atomicFullOrderBookSequence := v.atomicFullOrderBook.Sequence + if snapshot, ok := v.snapshot[atomicFullOrderBookSequence]; ok { + log.Info("verify 开始校验", zap.Any("Sequence", snapshot.Sequence)) + + //写入全量快照 + if v.verifyLogDirectory != "" { + data, _ := json.Marshal(snapshot) + v.writeSnapshot(fmt.Sprintf("%d.snapshot.json", snapshot.Sequence), data) + } + + sortedAtomicFullOrderBook, err := v.level3Builder.DepthResponse2FullOrderBook(v.atomicFullOrderBook) + if err != nil { + log.Panic(fmt.Sprintf("verify order book快照校验失败, DepthResponse2FullOrderBook 转换失败, Sequence: %d, error: %s", snapshot.Sequence, err.Error())) + } + if err := v.diffOrderBook(snapshot, sortedAtomicFullOrderBook); err != nil { + if v.verifyLogDirectory != "" { + data, _ := json.Marshal(sortedAtomicFullOrderBook) + v.writeSnapshot(fmt.Sprintf("%d.atomicFullOrderBook.json", v.atomicFullOrderBook.Sequence), data) + } + log.Panic(fmt.Sprintf("verify order book快照校验失败, Sequence: %d, error: %s", snapshot.Sequence, err.Error())) + v.resetVerify(true) + continue + } else { + //开启一次新的消息流 + v.getNewFile(fmt.Sprintf("%d.log", snapshot.Sequence)) + } + + log.Info("verify Success !!! order book 快照比对成功", zap.Any("Sequence", snapshot.Sequence)) + v.resetVerify(true) + continue + } + + //以下逻辑提高校验效率，去掉无用的全量和后续增量获取 + if atomicFullOrderBookSequence > snapshot.Sequence+uint64(snapshotMaxLen)-uint64(len(v.snapshot)) { + log.Warn("获取到太大的全量", zap.Any("full", atomicFullOrderBookSequence), zap.Any("curr", snapshot.Sequence), zap.Any("len", len(v.snapshot))) + v.resetVerify(false) + continue + } + + if atomicFullOrderBookSequence < snapshot.Sequence-uint64(len(v.snapshot)) { + log.Warn("获取到太小的全量", zap.Any("full", atomicFullOrderBookSequence), zap.Any("curr", snapshot.Sequence), zap.Any("len", len(v.snapshot))) + v.resetVerify(false) + continue + } + } + + //最多20份快照 + if len(v.snapshot) > snapshotMaxLen { + log.Warn("跳过校验...") + v.resetVerify(false) + } + } + } +} + +func (v *Verify) diffOrderBook(snapshot, atomicFullOrderBook *orderbook.FullOrderBook) error { + if snapshot.Sequence != atomicFullOrderBook.Sequence { + return errors.New(fmt.Sprintf("sequence 不一致: %d - %d", snapshot.Sequence, atomicFullOrderBook.Sequence)) + } + + if err := v.diffOrderBookOneSide(snapshot.Asks, atomicFullOrderBook.Asks); err != nil { + return errors.New("asks 比对不一致, " + err.Error()) + } + + if err := v.diffOrderBookOneSide(snapshot.Bids, atomicFullOrderBook.Bids); err != nil { + return errors.New("bids 比对不一致, " + err.Error()) + } + + return nil +} + +func diff(a string, b string) error { + if a == b { + return nil + } + + aF, err := decimal.NewFromString(a) + if err != nil { + return err + } + bF, err := decimal.NewFromString(b) + if err != nil { + return err + } + + if !aF.Equal(bF) { + return errors.New("不相等: " + a + " != " + b) + } + + return nil +} + +func (v *Verify) diffOrderBookOneSide(items, atomicItems [][3]string) error { + if len(items) != len(atomicItems) { + return errors.New("比对失败，深度不一样") + } + + for index, item := range items { + //[3]string{"orderId", "price", "size"} + if item[0] != atomicItems[index][0] { + return errors.New(fmt.Sprintf("比对失败, index: %d, 订单id: %s != %s", index, item[0], atomicItems[index][0])) + } + + if err := diff(item[1], atomicItems[index][1]); err != nil { + return errors.New(fmt.Sprintf("price 比对失败, order id: %s, error: %s", item[0], err.Error())) + } + + if err := diff(item[2], atomicItems[index][2]); err != nil { + return errors.New(fmt.Sprintf("size 比对失败, order id: %s, error: %s", item[0], err.Error())) + } + } + + return nil +} + +func (v *Verify) getNewFile(filename string) { + //先关闭之前 + if v.update != nil { + _ = v.update.Close() + } + + filename = v.verifyLogDirectory + v.uniqStr + "-update-" + filename + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + panic(err) + } + + v.update = f +} + +func (v *Verify) writeWsMsg(data *sdk.WebSocketDownstreamMessage) { + if v.verifyLogDirectory == "" { + return + } + + if v.update == nil { + //刚开始未知seq，初始化一个 file + v.getNewFile(fmt.Sprintf("%s.log", time.Now().Format("2006-01-02-15-04-05"))) + } + msg, err := json.Marshal(data) + if err != nil { + panic(err) + } + msg = append(msg, '\n') + + if _, err := v.update.Write(msg); err != nil { + panic(err) + } +} + +func (v *Verify) writeSnapshot(filename string, data []byte) { + filename = v.verifyLogDirectory + v.uniqStr + "-" + filename + f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + panic(err) + } + defer f.Close() + + if _, err := f.Write(data); err != nil { + panic(err) + } +} + +func (v *Verify) checkLogDirExists() { + if v.verifyLogDirectory == "/" || v.verifyLogDirectory == "" { + v.verifyLogDirectory = "" + log.Warn("日志目录为空，不记录日志") + return + } + + if _, err := os.Stat(v.verifyLogDirectory); os.IsNotExist(err) { + panic("日志目录不存在: " + v.verifyLogDirectory) + } +} diff --git a/pkg/includes/exchanges.go b/pkg/includes/exchanges.go new file mode 100644 index 0000000..6723aba --- /dev/null +++ b/pkg/includes/exchanges.go @@ -0,0 +1,5 @@ +package includes + +import ( + _ "github.com/Kucoin/kucoin-level3-sdk/pkg/exchanges/kucoin-v2" +) diff --git a/pkg/services/log/config.go b/pkg/services/log/config.go new file mode 100644 index 0000000..5b1c880 --- /dev/null +++ b/pkg/services/log/config.go @@ -0,0 +1,90 @@ +package log + +import ( + "sort" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type Config struct { + // Level is the minimum enabled logging level. Note that this is a dynamic + // level, so calling Config.Level.SetLevel will atomically change the log + // level of all loggers descended from this config. + Level zap.AtomicLevel `json:"level" yaml:"level"` + // Development puts the logger in development mode, which changes the + // behavior of DPanicLevel and takes stacktraces more liberally. + Development bool `json:"development" yaml:"development"` + // DisableCaller stops annotating logs with the calling function's file + // name and line number. By default, all logs are annotated. + DisableCaller bool `json:"disableCaller" yaml:"disableCaller"` + // DisableStacktrace completely disables automatic stacktrace capturing. By + // default, stacktraces are captured for WarnLevel and above logs in + // development and ErrorLevel and above in production. + DisableStacktrace bool `json:"disableStacktrace" yaml:"disableStacktrace"` + // Sampling sets a sampling policy. A nil SamplingConfig disables sampling. + Sampling *zap.SamplingConfig `json:"sampling" yaml:"sampling"` + // Encoding sets the logger's encoding. Valid values are "json" and + // "console", as well as any third-party encodings registered via + // RegisterEncoder. + Encoder zapcore.Encoder + + WriteSyncer zapcore.WriteSyncer + + InitialFields map[string]interface{} `json:"initialFields" yaml:"initialFields"` +} + +func (cfg Config) Build(opts ...zap.Option) *zap.Logger { + core := zapcore.NewCore(cfg.Encoder, cfg.WriteSyncer, cfg.Level) + + log := zap.New(core, cfg.buildOptions()...) + if len(opts) > 0 { + log = log.WithOptions(opts...) + } + + return log +} + +func (cfg Config) buildOptions() []zap.Option { + opts := make([]zap.Option, 0, 4) + + if cfg.Development { + opts = append(opts, zap.Development()) + } + + if !cfg.DisableCaller { + opts = append(opts, zap.AddCaller()) + opts = append(opts, zap.AddCallerSkip(1)) + } + + //stacktrace + stackLevel := zap.ErrorLevel + if cfg.Development { + stackLevel = zap.WarnLevel + } + if !cfg.DisableStacktrace { + opts = append(opts, zap.AddStacktrace(stackLevel)) + } + + if cfg.Sampling != nil { + opts = append(opts, zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewSampler(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter)) + })) + } + + if len(cfg.InitialFields) > 0 { + fs := make([]zap.Field, 0, len(cfg.InitialFields)) + keys := make([]string, 0, len(cfg.InitialFields)) + for k := range cfg.InitialFields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fs = append(fs, zap.Any(k, cfg.InitialFields[k])) + } + opts = append(opts, zap.Fields(fs...)) + } + + return opts +} diff --git a/pkg/services/log/log_test.go b/pkg/services/log/log_test.go new file mode 100644 index 0000000..3fa0e76 --- /dev/null +++ b/pkg/services/log/log_test.go @@ -0,0 +1,11 @@ +package log + +import ( + "testing" +) + +func TestLogMessages(t *testing.T) { + New(true) + defer Sync() + Info("this is a test message") +} diff --git a/pkg/services/log/logger.go b/pkg/services/log/logger.go new file mode 100644 index 0000000..6afac20 --- /dev/null +++ b/pkg/services/log/logger.go @@ -0,0 +1,137 @@ +//package log +//see: https://github.com/uber-go/zap +//demo: +// _ = log.New(false) +// defer log.Sync() +// log.Info("this is a test message") +package log + +import ( + "errors" + "io" + "os" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" +) + +var logger *zap.Logger + +func Logger() *zap.Logger { + return logger +} + +func SetLogger(log *zap.Logger) error { + if logger != nil { + return errors.New("logger is not nil") + } + + logger = log + return nil +} + +func New(development bool) { + if logger != nil { + return + } + + var config Config + + if development { + config = NewDevelopment() + } else { + config = NewProduction() + } + + opts := make([]zap.Option, 0, 1) + + logger = config.Build(opts...) +} + +func NewDevelopment() Config { + encoderConfig := zap.NewDevelopmentEncoderConfig() + encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + + //https://github.com/natefinch/lumberjack + var w io.Writer + w = os.Stdout + sink := zapcore.AddSync(w) + + return Config{ + Level: zap.NewAtomicLevelAt(zap.DebugLevel), + Development: true, + Encoder: zapcore.NewConsoleEncoder(encoderConfig), + WriteSyncer: sink, + } +} + +func NewProduction() Config { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + //https://github.com/natefinch/lumberjack + var w io.Writer + w = os.Stdout + if cfg.AppConfig.App.LogFile != "" { + w = &lumberjack.Logger{ + Filename: cfg.AppConfig.App.LogFile, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 1, //days + Compress: true, // disabled by default + } + } + sink := zapcore.AddSync(w) + + return Config{ + Level: zap.NewAtomicLevelAt(zap.InfoLevel), + Development: false, + Encoder: zapcore.NewJSONEncoder(encoderConfig), + WriteSyncer: sink, + InitialFields: map[string]interface{}{ + "app_name": cfg.AppConfig.App.Name, + "market": cfg.AppConfig.MarketName(), + }, + //Sampling: &zap.SamplingConfig{ + // Initial: 100, + // Thereafter: 100, + //}, + } +} + +func Sync() error { + return logger.Sync() +} + +// +func Debug(msg string, fields ...zap.Field) { + logger.Debug(msg, fields...) +} + +func Info(msg string, fields ...zap.Field) { + logger.Info(msg, fields...) +} + +func Warn(msg string, fields ...zap.Field) { + logger.Warn(msg, fields...) +} + +func Error(msg string, fields ...zap.Field) { + logger.Error(msg, fields...) +} + +//DPanic DPanic means "development panic" +//Deprecated: deprecated +func DPanic(msg string, fields ...zap.Field) { + logger.DPanic(msg, fields...) +} + +func Panic(msg string, fields ...zap.Field) { + logger.Panic(msg, fields...) +} + +func Fatal(msg string, fields ...zap.Field) { + logger.Fatal(msg, fields...) +} diff --git a/pkg/services/redis/redis.go b/pkg/services/redis/redis.go new file mode 100644 index 0000000..2fdc1e3 --- /dev/null +++ b/pkg/services/redis/redis.go @@ -0,0 +1,63 @@ +package redis + +import ( + "errors" + "time" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/cfg" + "github.com/Kucoin/kucoin-level3-sdk/pkg/services/log" + "github.com/go-redis/redis" + "go.uber.org/zap" +) + +var redisConnections = make(map[string]*redis.Client) + +func newRedis(addr string, password string, db int) (*redis.Client, error) { + log.Info("connect redis: " + addr) + redisPool := redis.NewClient(&redis.Options{ + Addr: addr, + Password: password, // no password set + DB: db, // use default DB + //DialTimeout: 10 * time.Second, + //ReadTimeout: 30 * time.Second, + //WriteTimeout: 30 * time.Second, + //PoolSize: 10, + //PoolTimeout: 30 * time.Second, + }) + + if err := redisPool.Ping().Err(); err != nil { + return nil, errors.New("redis connect failed: " + err.Error()) + } + + time.Now().Format("20") + return redisPool, nil +} + +func InitConnections() { + conn := "default" + config := cfg.AppConfig.Redis + redisPool, err := newRedis(config.Addr, config.Password, config.Db) + if err != nil { + log.Panic("conn: " + conn + " newRedis redis err: " + err.Error()) + } + + redisConnections[conn] = redisPool +} + +func Connection(conn string) *redis.Client { + if conn == "" { + conn = "default" + } + + return redisConnections[conn] +} + +func Publish(conn string, channel string, message interface{}) error { + if err := Connection(conn).Publish(channel, message).Err(); err != nil { + log.Error("redis publish error, channel: "+channel, zap.Error(err), zap.Any("message", message)) + return err + } + + log.Debug("redis publish channel:"+channel, zap.Any("message", message)) + return nil +} diff --git a/pkg/utils/helper/helper.go b/pkg/utils/helper/helper.go new file mode 100644 index 0000000..8123e95 --- /dev/null +++ b/pkg/utils/helper/helper.go @@ -0,0 +1,14 @@ +package helper + +import ( + "encoding/json" +) + +// ToJsonString converts any value to JSON string. +func ToJsonString(v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + return "" + } + return string(b) +} diff --git a/pkg/utils/orderbook/base/helper.go b/pkg/utils/orderbook/base/helper.go new file mode 100644 index 0000000..2338372 --- /dev/null +++ b/pkg/utils/orderbook/base/helper.go @@ -0,0 +1,42 @@ +package base + +import ( + "encoding/json" + "errors" + "fmt" +) + +func Min(x, y int) int { + if x < y { + return x + } + + return y +} + +// ToJsonString converts any value to JSON string. +func ToJsonString(v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + return "" + } + return string(b) +} + +const ( + //Sort price from low to high + AskSide = "asks" + //Sort price from high to low + BidSide = "bids" +) + +func CheckSide(side string) error { + switch side { + case AskSide: + case BidSide: + default: + return errors.New(fmt.Sprintf("error side, side should be %s or %s", AskSide, BidSide)) + } + + return nil +} diff --git a/pkg/utils/orderbook/level2/order.go b/pkg/utils/orderbook/level2/order.go new file mode 100644 index 0000000..a77d665 --- /dev/null +++ b/pkg/utils/orderbook/level2/order.go @@ -0,0 +1,34 @@ +package level2 + +import ( + "fmt" + + "github.com/shopspring/decimal" +) + +//Price, Quantity +type Order struct { + Price decimal.Decimal + Size decimal.Decimal + Info interface{} +} + +func NewOrder(price string, size string, info interface{}) (order *Order, err error) { + priceValue, err := decimal.NewFromString(price) + if err != nil { + return nil, fmt.Errorf("NewOrder failed, price: `%s`, error: %v", price, err) + } + + sizeValue, err := decimal.NewFromString(size) + if err != nil { + return nil, fmt.Errorf("NewOrder failed, size: `%s`, error: %v", size, err) + } + + order = &Order{ + Price: priceValue, + Size: sizeValue, + Info: info, + } + + return +} diff --git a/pkg/utils/orderbook/level2/order_book.go b/pkg/utils/orderbook/level2/order_book.go new file mode 100644 index 0000000..4fb34f9 --- /dev/null +++ b/pkg/utils/orderbook/level2/order_book.go @@ -0,0 +1,137 @@ +package level2 + +import ( + "encoding/json" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/base" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/skiplist" + "github.com/shopspring/decimal" +) + +type OrderBook struct { + Asks *skiplist.SkipList //Sort price from low to high + Bids *skiplist.SkipList //Sort price from high to low +} + +func NewOrderBook() *OrderBook { + return &OrderBook{ + newAskOrders(), + newBidOrders(), + } +} + +func isEqual(l, r interface{}) bool { + switch val := l.(type) { + case decimal.Decimal: + cVal := r.(decimal.Decimal) + if !val.Equals(cVal) { + return false + } + default: + if val != r { + return false + } + } + return true +} + +func newAskOrders() *skiplist.SkipList { + return skiplist.NewCustomMap(func(l, r interface{}) bool { + return l.(decimal.Decimal).LessThan(r.(decimal.Decimal)) + }, isEqual) +} + +func newBidOrders() *skiplist.SkipList { + return skiplist.NewCustomMap(func(l, r interface{}) bool { + return l.(decimal.Decimal).GreaterThan(r.(decimal.Decimal)) + }, isEqual) +} + +func (ob *OrderBook) getOrderBookBySide(side string) (*skiplist.SkipList, error) { + if err := base.CheckSide(side); err != nil { + return nil, err + } + + if side == base.AskSide { + return ob.Asks, nil + } + + return ob.Bids, nil +} + +//addToOrderBookSide +func (ob *OrderBook) SetOrder(side string, order *Order) error { + orderBook, err := ob.getOrderBookBySide(side) + if err != nil { + return err + } + + if order.Size.IsZero() { + orderBook.Delete(order.Price) + return nil + } + + orderBook.Set(order.Price, order) + return nil +} + +func (ob *OrderBook) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + base.AskSide: ob.GetPartOrderBookBySide(base.AskSide, 0), + base.BidSide: ob.GetPartOrderBookBySide(base.BidSide, 0), + }) +} + +func (ob *OrderBook) GetPartOrderBookBySide(side string, number int) [][2]string { + if err := base.CheckSide(side); err != nil { + return nil + } + + var it skiplist.Iterator + if side == base.AskSide { + it = ob.Asks.Iterator() + if number == 0 { + number = ob.Asks.Len() + } else { + number = base.Min(number, ob.Asks.Len()) + } + } else { + it = ob.Bids.Iterator() + if number == 0 { + number = ob.Bids.Len() + } else { + number = base.Min(number, ob.Bids.Len()) + } + } + + arr := make([][2]string, number) + it.Next() + + for i := 0; i < number; i++ { + order := it.Value().(*Order) + arr[i] = [2]string{order.Price.String(), order.Size.String()} + if !it.Next() { + break + } + } + + return arr +} + +func (ob *OrderBook) GetOrderBookTickerOrder() (askOrder, bidOrder *Order) { + askIT := ob.Asks.Iterator() + askIT.Next() + ask := askIT.Value() + switch ask.(type) { + case *Order: + askOrder = ask.(*Order) + } + bidIT := ob.Bids.Iterator() + bidIT.Next() + bid := bidIT.Value() + switch bid.(type) { + case *Order: + bidOrder = bid.(*Order) + } + return +} diff --git a/pkg/utils/orderbook/level3/order.go b/pkg/utils/orderbook/level3/order.go new file mode 100644 index 0000000..01a054c --- /dev/null +++ b/pkg/utils/orderbook/level3/order.go @@ -0,0 +1,43 @@ +package level3 + +import ( + "fmt" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/base" + "github.com/shopspring/decimal" +) + +type Order struct { + OrderId string + Side string + Price decimal.Decimal + Size decimal.Decimal + Time uint64 + Info interface{} +} + +func NewOrder(orderId string, side string, price string, size string, time uint64, info interface{}) (order *Order, err error) { + if err := base.CheckSide(side); err != nil { + return nil, err + } + + priceValue, err := decimal.NewFromString(price) + if err != nil { + return nil, fmt.Errorf("NewOrder failed, price: `%s`, error: %v", price, err) + } + + sizeValue, err := decimal.NewFromString(size) + if err != nil { + return nil, fmt.Errorf("NewOrder failed, size: `%s`, error: %v", size, err) + } + + order = &Order{ + OrderId: orderId, + Side: side, + Price: priceValue, + Size: sizeValue, + Time: time, + Info: info, + } + return +} diff --git a/pkg/utils/orderbook/level3/order_book.go b/pkg/utils/orderbook/level3/order_book.go new file mode 100644 index 0000000..cb067c1 --- /dev/null +++ b/pkg/utils/orderbook/level3/order_book.go @@ -0,0 +1,273 @@ +package level3 + +import ( + "encoding/json" + "fmt" + + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/base" + "github.com/Kucoin/kucoin-level3-sdk/pkg/utils/orderbook/skiplist" + "github.com/shopspring/decimal" +) + +type OrderBook struct { + Sequence uint64 + Asks *skiplist.SkipList //Sort price from low to high + Bids *skiplist.SkipList //Sort price from high to low + orderPool map[string]*Order +} + +func NewOrderBook() *OrderBook { + return &OrderBook{ + Asks: newAskOrders(), + Bids: newBidOrders(), + orderPool: make(map[string]*Order), + } +} + +func isEqual(l, r interface{}) bool { + switch val := l.(type) { + case decimal.Decimal: + cVal := r.(decimal.Decimal) + if !val.Equals(cVal) { + return false + } + + case *Order: + cVal := r.(*Order) + if cVal.OrderId != val.OrderId { + return false + } + + default: + if val != r { + return false + } + } + return true +} + +func newAskOrders() *skiplist.SkipList { + return skiplist.NewCustomMap(func(l, r interface{}) bool { + if l.(*Order).Price.Equal(r.(*Order).Price) { + return l.(*Order).Time < r.(*Order).Time + } + + return l.(*Order).Price.LessThan(r.(*Order).Price) + }, isEqual) +} + +func newBidOrders() *skiplist.SkipList { + return skiplist.NewCustomMap(func(l, r interface{}) bool { + if l.(*Order).Price.Equal(r.(*Order).Price) { + return l.(*Order).Time < r.(*Order).Time + } + + return l.(*Order).Price.GreaterThan(r.(*Order).Price) + }, isEqual) +} + +func (ob *OrderBook) getOrderBookBySide(side string) (*skiplist.SkipList, error) { + if err := base.CheckSide(side); err != nil { + return nil, err + } + + if side == base.AskSide { + return ob.Asks, nil + } + + return ob.Bids, nil +} + +func (ob *OrderBook) AddOrder(order *Order) error { + orderBook, err := ob.getOrderBookBySide(order.Side) + if err != nil { + return err + } + + orderBook.Set(order, order) + ob.orderPool[order.OrderId] = order + return nil +} + +func (ob *OrderBook) RemoveByOrderId(orderId string) error { + order, ok := ob.orderPool[orderId] + if !ok { + return nil + } + + if err := ob.removeOrder(order); err != nil { + return err + } + return nil +} + +func (ob *OrderBook) removeOrder(order *Order) error { + orderBook, err := ob.getOrderBookBySide(order.Side) + if err != nil { + return err + } + if _, ok := orderBook.Delete(order); ok { + delete(ob.orderPool, order.OrderId) + } + + return nil +} + +func (ob *OrderBook) GetOrder(orderId string) *Order { + order, ok := ob.orderPool[orderId] + if !ok { + return nil + } + + return order +} + +func (ob *OrderBook) MatchOrder(orderId string, size decimal.Decimal) error { + order, ok := ob.orderPool[orderId] + if !ok { + return nil + } + + newSize := order.Size.Sub(size) + if newSize.LessThan(decimal.Zero) { + return fmt.Errorf("oldSize: %s, size: %s, sub result less than zero", order.Size.String(), size) + } + + order.Size = newSize + if order.Size.Equal(decimal.Zero) { + if err := ob.removeOrder(order); err != nil { + return err + } + return nil + } + + return nil +} + +func (ob *OrderBook) ChangeOrder(orderId string, size decimal.Decimal) error { + order, ok := ob.orderPool[orderId] + if !ok { + return nil + } + + order.Size = size + if order.Size.Equal(decimal.Zero) { + if err := ob.removeOrder(order); err != nil { + return err + } + return nil + } + return nil +} + +func (ob *OrderBook) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "sequence": ob.Sequence, + base.AskSide: ob.GetL3PartOrderBookBySide(base.AskSide, 0), + base.BidSide: ob.GetL3PartOrderBookBySide(base.BidSide, 0), + }) +} + +// Level3 OrderBook +func (ob *OrderBook) GetL3PartOrderBookBySide(side string, number int) [][3]string { + if err := base.CheckSide(side); err != nil { + return nil + } + + var it skiplist.Iterator + if side == base.AskSide { + it = ob.Asks.Iterator() + if number == 0 { + number = ob.Asks.Len() + } else { + number = base.Min(number, ob.Asks.Len()) + } + } else { + it = ob.Bids.Iterator() + if number == 0 { + number = ob.Bids.Len() + } else { + number = base.Min(number, ob.Bids.Len()) + } + } + arr := make([][3]string, 0, number) + + for it.Next() { + if len(arr) >= number { + break + } + + order := it.Value().(*Order) + arr = append(arr, [3]string{order.OrderId, order.Price.String(), order.Size.String()}) + } + + return arr +} + +// Level2 OrderBook +func (ob *OrderBook) GetPartOrderBookBySide(side string, number int) [][2]string { + if err := base.CheckSide(side); err != nil { + return nil + } + + var it skiplist.Iterator + if side == base.AskSide { + it = ob.Asks.Iterator() + if number == 0 { + number = ob.Asks.Len() + } else { + number = base.Min(number, ob.Asks.Len()) + } + } else { + it = ob.Bids.Iterator() + if number == 0 { + number = ob.Bids.Len() + } else { + number = base.Min(number, ob.Bids.Len()) + } + } + arr := make([][2]string, 0, number) + + lastPrice := decimal.Zero + lastPriceSize := decimal.Zero + + for it.Next() { + if len(arr) >= number { + break + } + + order := it.Value().(*Order) + if lastPrice.Equal(decimal.Zero) { + lastPrice = order.Price + lastPriceSize = order.Size + continue + } + if lastPrice.Equal(order.Price) { + lastPriceSize = lastPriceSize.Add(order.Size) + continue + } + arr = append(arr, [2]string{lastPrice.String(), lastPriceSize.String()}) + lastPrice = order.Price + lastPriceSize = order.Size + } + + return arr +} + +func (ob *OrderBook) GetOrderBookTickerOrder() (askOrder, bidOrder *Order) { + askIT := ob.Asks.Iterator() + askIT.Next() + ask := askIT.Value() + switch ask.(type) { + case *Order: + askOrder = ask.(*Order) + } + bidIT := ob.Bids.Iterator() + bidIT.Next() + bid := bidIT.Value() + switch bid.(type) { + case *Order: + bidOrder = bid.(*Order) + } + return +} diff --git a/pkg/utils/orderbook/skiplist/skiplist.go b/pkg/utils/orderbook/skiplist/skiplist.go new file mode 100644 index 0000000..4fc8186 --- /dev/null +++ b/pkg/utils/orderbook/skiplist/skiplist.go @@ -0,0 +1,536 @@ +// Copyright 2012 Google Inc. All rights reserved. +// Author: Ric Szopa (Ryszard) + +// Package skiplist implements skip list based maps and sets. +// +// Skip lists are a data structure that can be used in place of +// balanced trees. Skip lists use probabilistic balancing rather than +// strictly enforced balancing and as a result the algorithms for +// insertion and deletion in skip lists are much simpler and +// significantly faster than equivalent algorithms for balanced trees. +// +// Skip lists were first described in Pugh, William (June 1990). "Skip +// lists: a probabilistic alternative to balanced +// trees". Communications of the ACM 33 (6): 668–676 +// source from: https://github.com/ryszard/goskiplist +package skiplist + +import ( + "math/rand" +) + +// TODO(ryszard): +// - A separately seeded source of randomness + +// p is the fraction of nodes with level i pointers that also have +// level i+1 pointers. p equal to 1/4 is a good value from the point +// of view of speed and space requirements. If variability of running +// times is a concern, 1/2 is a better value for p. +const p = 0.25 + +const DefaultMaxLevel = 32 + +// A node is a container for key-value pairs that are stored in a skip +// list. +type node struct { + forward []*node + backward *node + key, value interface{} +} + +// next returns the next node in the skip list containing n. +func (n *node) next() *node { + if len(n.forward) == 0 { + return nil + } + return n.forward[0] +} + +// previous returns the previous node in the skip list containing n. +func (n *node) previous() *node { + return n.backward +} + +// hasNext returns true if n has a next node. +func (n *node) hasNext() bool { + return n.next() != nil +} + +// hasPrevious returns true if n has a previous node. +func (n *node) hasPrevious() bool { + return n.previous() != nil +} + +func (n *node) Get() interface{} { + return n.value +} + +func (n *node) Set(value interface{}) { + n.value = value +} + +// A SkipList is a map-like data structure that maintains an ordered +// collection of key-value pairs. Insertion, lookup, and deletion are +// all O(log n) operations. A SkipList can efficiently store up to +// 2^MaxLevel items. +// +// To iterate over a skip list (where s is a +// *SkipList): +// +// for i := s.Iterator(); i.Next(); { +// // do something with i.Key() and i.Value() +// } +type SkipList struct { + lessThan func(l, r interface{}) bool + isEqual func(l, r interface{}) bool + header *node + footer *node + length int + // MaxLevel determines how many items the SkipList can store + // efficiently (2^MaxLevel). + // + // It is safe to increase MaxLevel to accomodate more + // elements. If you decrease MaxLevel and the skip list + // already contains nodes on higer levels, the effective + // MaxLevel will be the greater of the new MaxLevel and the + // level of the highest node. + // + // A SkipList with MaxLevel equal to 0 is equivalent to a + // standard linked list and will not have any of the nice + // properties of skip lists (probably not what you want). + MaxLevel int +} + +// Len returns the length of s. +func (s *SkipList) Len() int { + return s.length +} + +// Iterator is an interface that you can use to iterate through the +// skip list (in its entirety or fragments). For an use example, see +// the documentation of SkipList. +// +// Key and Value return the key and the value of the current node. +type Iterator interface { + // Next returns true if the iterator contains subsequent elements + // and advances its state to the next element if that is possible. + Next() (ok bool) + // Previous returns true if the iterator contains previous elements + // and rewinds its state to the previous element if that is possible. + Previous() (ok bool) + // Key returns the current key. + Key() interface{} + // Value returns the current value. + Value() interface{} + // Seek reduces iterative seek costs for searching forward into the Skip List + // by remarking the range of keys over which it has scanned before. If the + // requested key occurs prior to the point, the Skip List will start searching + // as a safeguard. It returns true if the key is within the known range of + // the list. + Seek(key interface{}) (ok bool) + // Close this iterator to reap resources associated with it. While not + // strictly required, it will provide extra hints for the garbage collector. + Close() +} + +type iter struct { + current *node + key interface{} + list *SkipList + value interface{} +} + +func (i iter) Key() interface{} { + return i.key +} + +func (i iter) Value() interface{} { + return i.value +} + +func (i *iter) Next() bool { + if !i.current.hasNext() { + return false + } + + i.current = i.current.next() + i.key = i.current.key + i.value = i.current.value + + return true +} + +func (i *iter) Previous() bool { + if !i.current.hasPrevious() { + return false + } + + i.current = i.current.previous() + i.key = i.current.key + i.value = i.current.value + + return true +} + +func (i *iter) Seek(key interface{}) (ok bool) { + current := i.current + list := i.list + + // If the existing iterator outside of the known key range, we should set the + // position back to the beginning of the list. + if current == nil { + current = list.header + } + + // If the target key occurs before the current key, we cannot take advantage + // of the heretofore spent traversal cost to find it; resetting back to the + // beginning is the safest choice. + if current.key != nil && list.lessThan(key, current.key) { + current = list.header + } + + // We should back up to the so that we can seek to our present value if that + // is requested for whatever reason. + if current.backward == nil { + current = list.header + } else { + current = current.backward + } + + current = list.getPath(current, nil, key) + + if current == nil { + return + } + + i.current = current + i.key = current.key + i.value = current.value + + return true +} + +func (i *iter) Close() { + i.key = nil + i.value = nil + i.current = nil + i.list = nil +} + +type rangeIterator struct { + iter + upperLimit interface{} + lowerLimit interface{} +} + +func (i *rangeIterator) Next() bool { + if !i.current.hasNext() { + return false + } + + next := i.current.next() + + if !i.list.lessThan(next.key, i.upperLimit) { + return false + } + + i.current = i.current.next() + i.key = i.current.key + i.value = i.current.value + return true +} + +func (i *rangeIterator) Previous() bool { + if !i.current.hasPrevious() { + return false + } + + previous := i.current.previous() + + if i.list.lessThan(previous.key, i.lowerLimit) { + return false + } + + i.current = i.current.previous() + i.key = i.current.key + i.value = i.current.value + return true +} + +func (i *rangeIterator) Seek(key interface{}) (ok bool) { + if i.list.lessThan(key, i.lowerLimit) { + return + } else if !i.list.lessThan(key, i.upperLimit) { + return + } + + return i.iter.Seek(key) +} + +func (i *rangeIterator) Close() { + i.iter.Close() + i.upperLimit = nil + i.lowerLimit = nil +} + +// Iterator returns an Iterator that will go through all elements s. +func (s *SkipList) Iterator() Iterator { + return &iter{ + current: s.header, + list: s, + } +} + +// Seek returns a bidirectional iterator starting with the first element whose +// key is greater or equal to key; otherwise, a nil iterator is returned. +func (s *SkipList) Seek(key interface{}) Iterator { + current := s.getPath(s.header, nil, key) + if current == nil { + return nil + } + + return &iter{ + current: current, + key: current.key, + list: s, + value: current.value, + } +} + +// SeekToFirst returns a bidirectional iterator starting from the first element +// in the list if the list is populated; otherwise, a nil iterator is returned. +func (s *SkipList) SeekToFirst() Iterator { + if s.length == 0 { + return nil + } + + current := s.header.next() + + return &iter{ + current: current, + key: current.key, + list: s, + value: current.value, + } +} + +// SeekToLast returns a bidirectional iterator starting from the last element +// in the list if the list is populated; otherwise, a nil iterator is returned. +func (s *SkipList) SeekToLast() Iterator { + current := s.footer + if current == nil { + return nil + } + + return &iter{ + current: current, + key: current.key, + list: s, + value: current.value, + } +} + +// Range returns an iterator that will go through all the +// elements of the skip list that are greater or equal than from, but +// less than to. +func (s *SkipList) Range(from, to interface{}) Iterator { + start := s.getPath(s.header, nil, from) + return &rangeIterator{ + iter: iter{ + current: &node{ + forward: []*node{start}, + backward: start, + }, + list: s, + }, + upperLimit: to, + lowerLimit: from, + } +} + +func (s *SkipList) level() int { + return len(s.header.forward) - 1 +} + +func maxInt(x, y int) int { + if x > y { + return x + } + return y +} + +func (s *SkipList) effectiveMaxLevel() int { + return maxInt(s.level(), s.MaxLevel) +} + +// Returns a new random level. +func (s SkipList) randomLevel() (n int) { + for n = 0; n < s.effectiveMaxLevel() && rand.Float64() < p; n++ { + } + return +} + +// Get returns the value associated with key from s (nil if the key is +// not present in s). The second return value is true when the key is +// present. +func (s *SkipList) Get(key interface{}) (value interface{}, ok bool) { + candidate := s.getPath(s.header, nil, key) + + // if candidate == nil || candidate.key != key { + if candidate == nil || !s.isEqual(candidate.key, key) { + return nil, false + } + + return candidate.value, true +} + +// GetNode returns the node associated with key from s (nil if the key is +// not present in s). The second return value is true when the key is +// present. +func (s *SkipList) GetNode(key interface{}) (value *node, ok bool) { + candidate := s.getPath(s.header, nil, key) + + // if candidate == nil || candidate.key != key { + if candidate == nil || !s.isEqual(candidate.key, key) { + return nil, false + } + + return candidate, true +} + +// GetGreaterOrEqual finds the node whose key is greater than or equal +// to min. It returns its value, its actual key, and whether such a +// node is present in the skip list. +func (s *SkipList) GetGreaterOrEqual(min interface{}) (actualKey, value interface{}, ok bool) { + candidate := s.getPath(s.header, nil, min) + + if candidate != nil { + return candidate.key, candidate.value, true + } + return nil, nil, false +} + +// getPath populates update with nodes that constitute the path to the +// node that may contain key. The candidate node will be returned. If +// update is nil, it will be left alone (the candidate node will still +// be returned). If update is not nil, but it doesn't have enough +// slots for all the nodes in the path, getPath will panic. +func (s *SkipList) getPath(current *node, update []*node, key interface{}) *node { + depth := len(current.forward) - 1 + + for i := depth; i >= 0; i-- { + for current.forward[i] != nil && s.lessThan(current.forward[i].key, key) { + current = current.forward[i] + } + if update != nil { + update[i] = current + } + } + return current.next() +} + +// Sets set the value associated with key in s. +func (s *SkipList) Set(key, value interface{}) { + if key == nil { + panic("goskiplist: nil keys are not supported") + } + // s.level starts from 0, so we need to allocate one. + update := make([]*node, s.level()+1, s.effectiveMaxLevel()+1) + candidate := s.getPath(s.header, update, key) + + // if candidate != nil && candidate.key == key { + if candidate != nil && s.isEqual(candidate.key, key) { + candidate.value = value + return + } + + newLevel := s.randomLevel() + + if currentLevel := s.level(); newLevel > currentLevel { + // there are no pointers for the higher levels in + // update. Header should be there. Also add higher + // level links to the header. + for i := currentLevel + 1; i <= newLevel; i++ { + update = append(update, s.header) + s.header.forward = append(s.header.forward, nil) + } + } + + newNode := &node{ + forward: make([]*node, newLevel+1, s.effectiveMaxLevel()+1), + key: key, + value: value, + } + + if previous := update[0]; previous.key != nil { + newNode.backward = previous + } + + for i := 0; i <= newLevel; i++ { + newNode.forward[i] = update[i].forward[i] + update[i].forward[i] = newNode + } + + s.length++ + + if newNode.forward[0] != nil { + if newNode.forward[0].backward != newNode { + newNode.forward[0].backward = newNode + } + } + + if s.footer == nil || s.lessThan(s.footer.key, key) { + s.footer = newNode + } +} + +// Delete removes the node with the given key. +// +// It returns the old value and whether the node was present. +func (s *SkipList) Delete(key interface{}) (value interface{}, ok bool) { + if key == nil { + panic("goskiplist: nil keys are not supported") + } + update := make([]*node, s.level()+1, s.effectiveMaxLevel()) + candidate := s.getPath(s.header, update, key) + + // if candidate == nil || candidate.key != key { + if candidate == nil || !s.isEqual(candidate.key, key) { + return nil, false + } + + previous := candidate.backward + if s.footer == candidate { + s.footer = previous + } + + next := candidate.next() + if next != nil { + next.backward = previous + } + + for i := 0; i <= s.level() && update[i].forward[i] == candidate; i++ { + update[i].forward[i] = candidate.forward[i] + } + + for s.level() > 0 && s.header.forward[s.level()] == nil { + s.header.forward = s.header.forward[:s.level()] + } + s.length-- + + return candidate.value, true +} + +// NewCustomMap returns a new SkipList that will use lessThan as the +// comparison function. lessThan should define a linear order on keys +// you intend to use with the SkipList. +func NewCustomMap(lessThan func(l, r interface{}) bool, isEqual func(l, r interface{}) bool) *SkipList { + return &SkipList{ + lessThan: lessThan, + isEqual: isEqual, + header: &node{ + forward: []*node{nil}, + }, + MaxLevel: DefaultMaxLevel, + } +} diff --git a/pkg/utils/orderbook/skiplist/skiplist_test.go b/pkg/utils/orderbook/skiplist/skiplist_test.go new file mode 100644 index 0000000..5fc17c6 --- /dev/null +++ b/pkg/utils/orderbook/skiplist/skiplist_test.go @@ -0,0 +1,846 @@ +// Copyright 2012 Google Inc. All rights reserved. +// Author: Ric Szopa (Ryszard) + +// Package skiplist implements skip list based maps and sets. +// +// Skip lists are a data structure that can be used in place of +// balanced trees. Skip lists use probabilistic balancing rather than +// strictly enforced balancing and as a result the algorithms for +// insertion and deletion in skip lists are much simpler and +// significantly faster than equivalent algorithms for balanced trees. +// +// Skip lists were first described in Pugh, William (June 1990). "Skip +// lists: a probabilistic alternative to balanced +// trees". Communications of the ACM 33 (6): 668–676 +package skiplist + +import ( + "fmt" + "math/rand" + "sort" + "testing" +) + +func (s *SkipList) printRepr() { + + fmt.Printf("header:\n") + for i, link := range s.header.forward { + if link != nil { + fmt.Printf("\t%d: -> %v\n", i, link.key) + } else { + fmt.Printf("\t%d: -> END\n", i) + } + } + + for node := s.header.next(); node != nil; node = node.next() { + fmt.Printf("%v: %v (level %d)\n", node.key, node.value, len(node.forward)) + for i, link := range node.forward { + if link != nil { + fmt.Printf("\t%d: -> %v\n", i, link.key) + } else { + fmt.Printf("\t%d: -> END\n", i) + } + } + } + fmt.Println() +} + +func isEqual(l, r interface{}) bool { + switch val := l.(type) { + default: + if val != r { + return false + } + } + return true +} + +func TestInitialization(t *testing.T) { + s := NewCustomMap(func(l, r interface{}) bool { + return l.(int) < r.(int) + }, isEqual) + if !s.lessThan(1, 2) { + t.Errorf("Less than doesn't work correctly.") + } +} + +func TestEmptyNodeNext(t *testing.T) { + n := new(node) + if next := n.next(); next != nil { + t.Errorf("Next() should be nil for an empty node.") + } + + if n.hasNext() { + t.Errorf("hasNext() should be false for an empty node.") + } +} + +func TestEmptyNodePrev(t *testing.T) { + n := new(node) + if previous := n.previous(); previous != nil { + t.Errorf("Previous() should be nil for an empty node.") + } + + if n.hasPrevious() { + t.Errorf("hasPrevious() should be false for an empty node.") + } +} + +// NewIntKey returns a SkipList that accepts int keys. +func newIntMap() *SkipList { + return NewCustomMap(func(l, r interface{}) bool { + return l.(int) < r.(int) + }, isEqual) +} + +func TestNodeHasNext(t *testing.T) { + s := newIntMap() + s.Set(0, 0) + node := s.header.next() + if node.key != 0 { + t.Fatalf("We got the wrong node: %v.", node) + } + + if node.hasNext() { + t.Errorf("%v should be the last node.", node) + } +} + +func TestNodeHasPrev(t *testing.T) { + s := newIntMap() + s.Set(0, 0) + node := s.header.previous() + if node != nil { + t.Fatalf("Expected no previous entry, got %v.", node) + } +} + +func (s *SkipList) check(t *testing.T, key, wanted int) { + if got, _ := s.Get(key); got != wanted { + t.Errorf("For key %v wanted value %v, got %v.", key, wanted, got) + } +} + +func TestGet(t *testing.T) { + s := newIntMap() + s.Set(0, 0) + + if value, present := s.Get(0); !(value == 0 && present) { + t.Errorf("%v, %v instead of %v, %v", value, present, 0, true) + } + + if value, present := s.Get(100); value != nil || present { + t.Errorf("%v, %v instead of %v, %v", value, present, nil, false) + } +} + +func TestGetGreaterOrEqual(t *testing.T) { + s := newIntMap() + + if _, value, present := s.GetGreaterOrEqual(5); !(value == nil && !present) { + t.Errorf("s.GetGreaterOrEqual(5) should have returned nil and false for an empty map, not %v and %v.", value, present) + } + + s.Set(0, 0) + + if _, value, present := s.GetGreaterOrEqual(5); !(value == nil && !present) { + t.Errorf("s.GetGreaterOrEqual(5) should have returned nil and false for an empty map, not %v and %v.", value, present) + } + + s.Set(10, 10) + + if key, value, present := s.GetGreaterOrEqual(5); !(value == 10 && key == 10 && present) { + t.Errorf("s.GetGreaterOrEqual(5) should have returned 10 and true, not %v and %v.", value, present) + } +} + +func TestSet(t *testing.T) { + s := newIntMap() + if l := s.Len(); l != 0 { + t.Errorf("Len is not 0, it is %v", l) + } + + s.Set(0, 0) + s.Set(1, 1) + if l := s.Len(); l != 2 { + t.Errorf("Len is not 2, it is %v", l) + } + s.check(t, 0, 0) + if t.Failed() { + t.Errorf("header.Next() after s.Set(0, 0) and s.Set(1, 1): %v.", s.header.next()) + } + s.check(t, 1, 1) + +} + +func TestChange(t *testing.T) { + s := newIntMap() + s.Set(0, 0) + s.Set(1, 1) + s.Set(2, 2) + + s.Set(0, 7) + if value, _ := s.Get(0); value != 7 { + t.Errorf("Value should be 7, not %d", value) + } + s.Set(1, 8) + if value, _ := s.Get(1); value != 8 { + t.Errorf("Value should be 8, not %d", value) + } + +} + +func TestDelete(t *testing.T) { + s := newIntMap() + for i := 0; i < 10; i++ { + s.Set(i, i) + } + for i := 0; i < 10; i += 2 { + s.Delete(i) + } + + for i := 0; i < 10; i += 2 { + if _, present := s.Get(i); present { + t.Errorf("%d should not be present in s", i) + } + } + + if v, present := s.Delete(10000); v != nil || present { + t.Errorf("Deleting a non-existent key should return nil, false, and not %v, %v.", v, present) + } + + if t.Failed() { + s.printRepr() + } + +} + +func TestLen(t *testing.T) { + s := newIntMap() + for i := 0; i < 10; i++ { + s.Set(i, i) + } + if length := s.Len(); length != 10 { + t.Errorf("Length should be equal to 10, not %v.", length) + s.printRepr() + } + for i := 0; i < 5; i++ { + s.Delete(i) + } + if length := s.Len(); length != 5 { + t.Errorf("Length should be equal to 5, not %v.", length) + } + + s.Delete(10000) + + if length := s.Len(); length != 5 { + t.Errorf("Length should be equal to 5, not %v.", length) + } + +} + +func TestIteration(t *testing.T) { + s := newIntMap() + for i := 0; i < 20; i++ { + s.Set(i, i) + } + + seen := 0 + var lastKey int + + i := s.Iterator() + defer i.Close() + + for i.Next() { + seen++ + lastKey = i.Key().(int) + if i.Key() != i.Value() { + t.Errorf("Wrong value for key %v: %v.", i.Key(), i.Value()) + } + } + + if seen != s.Len() { + t.Errorf("Not all the items in s where iterated through (seen %d, should have seen %d). Last one seen was %d.", seen, s.Len(), lastKey) + } + + for i.Previous() { + if i.Key() != i.Value() { + t.Errorf("Wrong value for key %v: %v.", i.Key(), i.Value()) + } + + if i.Key().(int) >= lastKey { + t.Errorf("Expected key to descend but ascended from %v to %v.", lastKey, i.Key()) + } + + lastKey = i.Key().(int) + } + + if lastKey != 0 { + t.Errorf("Expected to count back to zero, but stopped at key %v.", lastKey) + } +} + +func TestRangeIteration(t *testing.T) { + s := newIntMap() + for i := 0; i < 20; i++ { + s.Set(i, i) + } + + max, min := 0, 100000 + var lastKey, seen int + + i := s.Range(5, 10) + defer i.Close() + + for i.Next() { + seen++ + lastKey = i.Key().(int) + if lastKey > max { + max = lastKey + } + if lastKey < min { + min = lastKey + } + if i.Key() != i.Value() { + t.Errorf("Wrong value for key %v: %v.", i.Key(), i.Value()) + } + } + + if seen != 5 { + t.Errorf("The number of items yielded is incorrect (should be 5, was %v)", seen) + } + if min != 5 { + t.Errorf("The smallest element should have been 5, not %v", min) + } + + if max != 9 { + t.Errorf("The largest element should have been 9, not %v", max) + } + + if i.Seek(4) { + t.Error("Allowed to seek to invalid range.") + } + + if !i.Seek(5) { + t.Error("Could not seek to an allowed range.") + } + if i.Key().(int) != 5 || i.Value().(int) != 5 { + t.Errorf("Expected 5 for key and 5 for value, got %d and %d", i.Key(), i.Value()) + } + + if !i.Seek(7) { + t.Error("Could not seek to an allowed range.") + } + if i.Key().(int) != 7 || i.Value().(int) != 7 { + t.Errorf("Expected 7 for key and 7 for value, got %d and %d", i.Key(), i.Value()) + } + + if i.Seek(10) { + t.Error("Allowed to seek to invalid range.") + } + + i.Seek(9) + + seen = 0 + min = 100000 + max = -1 + + for i.Previous() { + seen++ + lastKey = i.Key().(int) + if lastKey > max { + max = lastKey + } + if lastKey < min { + min = lastKey + } + if i.Key() != i.Value() { + t.Errorf("Wrong value for key %v: %v.", i.Key(), i.Value()) + } + } + + if seen != 4 { + t.Errorf("The number of items yielded is incorrect (should be 5, was %v)", seen) + } + if min != 5 { + t.Errorf("The smallest element should have been 5, not %v", min) + } + + if max != 8 { + t.Errorf("The largest element should have been 9, not %v", max) + } +} + +func TestSomeMore(t *testing.T) { + s := newIntMap() + insertions := [...]int{4, 1, 2, 9, 10, 7, 3} + for _, i := range insertions { + s.Set(i, i) + } + for _, i := range insertions { + s.check(t, i, i) + } + +} + +func makeRandomList(n int) *SkipList { + s := newIntMap() + for i := 0; i < n; i++ { + insert := rand.Int() + s.Set(insert, insert) + } + return s +} + +func LookupBenchmark(b *testing.B, n int) { + b.StopTimer() + s := makeRandomList(n) + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Get(rand.Int()) + } +} + +func SetBenchmark(b *testing.B, n int) { + b.StopTimer() + values := []int{} + for i := 0; i < b.N; i++ { + values = append(values, rand.Int()) + } + s := newIntMap() + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Set(values[i], values[i]) + } +} + +// Make sure that all the keys are unique and are returned in order. +func TestSanity(t *testing.T) { + s := newIntMap() + for i := 0; i < 10000; i++ { + insert := rand.Int() + s.Set(insert, insert) + } + var last int = 0 + + i := s.Iterator() + defer i.Close() + + for i.Next() { + if last != 0 && i.Key().(int) <= last { + t.Errorf("Not in order!") + } + last = i.Key().(int) + } + + for i.Previous() { + if last != 0 && i.Key().(int) > last { + t.Errorf("Not in order!") + } + last = i.Key().(int) + } +} + +func TestSetMaxLevelInFlight(t *testing.T) { + s := newIntMap() + s.MaxLevel = 2 + for i := 0; i < 64; i++ { + insert := 2 * rand.Int() + s.Set(insert, insert) + } + + s.MaxLevel = 64 + for i := 0; i < 65536; i++ { + insert := 2*rand.Int() + 1 + s.Set(insert, insert) + } + + i := s.Iterator() + defer i.Close() + + for i.Next() { + if v, _ := s.Get(i.Key()); v != i.Key() { + t.Errorf("Bad values in the skip list (%v). Inserted before the call to s.SetMax(): %t.", v, i.Key().(int)%2 == 0) + } + } +} + +func TestDeletingHighestLevelNodeDoesntBreakSkiplist(t *testing.T) { + s := newIntMap() + elements := []int{1, 3, 5, 7, 0, 4, 5, 10, 11} + + for _, i := range elements { + s.Set(i, i) + } + + highestLevelNode := s.header.forward[len(s.header.forward)-1] + + s.Delete(highestLevelNode.key) + + seen := 0 + i := s.Iterator() + defer i.Close() + + for i.Next() { + seen++ + } + if seen == 0 { + t.Errorf("Iteration is broken (no elements seen).") + } +} + +func TestIteratorPrevHoles(t *testing.T) { + m := newIntMap() + + i := m.Iterator() + defer i.Close() + + m.Set(0, 0) + m.Set(1, 1) + m.Set(2, 2) + + if !i.Next() { + t.Errorf("Expected iterator to move successfully to the next.") + } + + if !i.Next() { + t.Errorf("Expected iterator to move successfully to the next.") + } + + if !i.Next() { + t.Errorf("Expected iterator to move successfully to the next.") + } + + if i.Key().(int) != 2 || i.Value().(int) != 2 { + t.Errorf("Expected iterator to reach key 2 and value 2, got %v and %v.", i.Key(), i.Value()) + } + + if !i.Previous() { + t.Errorf("Expected iterator to move successfully to the previous.") + } + + if i.Key().(int) != 1 || i.Value().(int) != 1 { + t.Errorf("Expected iterator to reach key 1 and value 1, got %v and %v.", i.Key(), i.Value()) + } + + if !i.Next() { + t.Errorf("Expected iterator to move successfully to the next.") + } + + m.Delete(1) + + if !i.Previous() { + t.Errorf("Expected iterator to move successfully to the previous.") + } + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } +} + +func TestIteratorSeek(t *testing.T) { + m := newIntMap() + + i := m.Seek(0) + + if i != nil { + t.Errorf("Expected nil iterator, but got %v.", i) + } + + i = m.SeekToFirst() + + if i != nil { + t.Errorf("Expected nil iterator, but got %v.", i) + } + + i = m.SeekToLast() + + if i != nil { + t.Errorf("Expected nil iterator, but got %v.", i) + } + + m.Set(0, 0) + + i = m.SeekToFirst() + defer i.Close() + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + i = m.SeekToLast() + defer i.Close() + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + m.Set(1, 1) + + i = m.SeekToFirst() + defer i.Close() + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + i = m.SeekToLast() + defer i.Close() + + if i.Key().(int) != 1 || i.Value().(int) != 1 { + t.Errorf("Expected iterator to reach key 1 and value 1, got %v and %v.", i.Key(), i.Value()) + } + + m.Set(2, 2) + + i = m.SeekToFirst() + defer i.Close() + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + i = m.SeekToLast() + defer i.Close() + + if i.Key().(int) != 2 || i.Value().(int) != 2 { + t.Errorf("Expected iterator to reach key 2 and value 2, got %v and %v.", i.Key(), i.Value()) + } + + i = m.Seek(0) + defer i.Close() + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + i = m.Seek(2) + defer i.Close() + + if i.Key().(int) != 2 || i.Value().(int) != 2 { + t.Errorf("Expected iterator to reach key 2 and value 2, got %v and %v.", i.Key(), i.Value()) + } + + i = m.Seek(1) + defer i.Close() + + if i.Key().(int) != 1 || i.Value().(int) != 1 { + t.Errorf("Expected iterator to reach key 1 and value 1, got %v and %v.", i.Key(), i.Value()) + } + + i = m.Seek(3) + + if i != nil { + t.Errorf("Expected to receive nil iterator, got %v.", i) + } + + m.Set(4, 4) + + i = m.Seek(4) + defer i.Close() + + if i.Key().(int) != 4 || i.Value().(int) != 4 { + t.Errorf("Expected iterator to reach key 4 and value 4, got %v and %v.", i.Key(), i.Value()) + } + + i = m.Seek(3) + defer i.Close() + + if i.Key().(int) != 4 || i.Value().(int) != 4 { + t.Errorf("Expected iterator to reach key 4 and value 4, got %v and %v.", i.Key(), i.Value()) + } + + m.Delete(4) + + i = m.SeekToFirst() + defer i.Close() + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + i = m.SeekToLast() + defer i.Close() + + if i.Key().(int) != 2 || i.Value().(int) != 2 { + t.Errorf("Expected iterator to reach key 2 and value 2, got %v and %v.", i.Key(), i.Value()) + } + + if !i.Seek(2) { + t.Error("Expected iterator to seek to key.") + } + + if i.Key().(int) != 2 || i.Value().(int) != 2 { + t.Errorf("Expected iterator to reach key 2 and value 2, got %v and %v.", i.Key(), i.Value()) + } + + if !i.Seek(1) { + t.Error("Expected iterator to seek to key.") + } + + if i.Key().(int) != 1 || i.Value().(int) != 1 { + t.Errorf("Expected iterator to reach key 1 and value 1, got %v and %v.", i.Key(), i.Value()) + } + + if !i.Seek(0) { + t.Error("Expected iterator to seek to key.") + } + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } + + i = m.SeekToFirst() + defer i.Close() + + if !i.Seek(0) { + t.Error("Expected iterator to seek to key.") + } + + if i.Key().(int) != 0 || i.Value().(int) != 0 { + t.Errorf("Expected iterator to reach key 0 and value 0, got %v and %v.", i.Key(), i.Value()) + } +} + +func BenchmarkLookup16(b *testing.B) { + LookupBenchmark(b, 16) +} + +func BenchmarkLookup256(b *testing.B) { + LookupBenchmark(b, 256) +} + +func BenchmarkLookup65536(b *testing.B) { + LookupBenchmark(b, 65536) +} + +func BenchmarkSet16(b *testing.B) { + SetBenchmark(b, 16) +} + +func BenchmarkSet256(b *testing.B) { + SetBenchmark(b, 256) +} + +func BenchmarkSet65536(b *testing.B) { + SetBenchmark(b, 65536) +} + +func BenchmarkRandomSeek(b *testing.B) { + b.StopTimer() + values := []int{} + s := newIntMap() + for i := 0; i < b.N; i++ { + r := rand.Int() + values = append(values, r) + s.Set(r, r) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + iterator := s.Seek(values[i]) + if iterator == nil { + b.Errorf("got incorrect value for index %d", i) + } + } +} + +const ( + lookAhead = 10 +) + +// This test is used for the baseline comparison of Iterator.Seek when +// performing forward sequential seek operations. +func BenchmarkForwardSeek(b *testing.B) { + b.StopTimer() + + values := []int{} + s := newIntMap() + valueCount := b.N + for i := 0; i < valueCount; i++ { + r := rand.Int() + values = append(values, r) + s.Set(r, r) + } + sort.Ints(values) + + b.StartTimer() + for i := 0; i < b.N; i++ { + key := values[i] + iterator := s.Seek(key) + if i < valueCount-lookAhead { + nextKey := values[i+lookAhead] + + iterator = s.Seek(nextKey) + if iterator.Key().(int) != nextKey || iterator.Value().(int) != nextKey { + b.Errorf("%d. expected %d key and %d value, got %d key and %d value", i, nextKey, nextKey, iterator.Key(), iterator.Value()) + } + } + } +} + +// This test demonstrates the amortized cost of a forward sequential seek. +func BenchmarkForwardSeekReusedIterator(b *testing.B) { + b.StopTimer() + + values := []int{} + s := newIntMap() + valueCount := b.N + for i := 0; i < valueCount; i++ { + r := rand.Int() + values = append(values, r) + s.Set(r, r) + + } + sort.Ints(values) + + b.StartTimer() + for i := 0; i < b.N; i++ { + key := values[i] + iterator := s.Seek(key) + if i < valueCount-lookAhead { + nextKey := values[i+lookAhead] + + if !iterator.Seek(nextKey) { + b.Errorf("%d. expected iterator to seek to %d key; failed.", i, nextKey) + } else if iterator.Key().(int) != nextKey || iterator.Value().(int) != nextKey { + b.Errorf("%d. expected %d key and %d value, got %d key and %d value", i, nextKey, nextKey, iterator.Key(), iterator.Value()) + } + } + } +} + +// newStringMap returns a SkipList that accepts string keys. +func newStringMap() *SkipList { + return NewCustomMap(func(l, r interface{}) bool { + return l.(string) < r.(string) + }, isEqual) +} + +func TestNewStringMap(t *testing.T) { + s := newStringMap() + s.Set("a", 1) + s.Set("b", 2) + if value, _ := s.Get("a"); value != 1 { + t.Errorf("Expected 1, got %v.", value) + } +} + +func TestGetNilKey(t *testing.T) { + s := newStringMap() + if v, present := s.Get(nil); v != nil || present { + t.Errorf("s.Get(nil) should return nil, false (not %v, %v).", v, present) + } +} + +func TestSetNilKey(t *testing.T) { + s := newStringMap() + + defer func() { + if err := recover(); err == nil { + t.Errorf("s.Set(nil, 0) should have panicked.") + } + }() + + s.Set(nil, 0) + +} diff --git a/runtime/.gitignore b/runtime/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/runtime/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/service/redis.go b/service/redis.go deleted file mode 100644 index 787bbbc..0000000 --- a/service/redis.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "github.com/Kucoin/kucoin-level3-sdk/utils/log" - "github.com/go-redis/redis" -) - -type Redis struct { - redisPool *redis.Client -} - -const RedisKeyPrefix = "kucoinMarket:rpcKey:" - -func NewRedis(addr, password string, db int, rpcKey string, symbol string, rpcPort string) *Redis { - log.Warn("connect to redis: " + addr) - - redisPool := redis.NewClient(&redis.Options{ - Addr: addr, - Password: password, // no password set - DB: db, // use default DB - //DialTimeout: 10 * time.Second, - //ReadTimeout: 30 * time.Second, - //WriteTimeout: 30 * time.Second, - //PoolSize: 10, - //PoolTimeout: 30 * time.Second, - }) - - //redisKey like: kucoinMarket:rpcKey:KCS-USDT:rpcKey - if err := redisPool.Set(RedisKeyPrefix+symbol+":"+rpcKey, rpcPort, 0).Err(); err != nil { - panic("connect to redis failed: " + err.Error()) - } - - return &Redis{ - redisPool: redisPool, - } -} - -func (r *Redis) Publish(channel string, message interface{}) error { - return r.redisPool.Publish(channel, message).Err() -} diff --git a/utils/log/log.go b/utils/log/log.go deleted file mode 100644 index 3fbb7f6..0000000 --- a/utils/log/log.go +++ /dev/null @@ -1,24 +0,0 @@ -package log - -import ( - "log" - "os" -) - -var logger = log.New(os.Stdout, "", log.LstdFlags) - -func Info(format string, v ...interface{}) { - logger.Printf("[Info] "+format+"\n", v...) -} - -func Warn(format string, v ...interface{}) { - logger.Printf("\033[33m[Warn] "+format+"\033[0m\n", v...) -} - -func Error(format string, v ...interface{}) { - logger.Printf("\033[31m[Error] "+format+"\033[0m\n", v...) -} - -func Fatal(format string, v ...interface{}) { - logger.Fatalf("\033[31m[Fatal] "+format+"\033[0m\n", v...) -} diff --git a/utils/log/log_test.go b/utils/log/log_test.go deleted file mode 100644 index 84d1806..0000000 --- a/utils/log/log_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package log - -import "testing" - -func TestInfo(t *testing.T) { - Info("this is a info msg: %s", "hello world.") -} - -func TestWarn(t *testing.T) { - Warn("this is a warn msg: %s", "hello world.") -} - -func TestError(t *testing.T) { - Error("this is a error msg: %s", "hello world.") -} - -func TestFatal(t *testing.T) { - //Fatal("this is a error msg: %s", "hello world.") -} diff --git a/utils/recovery/recovery.go b/utils/recovery/recovery.go deleted file mode 100644 index 9eca5bb..0000000 --- a/utils/recovery/recovery.go +++ /dev/null @@ -1,91 +0,0 @@ -package recovery - -import ( - "bytes" - "fmt" - "io/ioutil" - "runtime" - "time" -) - -func Recover(handler func(stack string)) func() { - return func() { - if r := recover(); r != nil { - stack := Stack(3) - reset := string([]byte{27, 91, 48, 109}) - handler(fmt.Sprintf("[%s] panic recovered: %s\n%s%s", TimeFormat(time.Now()), r, stack, reset)) - } - } -} - -var ( - dunno = []byte("???") - centerDot = []byte("·") - dot = []byte(".") - slash = []byte("/") -) - -// stack returns a nicely formatted stack frame, skipping skip frames. -func Stack(skip int) string { - buf := new(bytes.Buffer) // the returned data - // As we loop, we open files and read them. These variables record the currently - // loaded file. - var lines [][]byte - var lastFile string - for i := skip; ; i++ { // Skip the expected number of frames - pc, file, line, ok := runtime.Caller(i) - if !ok { - break - } - // Print this much at least. If we can't find the source, it won't show. - fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc) - if file != lastFile { - data, err := ioutil.ReadFile(file) - if err != nil { - continue - } - lines = bytes.Split(data, []byte{'\n'}) - lastFile = file - } - fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line)) - } - return string(buf.Bytes()) -} - -// source returns a space-trimmed slice of the n'th line. -func source(lines [][]byte, n int) []byte { - n-- // in stack trace, lines are 1-indexed but our array is 0-indexed - if n < 0 || n >= len(lines) { - return dunno - } - return bytes.TrimSpace(lines[n]) -} - -// function returns, if possible, the name of the function containing the PC. -func function(pc uintptr) []byte { - fn := runtime.FuncForPC(pc) - if fn == nil { - return dunno - } - name := []byte(fn.Name()) - // The name includes the path name to the package, which is unnecessary - // since the file name is already included. Plus, it has center dots. - // That is, we see - // runtime/debug.*T·ptrmethod - // and want - // *T.ptrmethod - // Also the package path might contains dot (e.g. code.google.com/...), - // so first eliminate the path prefix - if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 { - name = name[lastslash+1:] - } - if period := bytes.Index(name, dot); period >= 0 { - name = name[period+1:] - } - name = bytes.Replace(name, centerDot, dot, -1) - return name -} - -func TimeFormat(t time.Time) string { - return t.Format("2006/01/02 15:04:05") -}