Add Updater auto (#27)

* Add updater auto (BEP2)

* Add omitempty property to other asset fields

* Refactor code

* Implement generating of tokens list

* Implement createTokenListJSON

* Fix review comments

configs/example.config.yaml

@@ -2,7 +2,12 @@ app:
   log_level: info
 
 client_urls:
-  binancedex: "https://dex-atlantic.binance.org"
+  binance:
+    explorer: https://explorer.binance.org
+    dex: https://dex-atlantic.binance.org/api
+
+urls:
+  tw_assets_app: https://assets.trustwalletapp.com
 
 validators_settings:
   root_folder:
@@ -94,4 +99,7 @@ validators_settings:
         description: Binance-Peg tokens.
       - id: deflationary
         name: Deflationary
-        description: Tokens that are deflationary or use mechanism to burn a token on transfer/swap. 
+        description: Tokens that are deflationary or use mechanism to burn a token on transfer/swap.
+      - id: memes
+        name: Memes
+        description: A cryptocurrency that is associated with some theme, often as a joke rather than a serious product.

go.mod

@@ -4,8 +4,8 @@ go 1.17
 
 require (
 	github.com/sirupsen/logrus v1.8.1
-	github.com/trustwallet/go-libs v0.2.17
-	github.com/trustwallet/go-primitives v0.0.16-0.20211207135220-04b1926f0aa0
+	github.com/trustwallet/go-libs v0.2.19
+	github.com/trustwallet/go-primitives v0.0.16
 )
 
 require (
@@ -14,7 +14,9 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

go.sum

@@ -229,6 +229,7 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
 github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -242,6 +243,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -270,12 +272,12 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/trustwallet/go-libs v0.2.17 h1:l2zr5Ow19GTPnoWbhjExAhTbz7xLcYPXBa10EIgItjI=
-github.com/trustwallet/go-libs v0.2.17/go.mod h1:gBDrDkN5wOY+8kwlU6pnv3l/seGJy2qH1OpTKe7NnXA=
-github.com/trustwallet/go-primitives v0.0.14-0.20211130135322-c76db95f1406 h1:lezUwyiPYCChVu12XPGlc51cG4GE6hE0ONrKNG/UDQ8=
-github.com/trustwallet/go-primitives v0.0.14-0.20211130135322-c76db95f1406/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
+github.com/trustwallet/go-libs v0.2.19 h1:GTdMWYGJpTiWy7XOitveeviQIUYazU4PVxnjUBruT3g=
+github.com/trustwallet/go-libs v0.2.19/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
 github.com/trustwallet/go-primitives v0.0.16-0.20211207135220-04b1926f0aa0 h1:Dd2L3iCptLdCQFt4bBH5wqrW50J7JqmEcP8aDHSZY0g=
 github.com/trustwallet/go-primitives v0.0.16-0.20211207135220-04b1926f0aa0/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
+github.com/trustwallet/go-primitives v0.0.16 h1:RoivfOFmMx8C4felrwWHuIWtWLm7Gd9Fj0mLIBZ8qEY=
+github.com/trustwallet/go-primitives v0.0.16/go.mod h1:jLqd7rm+4EYG5JdpxhngM9HwbqfEXzKy/wK4vUB7STs=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

pkg/file/cache.go

@@ -4,33 +4,33 @@ import (
 	"sync"
 )
 
-type FileService struct {
+type Service struct {
 	mu    *sync.RWMutex
 	cache map[string]*AssetFile
 }
 
-func NewFileService(filePaths ...string) *FileService {
+func NewService(filePaths ...string) *Service {
 	var filesMap = make(map[string]*AssetFile)
 
 	for _, path := range filePaths {
 		assetFile := NewAssetFile(path)
 		filesMap[path] = assetFile
 	}
 
-	return &FileService{
+	return &Service{
 		mu:    &sync.RWMutex{},
 		cache: filesMap,
 	}
 }
 
-func (f *FileService) GetAssetFile(path string) (*AssetFile, error) {
+func (f *Service) GetAssetFile(path string) (*AssetFile, error) {
 	f.mu.RLock()
 	defer f.mu.RUnlock()
 
 	return f.getFile(path)
 }
 
-func (f *FileService) getFile(path string) (*AssetFile, error) {
+func (f *Service) getFile(path string) (*AssetFile, error) {
 	if file, exists := f.cache[path]; exists {
 		err := file.Open()
 		if err != nil {

pkg/files.go

@@ -1,6 +1,13 @@
 package pkg
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"image/png"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -23,3 +30,73 @@ func IsFileAllowedInPR(path string) bool {
 
 	return false
 }
+
+func FileExists(path string) bool {
+	_, err := os.Stat(path)
+
+	return !os.IsNotExist(err)
+}
+
+func CreateDirPath(path string) {
+	dirPath := filepath.Dir(path)
+	os.MkdirAll(dirPath, os.ModePerm)
+}
+
+func CreatePNGFromURL(logoURL, logoPath string) error {
+	imgBytes, err := GetHTTPResponseBytes(logoURL)
+	if err != nil {
+		return err
+	}
+
+	img, err := png.Decode(bytes.NewReader(imgBytes))
+	if err != nil {
+		return fmt.Errorf("failed to decode image bytes: %v", err)
+	}
+
+	out, err := os.Create(logoPath)
+	if err != nil {
+		return fmt.Errorf("failed to create file: %v", err)
+	}
+	defer out.Close()
+
+	err = png.Encode(out, img)
+	if err != nil {
+		return fmt.Errorf("failed to encode image: %v", err)
+	}
+
+	return nil
+}
+
+func CreateJSONFile(path string, payload interface{}) error {
+	file, err := json.MarshalIndent(payload, "", "    ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal json: %v", err)
+	}
+
+	err = ioutil.WriteFile(path, file, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write json to file: %v", err)
+	}
+
+	return nil
+}
+
+func ReadJSONFile(path string, result interface{}) error {
+	jsonFile, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %v", err)
+	}
+	defer jsonFile.Close()
+
+	data, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return fmt.Errorf("failed to read data from file: %v", err)
+	}
+
+	err = json.Unmarshal(data, result)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal json: %v", err)
+	}
+
+	return nil
+}

pkg/http.go

@@ -21,7 +21,7 @@ func GetHTTPResponse(url string, v interface{}) error {
 
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("failed to read bytes from body: %v", err)
+		return fmt.Errorf("failed to read body: %v", err)
 	}
 
 	err = json.Unmarshal(body, v)
@@ -42,3 +42,18 @@ func GetHTTPResponseCode(url string) (int, error) {
 
 	return res.StatusCode, nil
 }
+
+func GetHTTPResponseBytes(url string) ([]byte, error) {
+	response, err := http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make GET request: %v", err)
+	}
+	defer response.Body.Close()
+
+	bytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read body: %v", err)
+	}
+
+	return bytes, nil
+}

pkg/validation/info/fields_validators.go

@@ -44,8 +44,8 @@ func ValidateAssetRequiredKeys(a AssetModel) error {
 	}
 
 	if len(fields) != len(requiredAssetFields) {
-		return fmt.Errorf("missing or empty required fields\n-%s",
-			strings.Join(difference(requiredAssetFields, fields), "\n"))
+		return fmt.Errorf("missing or empty required fields: %s",
+			strings.Join(difference(requiredAssetFields, fields), ", "))
 	}
 
 	return nil
@@ -118,8 +118,8 @@ func ValidateCoinRequiredKeys(c CoinModel) error {
 	}
 
 	if len(fields) != len(requiredCoinFields) {
-		return fmt.Errorf("missing or empty required fields\n-%s",
-			strings.Join(difference(requiredCoinFields, fields), "\n"))
+		return fmt.Errorf("missing or empty required fields: %s",
+			strings.Join(difference(requiredCoinFields, fields), ", "))
 	}
 
 	return nil
@@ -136,14 +136,14 @@ func ValidateCoinLinks(links []Link) error {
 		}
 
 		if !linkNameAllowed(*l.Name) {
-			return fmt.Errorf("invalid value for links.name filed, allowed only - %s",
+			return fmt.Errorf("invalid value for links.name filed, allowed only: %s",
 				strings.Join(supportedLinkNames(), ", "))
 		}
 
 		prefix := allowedLinkKeys[*l.Name]
 		if prefix != "" {
 			if !strings.HasPrefix(*l.URL, prefix) {
-				return fmt.Errorf("invalid value for links.url field, allowed only with prefixes - %s",
+				return fmt.Errorf("invalid value for links.url field, allowed only with prefixes: %s",
 					strings.Join(supportedLinkValues(), ", "))
 			}
 		}

pkg/validation/info/model.go

@@ -30,19 +30,19 @@ type (
 		Explorer      *string  `json:"explorer,omitempty"`
 		Status        *string  `json:"status,omitempty"`
 		ID            *string  `json:"id,omitempty"`
-		Research      string   `json:"research"`
+		Research      string   `json:"research,omitempty"`
 		Links         []Link   `json:"links,omitempty"`
-		ShortDesc     string   `json:"short_desc"`
-		Audit         string   `json:"audit"`
-		AuditReport   string   `json:"audit_report"`
-		Tags          []string `json:"tags"`
-		Code          string   `json:"code"`
-		Ticker        string   `json:"ticker"`
-		ExplorerEth   string   `json:"explorer-ETH"`
-		Address       string   `json:"address"`
-		Twitter       string   `json:"twitter"`
-		CoinMarketcap string   `json:"coinmarketcap"`
-		DataSource    string   `json:"data_source"`
+		ShortDesc     *string  `json:"short_desc,omitempty"`
+		Audit         *string  `json:"audit,omitempty"`
+		AuditReport   *string  `json:"audit_report,omitempty"`
+		Tags          []string `json:"tags,omitempty"`
+		Code          *string  `json:"code,omitempty"`
+		Ticker        *string  `json:"ticker,omitempty"`
+		ExplorerEth   *string  `json:"explorer-ETH,omitempty"`
+		Address       *string  `json:"address,omitempty"`
+		Twitter       *string  `json:"twitter,omitempty"`
+		CoinMarketcap *string  `json:"coinmarketcap,omitempty"`
+		DataSource    *string  `json:"data_source,omitempty"`
 	}
 )
 

src/client/binance/dex/client.go

@@ -0,0 +1,34 @@
+package dex
+
+import (
+	"net/url"
+	"strconv"
+
+	"github.com/trustwallet/go-libs/client"
+)
+
+type Client struct {
+	req client.Request
+}
+
+func NewClient(url string, errorHandler client.HttpErrorHandler) *Client {
+	return &Client{req: client.InitClient(url, errorHandler)}
+}
+
+func (c *Client) GetMarketPairs(limit int) (pairs []MarketPair, err error) {
+	params := url.Values{
+		"limit": {strconv.Itoa(limit)},
+	}
+	err = c.req.Get(&pairs, "/v1/markets", params)
+
+	return pairs, err
+}
+
+func (c *Client) GetTokensList(limit int) (tokens []Token, err error) {
+	params := url.Values{
+		"limit": {strconv.Itoa(limit)},
+	}
+	err = c.req.Get(&tokens, "/v1/tokens", params)
+
+	return tokens, err
+}

src/client/binance/dex/model.go

@@ -0,0 +1,16 @@
+package dex
+
+type (
+	MarketPair struct {
+		BaseAssetSymbol  string `json:"base_asset_symbol"`
+		LotSize          string `json:"lot_size"`
+		QuoteAssetSymbol string `json:"quote_asset_symbol"`
+		TickSize         string `json:"tick_size"`
+	}
+
+	Token struct {
+		Symbol         string `json:"symbol"`
+		Name           string `json:"name"`
+		OriginalSymbol string `json:"original_symbol"`
+	}
+)

src/client/binance/explorer/client.go

@@ -0,0 +1,26 @@
+package explorer
+
+import (
+	"net/url"
+	"strconv"
+
+	"github.com/trustwallet/go-libs/client"
+)
+
+type Client struct {
+	req client.Request
+}
+
+func NewClient(url string, errorHandler client.HttpErrorHandler) *Client {
+	return &Client{req: client.InitClient(url, errorHandler)}
+}
+
+func (c *Client) GetBep2Assets(page, rows int) (assets Bep2Assets, err error) {
+	params := url.Values{
+		"page": {strconv.Itoa(page)},
+		"rows": {strconv.Itoa(rows)},
+	}
+	err = c.req.Get(&assets, "/api/v1/assets", params)
+
+	return assets, err
+}

src/client/binance/explorer/model.go

@@ -0,0 +1,15 @@
+package explorer
+
+type (
+	Bep2Asset struct {
+		Asset       string `json:"asset"`
+		Name        string `json:"name"`
+		AssetImg    string `json:"assetImg"`
+		MappedAsset string `json:"mappedAsset"`
+		Decimals    int    `json:"decimals"`
+	}
+
+	Bep2Assets struct {
+		AssetInfoList []Bep2Asset `json:"assetInfoList"`
+	}
+)