From 8d9fd546c935862c454c1d293b82302d3e9e877e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Men=C3=A9ndez?= Date: Wed, 6 Sep 2023 10:43:58 +0200 Subject: [PATCH] feature: support for multiple web3 providers (#72) Explained in #58 Changes: * Update database schema and API endpoints to include the id of the chain of each token * Delete chain id from service matadata on the database and remove the checks about the chain id * Update the scanner and token related API endpoints to use the web3 provider associated to each token * Handle a list of supported web3 providers as cmd/census3 entrypoint flag * New endpoint (GET /api/info) to get api related information such as supported chain ids * Include support for list of web3 providers on env file and docker-compose.yml --- .env | 3 +- api/README.md | 21 ++++++++++ api/api.go | 77 +++++++++++++--------------------- api/censuses.go | 5 --- api/errors.go | 10 +++++ api/tokens.go | 16 ++++++- api/types.go | 10 +++-- cmd/census3/main.go | 23 ++++++---- db/migrations/0001_census3.sql | 6 +-- db/queries/metadata.sql | 12 ------ db/queries/tokens.sql | 5 ++- db/sqlc/holders.sql.go | 6 ++- db/sqlc/metadata.sql.go | 37 ---------------- db/sqlc/models.go | 5 +-- db/sqlc/tokens.sql.go | 26 ++++++++---- docker-compose.yml | 2 +- service/helper_test.go | 3 +- service/holder_scanner_test.go | 54 ++++++++++++++---------- service/holders_scanner.go | 29 +++++++++---- state/holders.go | 4 +- state/holders_test.go | 22 +++++----- state/providers.go | 33 +++++++++++++++ state/web3.go | 3 +- state/web3_test.go | 4 +- 24 files changed, 234 insertions(+), 182 deletions(-) delete mode 100644 db/queries/metadata.sql delete mode 100644 db/sqlc/metadata.sql.go create mode 100644 state/providers.go diff --git a/.env b/.env index c79e581c..13f40bad 100644 --- a/.env +++ b/.env @@ -1,6 +1,5 @@ # A web3 endpoint provider -# WEB3_PROVIDER="https://mainnet.infura.io/v3/..." -WEB3_PROVIDER="https://web3.dappnode.net" +WEB3_PROVIDERS=https://rpc-endoint.example1.com,https://rpc-endoint.example2.com # Internal port for the service (80 and 443 are used by traefik) PORT=7788 diff --git a/api/README.md b/api/README.md index a2e98bb1..99707d86 100644 --- a/api/README.md +++ b/api/README.md @@ -1,10 +1,31 @@ # API endpoints Endpoints: + - [API info](#api-info) - [Tokens](#tokens) - [Strategies](#strategies) - [Censuses](#censuses) +## API Info + +### GET `/info` + +Show information about the API service. + +- 📥 response: + +```json +{ + "chainIDs": [1, 5] +} +``` + +- ⚠️ errors: + +| HTTP Status | Message | Internal error | +|:---:|:---|:---:| +| 500 | `error encoding API info` | 5023 | + ## Tokens ### GET `/token` diff --git a/api/api.go b/api/api.go index 637f5bbc..88e25ee6 100644 --- a/api/api.go +++ b/api/api.go @@ -1,13 +1,9 @@ package api import ( - "context" "database/sql" - "errors" - "fmt" - "time" + "encoding/json" - "github.com/ethereum/go-ethereum/ethclient" "github.com/vocdoni/census3/census" queries "github.com/vocdoni/census3/db/sqlc" "go.vocdoni.io/dvote/httprouter" @@ -16,37 +12,34 @@ import ( ) type Census3APIConf struct { - Hostname string - Port int - DataDir string - Web3URI string - GroupKey string + Hostname string + Port int + DataDir string + GroupKey string + Web3Providers map[int64]string } type census3API struct { conf Census3APIConf - web3 string db *sql.DB sqlc *queries.Queries endpoint *api.API censusDB *census.CensusDB + w3p map[int64]string } func Init(db *sql.DB, q *queries.Queries, conf Census3APIConf) error { newAPI := &census3API{ conf: conf, - web3: conf.Web3URI, db: db, sqlc: q, + w3p: conf.Web3Providers, } // get the current chainID - chainID, err := newAPI.setupChainID() - if err != nil { - log.Fatal(err) - } - log.Infow("starting API", "chainID", chainID, "web3", conf.Web3URI) + log.Infow("starting API", "chainID-web3Providers", conf.Web3Providers) // create a new http router with the hostname and port provided in the conf + var err error r := httprouter.HTTProuter{} if err = r.Init(conf.Hostname, conf.Port); err != nil { return err @@ -60,6 +53,9 @@ func Init(db *sql.DB, q *queries.Queries, conf Census3APIConf) error { return err } // init handlers + if err := newAPI.initAPIHandlers(); err != nil { + return err + } if err := newAPI.initTokenHandlers(); err != nil { return err } @@ -76,38 +72,23 @@ func Init(db *sql.DB, q *queries.Queries, conf Census3APIConf) error { return nil } -// setup function gets the chainID from the web3 uri and checks if it is -// registered in the database. If it is registered, the function compares both -// values and panics if they are not the same. If it is not registered, the -// function stores it. -func (capi *census3API) setupChainID() (int64, error) { - web3client, err := ethclient.Dial(capi.web3) - if err != nil { - return -1, fmt.Errorf("error dialing to the web3 endpoint: %w", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - // get the chainID from the web3 endpoint - chainID, err := web3client.ChainID(ctx) - if err != nil { - return -1, fmt.Errorf("error getting the chainID from the web3 endpoint: %w", err) +func (capi *census3API) initAPIHandlers() error { + return capi.endpoint.RegisterMethod("/info", "GET", + api.MethodAccessTypePublic, capi.getAPIInfo) +} + +func (capi *census3API) getAPIInfo(msg *api.APIdata, ctx *httprouter.HTTPContext) error { + chainIDs := []int64{} + for chainID := range capi.w3p { + chainIDs = append(chainIDs, chainID) } - // get the current chainID from the database - currentChainID, err := capi.sqlc.ChainID(ctx) + + info := map[string]any{"chainIDs": chainIDs} + res, err := json.Marshal(info) if err != nil { - // if it not exists register the value received from the web3 endpoint - if errors.Is(err, sql.ErrNoRows) { - _, err := capi.sqlc.SetChainID(ctx, chainID.Int64()) - if err != nil { - return -1, fmt.Errorf("error setting the chainID in the database: %w", err) - } - return chainID.Int64(), nil - } - return -1, fmt.Errorf("error getting chainID from the database: %w", err) - } - // compare both values - if currentChainID != chainID.Int64() { - return -1, fmt.Errorf("received chainID is not the same that registered one: %w", err) + log.Errorw(err, "error encoding api info") + return ErrEncodeAPIInfo } - return currentChainID, nil + + return ctx.Send(res, api.HTTPstatusOK) } diff --git a/api/censuses.go b/api/censuses.go index 190b74d5..c7c49ec3 100644 --- a/api/censuses.go +++ b/api/censuses.go @@ -57,10 +57,6 @@ func (capi *census3API) getCensus(msg *api.APIdata, ctx *httprouter.HTTPContext) } return ErrCantGetCensus } - chainID, err := qtx.ChainID(internalCtx) - if err != nil { - return ErrCantGetCensus - } res, err := json.Marshal(GetCensusResponse{ CensusID: uint64(censusID), StrategyID: uint64(currentCensus.StrategyID), @@ -68,7 +64,6 @@ func (capi *census3API) getCensus(msg *api.APIdata, ctx *httprouter.HTTPContext) URI: "ipfs://" + currentCensus.Uri.String, Size: int32(currentCensus.Size), Weight: new(big.Int).SetBytes(currentCensus.Weight).String(), - ChainID: uint64(chainID), Anonymous: currentCensus.CensusType == int64(census.AnonymousCensusType), }) if err != nil { diff --git a/api/errors.go b/api/errors.go index adbf0b5e..060344d1 100644 --- a/api/errors.go +++ b/api/errors.go @@ -58,6 +58,11 @@ var ( HTTPstatus: http.StatusConflict, Err: fmt.Errorf("token already created"), } + ErrChainIDNotSupported = apirest.APIerror{ + Code: 4013, + HTTPstatus: apirest.HTTPstatusBadRequest, + Err: fmt.Errorf("chain ID provided not supported"), + } ErrCantCreateToken = apirest.APIerror{ Code: 5000, HTTPstatus: apirest.HTTPstatusInternalErr, @@ -168,4 +173,9 @@ var ( HTTPstatus: apirest.HTTPstatusInternalErr, Err: fmt.Errorf("error getting last block number from web3 endpoint"), } + ErrEncodeAPIInfo = apirest.APIerror{ + Code: 5023, + HTTPstatus: apirest.HTTPstatusInternalErr, + Err: fmt.Errorf("error encoding API info"), + } ) diff --git a/api/tokens.go b/api/tokens.go index e44b1336..78110c3b 100644 --- a/api/tokens.go +++ b/api/tokens.go @@ -92,7 +92,12 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex w3 := state.Web3{} internalCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - if err := w3.Init(internalCtx, capi.web3, addr, tokenType); err != nil { + // get correct web3 uri provider + w3uri, exists := capi.w3p[req.ChainID] + if !exists { + return ErrChainIDNotSupported.With("chain ID not supported") + } + if err := w3.Init(internalCtx, w3uri, addr, tokenType); err != nil { log.Errorw(ErrInitializingWeb3, err.Error()) return ErrInitializingWeb3 } @@ -139,6 +144,7 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex TypeID: int64(tokenType), Synced: false, Tag: *tag, + ChainID: req.ChainID, }) if err != nil { if strings.Contains(err.Error(), "UNIQUE constraint failed") { @@ -194,9 +200,14 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext) // calculate the current scan progress tokenProgress := uint64(100) if !tokenData.Synced { + // get correct web3 uri provider + w3uri, exists := capi.w3p[tokenData.ChainID] + if !exists { + return ErrChainIDNotSupported.With("chain ID not supported") + } // get last block of the network, if something fails return progress 0 w3 := state.Web3{} - if err := w3.Init(internalCtx, capi.web3, address, state.TokenType(tokenData.TypeID)); err != nil { + if err := w3.Init(internalCtx, w3uri, address, state.TokenType(tokenData.TypeID)); err != nil { return ErrInitializingWeb3.WithErr(err) } // fetch the last block header and calculate progress @@ -232,6 +243,7 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext) // TODO: Only for the MVP, consider to remove it Tag: tokenData.Tag.String, DefaultStrategy: defaultStrategyID, + ChainID: tokenData.ChainID, } if tokenData.CreationBlock.Valid { tokenResponse.StartBlock = uint64(tokenData.CreationBlock.Int32) diff --git a/api/types.go b/api/types.go index 981fa38c..89a27b1f 100644 --- a/api/types.go +++ b/api/types.go @@ -1,9 +1,10 @@ package api type CreateTokenRequest struct { - ID string `json:"id"` - Type string `json:"type"` - Tag string `json:"tag"` + ID string `json:"id"` + Type string `json:"type"` + Tag string `json:"tag"` + ChainID int64 `json:"chainID"` } type GetTokenStatusResponse struct { @@ -24,6 +25,7 @@ type GetTokenResponse struct { Size uint32 `json:"size"` DefaultStrategy uint64 `json:"defaultStrategy,omitempty"` Tag string `json:"tag,omitempty"` + ChainID int64 `json:"chainID"` } type GetTokensItem struct { @@ -33,6 +35,7 @@ type GetTokensItem struct { Name string `json:"name"` Symbol string `json:"symbol"` Tag string `json:"tag,omitempty"` + ChainID int `json:"chainID"` } type GetTokensResponse struct { @@ -64,7 +67,6 @@ type GetCensusResponse struct { URI string `json:"uri"` Size int32 `json:"size"` Weight string `json:"weight"` - ChainID uint64 `json:"chainId"` Anonymous bool `json:"anonymous"` } diff --git a/cmd/census3/main.go b/cmd/census3/main.go index 78ae3a9c..432bab31 100644 --- a/cmd/census3/main.go +++ b/cmd/census3/main.go @@ -4,6 +4,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "time" @@ -11,6 +12,7 @@ import ( "github.com/vocdoni/census3/api" "github.com/vocdoni/census3/db" "github.com/vocdoni/census3/service" + "github.com/vocdoni/census3/state" "go.vocdoni.io/dvote/log" ) @@ -20,11 +22,11 @@ func main() { panic(err) } home += "/.census3" - url := flag.String("url", "", "ethereum web3 url") dataDir := flag.String("dataDir", home, "data directory for persistent storage") logLevel := flag.String("logLevel", "info", "log level (debug, info, warn, error)") port := flag.Int("port", 7788, "HTTP port for the API") connectKey := flag.String("connectKey", "", "connect group key for IPFS connect") + listOfWeb3Providers := flag.String("web3Providers", "", "the list of URL's of available web3 providers (separated with commas)") flag.Parse() log.Init(*logLevel, "stdout", nil) @@ -33,19 +35,26 @@ func main() { log.Fatal(err) } + web3Providers := strings.Split(*listOfWeb3Providers, ",") + w3p, err := state.CheckWeb3Providers(web3Providers) + if err != nil { + log.Fatal(err) + } + log.Info(w3p) + // Start the holder scanner - hc, err := service.NewHoldersScanner(db, q, *url) + hc, err := service.NewHoldersScanner(db, q, w3p) if err != nil { log.Fatal(err) } // Start the API err = api.Init(db, q, api.Census3APIConf{ - Hostname: "0.0.0.0", - Port: *port, - DataDir: *dataDir, - Web3URI: *url, - GroupKey: *connectKey, + Hostname: "0.0.0.0", + Port: *port, + DataDir: *dataDir, + Web3Providers: w3p, + GroupKey: *connectKey, }) if err != nil { log.Fatal(err) diff --git a/db/migrations/0001_census3.sql b/db/migrations/0001_census3.sql index 663b4d99..525f6501 100644 --- a/db/migrations/0001_census3.sql +++ b/db/migrations/0001_census3.sql @@ -1,8 +1,4 @@ -- +goose Up -CREATE TABLE metadata ( - chainID INTEGER PRIMARY KEY -); - CREATE TABLE strategies ( id INTEGER PRIMARY KEY, predicate TEXT NOT NULL @@ -30,6 +26,8 @@ CREATE TABLE tokens ( type_id INTEGER NOT NULL, synced BOOLEAN NOT NULL, tag TEXT, + chain_id INTEGER NOT NULL, + UNIQUE (id, chain_id), FOREIGN KEY (type_id) REFERENCES token_types(id) ON DELETE CASCADE ); CREATE INDEX idx_tokens_type_id ON tokens(type_id); diff --git a/db/queries/metadata.sql b/db/queries/metadata.sql deleted file mode 100644 index 94067e8d..00000000 --- a/db/queries/metadata.sql +++ /dev/null @@ -1,12 +0,0 @@ --- name: ChainID :one -SELECT chainID -FROM metadata -LIMIT 1; - --- name: SetChainID :execresult -INSERT INTO metadata ( - chainID -) -VALUES ( - ? -); \ No newline at end of file diff --git a/db/queries/tokens.sql b/db/queries/tokens.sql index 71f8cce2..b21a3b21 100644 --- a/db/queries/tokens.sql +++ b/db/queries/tokens.sql @@ -38,10 +38,11 @@ INSERT INTO tokens ( creation_block, type_id, synced, - tag + tag, + chain_id ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ); -- name: UpdateToken :execresult diff --git a/db/sqlc/holders.sql.go b/db/sqlc/holders.sql.go index 17fbf7f7..1159e81f 100644 --- a/db/sqlc/holders.sql.go +++ b/db/sqlc/holders.sql.go @@ -373,7 +373,7 @@ func (q *Queries) TokenHoldersByTokenIDAndMinBalance(ctx context.Context, arg To } const tokensByHolderID = `-- name: TokensByHolderID :many -SELECT tokens.id, tokens.name, tokens.symbol, tokens.decimals, tokens.total_supply, tokens.creation_block, tokens.type_id, tokens.synced, tokens.tag +SELECT tokens.id, tokens.name, tokens.symbol, tokens.decimals, tokens.total_supply, tokens.creation_block, tokens.type_id, tokens.synced, tokens.tag, tokens.chain_id FROM Tokens JOIN token_holders ON tokens.id = token_holders.token_id WHERE token_holders.holder_id = ? @@ -398,6 +398,7 @@ func (q *Queries) TokensByHolderID(ctx context.Context, holderID []byte) ([]Toke &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ); err != nil { return nil, err } @@ -413,7 +414,7 @@ func (q *Queries) TokensByHolderID(ctx context.Context, holderID []byte) ([]Toke } const tokensByHolderIDAndBlockID = `-- name: TokensByHolderIDAndBlockID :many -SELECT tokens.id, tokens.name, tokens.symbol, tokens.decimals, tokens.total_supply, tokens.creation_block, tokens.type_id, tokens.synced, tokens.tag +SELECT tokens.id, tokens.name, tokens.symbol, tokens.decimals, tokens.total_supply, tokens.creation_block, tokens.type_id, tokens.synced, tokens.tag, tokens.chain_id FROM Tokens JOIN token_holders ON tokens.id = token_holders.token_id WHERE token_holders.holder_id = ? AND token_holders.block_id = ? @@ -443,6 +444,7 @@ func (q *Queries) TokensByHolderIDAndBlockID(ctx context.Context, arg TokensByHo &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ); err != nil { return nil, err } diff --git a/db/sqlc/metadata.sql.go b/db/sqlc/metadata.sql.go deleted file mode 100644 index e37bbd76..00000000 --- a/db/sqlc/metadata.sql.go +++ /dev/null @@ -1,37 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.20.0 -// source: metadata.sql - -package queries - -import ( - "context" - "database/sql" -) - -const chainID = `-- name: ChainID :one -SELECT chainID -FROM metadata -LIMIT 1 -` - -func (q *Queries) ChainID(ctx context.Context) (int64, error) { - row := q.db.QueryRowContext(ctx, chainID) - var chainid int64 - err := row.Scan(&chainid) - return chainid, err -} - -const setChainID = `-- name: SetChainID :execresult -INSERT INTO metadata ( - chainID -) -VALUES ( - ? -) -` - -func (q *Queries) SetChainID(ctx context.Context, chainid int64) (sql.Result, error) { - return q.db.ExecContext(ctx, setChainID, chainid) -} diff --git a/db/sqlc/models.go b/db/sqlc/models.go index ccd5b33b..f478fd32 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -35,10 +35,6 @@ type Holder struct { ID annotations.Address } -type Metadatum struct { - Chainid int64 -} - type Strategy struct { ID int64 Predicate string @@ -61,6 +57,7 @@ type Token struct { TypeID int64 Synced bool Tag sql.NullString + ChainID int64 } type TokenHolder struct { diff --git a/db/sqlc/tokens.sql.go b/db/sqlc/tokens.sql.go index 18647ecc..8d6e5102 100644 --- a/db/sqlc/tokens.sql.go +++ b/db/sqlc/tokens.sql.go @@ -22,10 +22,11 @@ INSERT INTO tokens ( creation_block, type_id, synced, - tag + tag, + chain_id ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ` @@ -39,6 +40,7 @@ type CreateTokenParams struct { TypeID int64 Synced bool Tag sql.NullString + ChainID int64 } func (q *Queries) CreateToken(ctx context.Context, arg CreateTokenParams) (sql.Result, error) { @@ -52,6 +54,7 @@ func (q *Queries) CreateToken(ctx context.Context, arg CreateTokenParams) (sql.R arg.TypeID, arg.Synced, arg.Tag, + arg.ChainID, ) } @@ -79,7 +82,7 @@ func (q *Queries) ExistsToken(ctx context.Context, id annotations.Address) (bool } const listTokens = `-- name: ListTokens :many -SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag FROM tokens +SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag, chain_id FROM tokens ORDER BY type_id, name ` @@ -102,6 +105,7 @@ func (q *Queries) ListTokens(ctx context.Context) ([]Token, error) { &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ); err != nil { return nil, err } @@ -117,7 +121,7 @@ func (q *Queries) ListTokens(ctx context.Context) ([]Token, error) { } const tokenByID = `-- name: TokenByID :one -SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag FROM tokens +SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag, chain_id FROM tokens WHERE id = ? LIMIT 1 ` @@ -135,12 +139,13 @@ func (q *Queries) TokenByID(ctx context.Context, id annotations.Address) (Token, &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ) return i, err } const tokenByName = `-- name: TokenByName :one -SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag FROM tokens +SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag, chain_id FROM tokens WHERE name = ? LIMIT 1 ` @@ -158,12 +163,13 @@ func (q *Queries) TokenByName(ctx context.Context, name sql.NullString) (Token, &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ) return i, err } const tokenBySymbol = `-- name: TokenBySymbol :one -SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag FROM tokens +SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag, chain_id FROM tokens WHERE symbol = ? LIMIT 1 ` @@ -181,12 +187,13 @@ func (q *Queries) TokenBySymbol(ctx context.Context, symbol sql.NullString) (Tok &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ) return i, err } const tokensByStrategyID = `-- name: TokensByStrategyID :many -SELECT t.id, t.name, t.symbol, t.decimals, t.total_supply, t.creation_block, t.type_id, t.synced, t.tag, st.strategy_id, st.token_id, st.min_balance, st.method_hash FROM tokens t +SELECT t.id, t.name, t.symbol, t.decimals, t.total_supply, t.creation_block, t.type_id, t.synced, t.tag, t.chain_id, st.strategy_id, st.token_id, st.min_balance, st.method_hash FROM tokens t JOIN strategy_tokens st ON st.token_id = t.id WHERE st.strategy_id = ? ORDER BY t.name @@ -202,6 +209,7 @@ type TokensByStrategyIDRow struct { TypeID int64 Synced bool Tag sql.NullString + ChainID int64 StrategyID int64 TokenID []byte MinBalance []byte @@ -227,6 +235,7 @@ func (q *Queries) TokensByStrategyID(ctx context.Context, strategyID int64) ([]T &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, &i.StrategyID, &i.TokenID, &i.MinBalance, @@ -246,7 +255,7 @@ func (q *Queries) TokensByStrategyID(ctx context.Context, strategyID int64) ([]T } const tokensByType = `-- name: TokensByType :many -SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag FROM tokens +SELECT id, name, symbol, decimals, total_supply, creation_block, type_id, synced, tag, chain_id FROM tokens WHERE type_id = ? ORDER BY name ` @@ -270,6 +279,7 @@ func (q *Queries) TokensByType(ctx context.Context, typeID int64) ([]Token, erro &i.TypeID, &i.Synced, &i.Tag, + &i.ChainID, ); err != nil { return nil, err } diff --git a/docker-compose.yml b/docker-compose.yml index 3728a37b..963af3a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: volumes: - census3:/app/data command: - - "--url=${WEB3_PROVIDER}" + - "--web3Providers=${WEB3_PROVIDERS}" - "--port=${PORT}" - "--logLevel=${LOGLEVEL}" - "--connectKey=${CONNECT_KEY}" diff --git a/service/helper_test.go b/service/helper_test.go index da391eb5..614bdc40 100644 --- a/service/helper_test.go +++ b/service/helper_test.go @@ -64,7 +64,7 @@ func (testdb *TestDB) Close(t *testing.T) { } func testTokenParams(id, name, symbol string, decimals, creationBlock, totalSupply, - typeID uint64, synced bool, + typeID uint64, synced bool, chainID int64, ) queries.CreateTokenParams { return queries.CreateTokenParams{ ID: common.HexToAddress(id).Bytes(), @@ -75,5 +75,6 @@ func testTokenParams(id, name, symbol string, decimals, creationBlock, totalSupp CreationBlock: sql.NullInt32{Int32: int32(creationBlock), Valid: creationBlock != 0}, TypeID: int64(typeID), Synced: synced, + ChainID: chainID, } } diff --git a/service/holder_scanner_test.go b/service/holder_scanner_test.go index b76944a9..1db1d242 100644 --- a/service/holder_scanner_test.go +++ b/service/holder_scanner_test.go @@ -22,7 +22,10 @@ func TestNewHolderScanner(t *testing.T) { testdb := StartTestDB(t) defer testdb.Close(t) - hs, err := NewHoldersScanner(testdb.db, testdb.queries, web3uri) + w3p, err := state.CheckWeb3Providers([]string{web3uri}) + c.Assert(err, qt.IsNil) + + hs, err := NewHoldersScanner(testdb.db, testdb.queries, w3p) c.Assert(err, qt.IsNil) c.Assert(hs.lastBlock, qt.Equals, uint64(0)) @@ -35,11 +38,11 @@ func TestNewHolderScanner(t *testing.T) { }) c.Assert(err, qt.IsNil) - hs, err = NewHoldersScanner(testdb.db, testdb.queries, web3uri) + hs, err = NewHoldersScanner(testdb.db, testdb.queries, w3p) c.Assert(err, qt.IsNil) c.Assert(hs.lastBlock, qt.Equals, uint64(1000)) - _, err = NewHoldersScanner(nil, nil, web3uri) + _, err = NewHoldersScanner(nil, nil, w3p) c.Assert(err, qt.IsNotNil) } @@ -47,12 +50,15 @@ func TestHolderScannerStart(t *testing.T) { c := qt.New(t) twg := sync.WaitGroup{} + w3p, err := state.CheckWeb3Providers([]string{web3uri}) + c.Assert(err, qt.IsNil) + ctx, cancel := context.WithCancel(context.Background()) testdb := StartTestDB(t) defer testdb.Close(t) twg.Add(1) - hs, err := NewHoldersScanner(testdb.db, testdb.queries, web3uri) + hs, err := NewHoldersScanner(testdb.db, testdb.queries, w3p) c.Assert(err, qt.IsNil) go func() { hs.Start(ctx) @@ -69,7 +75,10 @@ func Test_tokenAddresses(t *testing.T) { testdb := StartTestDB(t) defer testdb.Close(t) - hs, err := NewHoldersScanner(testdb.db, testdb.queries, web3uri) + w3p, err := state.CheckWeb3Providers([]string{web3uri}) + c.Assert(err, qt.IsNil) + + hs, err := NewHoldersScanner(testdb.db, testdb.queries, w3p) c.Assert(err, qt.IsNil) res, err := hs.tokenAddresses() @@ -80,7 +89,7 @@ func Test_tokenAddresses(t *testing.T) { defer cancel() _, err = testdb.queries.CreateToken(ctx, testTokenParams("0x1", "test0", "test0", MonkeysDecimals, 0, MonkeysTotalSupply.Uint64(), - uint64(state.CONTRACT_TYPE_ERC20), false)) + uint64(state.CONTRACT_TYPE_ERC20), false, 5)) c.Assert(err, qt.IsNil) res, err = hs.tokenAddresses() @@ -89,7 +98,7 @@ func Test_tokenAddresses(t *testing.T) { _, err = testdb.queries.CreateToken(ctx, testTokenParams("0x2", "test2", "test3", MonkeysDecimals, 10, MonkeysTotalSupply.Uint64(), - uint64(state.CONTRACT_TYPE_ERC20), false)) + uint64(state.CONTRACT_TYPE_ERC20), false, 5)) c.Assert(err, qt.IsNil) res, err = hs.tokenAddresses() @@ -103,16 +112,19 @@ func Test_saveHolders(t *testing.T) { testdb := StartTestDB(t) defer testdb.Close(t) - hs, err := NewHoldersScanner(testdb.db, testdb.queries, "http://google.com") + w3p, err := state.CheckWeb3Providers([]string{web3uri}) c.Assert(err, qt.IsNil) - th := new(state.TokenHolders).Init(MonkeysAddress, state.CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + hs, err := NewHoldersScanner(testdb.db, testdb.queries, w3p) + c.Assert(err, qt.IsNil) + + th := new(state.TokenHolders).Init(MonkeysAddress, state.CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) // no registered token c.Assert(hs.saveHolders(th), qt.ErrorIs, ErrTokenNotExists) _, err = testdb.queries.CreateToken(context.Background(), testTokenParams( MonkeysAddress.String(), MonkeysName, MonkeysSymbol, MonkeysDecimals, MonkeysCreationBlock, MonkeysTotalSupply.Uint64(), - uint64(state.CONTRACT_TYPE_ERC20), false)) + uint64(state.CONTRACT_TYPE_ERC20), false, 5)) c.Assert(err, qt.IsNil) // check no new holders c.Assert(hs.saveHolders(th), qt.IsNil) @@ -121,13 +133,7 @@ func Test_saveHolders(t *testing.T) { holderBalance := new(big.Int).SetUint64(12) th.Append(holderAddr, holderBalance) th.BlockDone(MonkeysCreationBlock) - // check wrong web3 - c.Assert(hs.saveHolders(th), qt.IsNotNil) - // check new block created - _, err = testdb.queries.BlockByID(context.Background(), int64(MonkeysCreationBlock)) - c.Assert(err, qt.ErrorIs, sql.ErrNoRows) - // check good web3 - hs.web3 = web3uri + // check web3 c.Assert(hs.saveHolders(th), qt.IsNil) // check new holders res, err := testdb.queries.TokenHolderByTokenIDAndHolderID(context.Background(), @@ -165,7 +171,10 @@ func Test_scanHolders(t *testing.T) { testdb := StartTestDB(t) defer testdb.Close(t) - hs, err := NewHoldersScanner(testdb.db, testdb.queries, web3uri) + w3p, err := state.CheckWeb3Providers([]string{web3uri}) + c.Assert(err, qt.IsNil) + + hs, err := NewHoldersScanner(testdb.db, testdb.queries, w3p) c.Assert(err, qt.IsNil) // token does not exists @@ -176,7 +185,7 @@ func Test_scanHolders(t *testing.T) { _, err = testdb.queries.CreateToken(context.Background(), testTokenParams( MonkeysAddress.String(), MonkeysName, MonkeysSymbol, MonkeysDecimals, MonkeysCreationBlock, 10, - uint64(state.CONTRACT_TYPE_ERC20), false)) + uint64(state.CONTRACT_TYPE_ERC20), false, 5)) c.Assert(err, qt.IsNil) // token exists and the scanner gets the holders ctx2, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -199,14 +208,17 @@ func Test_calcTokenCreationBlock(t *testing.T) { testdb := StartTestDB(t) defer testdb.Close(t) - hs, err := NewHoldersScanner(testdb.db, testdb.queries, web3uri) + w3p, err := state.CheckWeb3Providers([]string{web3uri}) + c.Assert(err, qt.IsNil) + + hs, err := NewHoldersScanner(testdb.db, testdb.queries, w3p) c.Assert(err, qt.IsNil) c.Assert(hs.calcTokenCreationBlock(context.Background(), MonkeysAddress), qt.IsNotNil) _, err = testdb.queries.CreateToken(context.Background(), testTokenParams( MonkeysAddress.String(), MonkeysName, MonkeysSymbol, MonkeysDecimals, MonkeysCreationBlock, MonkeysTotalSupply.Uint64(), - uint64(state.CONTRACT_TYPE_ERC20), false)) + uint64(state.CONTRACT_TYPE_ERC20), false, 5)) c.Assert(err, qt.IsNil) c.Assert(hs.calcTokenCreationBlock(context.Background(), MonkeysAddress), qt.IsNil) diff --git a/service/holders_scanner.go b/service/holders_scanner.go index d62a41e6..06a7619d 100644 --- a/service/holders_scanner.go +++ b/service/holders_scanner.go @@ -28,7 +28,7 @@ var ( // the tokens stored on the database (located on 'dataDir/dbFilename'). It // keeps the database updated scanning the network using the web3 endpoint. type HoldersScanner struct { - web3 string + w3p map[int64]string tokens map[common.Address]*state.TokenHolders mutex sync.RWMutex db *sql.DB @@ -39,14 +39,14 @@ type HoldersScanner struct { // NewHoldersScanner function creates a new HolderScanner using the dataDir path // and the web3 endpoint URI provided. It sets up a sqlite3 database instance // and gets the number of last block scanned from it. -func NewHoldersScanner(db *sql.DB, q *queries.Queries, w3uri string) (*HoldersScanner, error) { +func NewHoldersScanner(db *sql.DB, q *queries.Queries, w3p map[int64]string) (*HoldersScanner, error) { if db == nil || q == nil { return nil, ErrNoDB } // create an empty scanner s := HoldersScanner{ + w3p: w3p, tokens: make(map[common.Address]*state.TokenHolders), - web3: w3uri, db: db, sqlc: q, } @@ -178,9 +178,14 @@ func (s *HoldersScanner) saveHolders(th *state.TokenHolders) error { } return nil } + // get correct web3 uri provider + w3uri, exists := s.w3p[th.ChainID] + if !exists { + return fmt.Errorf("chain ID not supported") + } // init web3 contract state w3 := state.Web3{} - if err := w3.Init(ctx, s.web3, th.Address(), th.Type()); err != nil { + if err := w3.Init(ctx, w3uri, th.Address(), th.Type()); err != nil { return err } // get current block number timestamp and root hash, required parameters to @@ -312,7 +317,7 @@ func (s *HoldersScanner) scanHolders(ctx context.Context, addr common.Address) ( if blockNumber, err := s.sqlc.LastBlockByTokenID(ctx, addr.Bytes()); err == nil { tokenLastBlock = uint64(blockNumber) } - th = new(state.TokenHolders).Init(addr, ttype, tokenLastBlock) + th = new(state.TokenHolders).Init(addr, ttype, tokenLastBlock, tokenInfo.ChainID) s.tokens[addr] = th } s.mutex.RUnlock() @@ -322,9 +327,14 @@ func (s *HoldersScanner) scanHolders(ctx context.Context, addr common.Address) ( if s.lastBlock < th.LastBlock() { s.lastBlock = th.LastBlock() } + // get correct web3 uri provider + w3uri, exists := s.w3p[th.ChainID] + if !exists { + return false, fmt.Errorf("chain ID not supported") + } // init web3 contract state w3 := state.Web3{} - if err := w3.Init(ctx, s.web3, addr, th.Type()); err != nil { + if err := w3.Init(ctx, w3uri, addr, th.Type()); err != nil { return th.IsSynced(), err } // try to update the TokenHolders struct and the current scanner last block @@ -368,9 +378,14 @@ func (s *HoldersScanner) calcTokenCreationBlock(ctx context.Context, addr common return fmt.Errorf("error getting token from database: %w", err) } ttype := state.TokenType(tokenInfo.TypeID) + // get correct web3 uri provider + w3uri, exists := s.w3p[tokenInfo.ChainID] + if !exists { + return fmt.Errorf("chain ID not supported") + } // init web3 contract state w3 := state.Web3{} - if err := w3.Init(ctx, s.web3, addr, ttype); err != nil { + if err := w3.Init(ctx, w3uri, addr, ttype); err != nil { return fmt.Errorf("error intializing web3 client for this token: %w", err) } // get creation block of the current token contract diff --git a/state/holders.go b/state/holders.go index 0dc90929..b2caee5e 100644 --- a/state/holders.go +++ b/state/holders.go @@ -23,18 +23,20 @@ type TokenHolders struct { blocks sync.Map lastBlock atomic.Uint64 synced atomic.Bool + ChainID int64 } // Init function fills the given TokenHolders struct with the address and type // given, also checks the block number provided as done. It returns the // TokenHolders struct updated. -func (h *TokenHolders) Init(addr common.Address, ctype TokenType, block uint64) *TokenHolders { +func (h *TokenHolders) Init(addr common.Address, ctype TokenType, block uint64, chainID int64) *TokenHolders { h.address = addr h.ctype = ctype h.holders = sync.Map{} h.blocks = sync.Map{} h.lastBlock.Store(block) h.synced.Store(false) + h.ChainID = chainID return h } diff --git a/state/holders_test.go b/state/holders_test.go index 6fd9831c..b9ad4cab 100644 --- a/state/holders_test.go +++ b/state/holders_test.go @@ -10,7 +10,7 @@ import ( func TestTokenHoldersInit(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 0) c.Assert(th.address.String(), qt.Equals, MonkeysAddress.String()) c.Assert(th.ctype, qt.Equals, CONTRACT_TYPE_ERC20) c.Assert(th.lastBlock.Load(), qt.Equals, MonkeysCreationBlock) @@ -19,7 +19,7 @@ func TestTokenHoldersInit(t *testing.T) { func TestHolders(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) c.Assert(th.address.String(), qt.Equals, MonkeysAddress.String()) c.Assert(th.ctype, qt.Equals, CONTRACT_TYPE_ERC20) c.Assert(th.lastBlock.Load(), qt.Equals, MonkeysCreationBlock) @@ -28,7 +28,7 @@ func TestHolders(t *testing.T) { func TestAppend(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) holderAddr := common.HexToAddress("0xe54d702f98E312aBA4318E3c6BDba98ab5e11012") holderBalance := new(big.Int).SetUint64(16000000000000000000) @@ -42,7 +42,7 @@ func TestAppend(t *testing.T) { func TestExists(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) holderAddr := common.HexToAddress("0xe54d702f98E312aBA4318E3c6BDba98ab5e11012") holderBalance := new(big.Int).SetUint64(16000000000000000000) @@ -53,7 +53,7 @@ func TestExists(t *testing.T) { func TestDel(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) holderAddr := common.HexToAddress("0xe54d702f98E312aBA4318E3c6BDba98ab5e11012") holderBalance := new(big.Int).SetUint64(16000000000000000000) @@ -70,7 +70,7 @@ func TestDel(t *testing.T) { func TestFlushHolders(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) holderAddr := common.HexToAddress("0xe54d702f98E312aBA4318E3c6BDba98ab5e11012") holderBalance := new(big.Int).SetUint64(16000000000000000000) @@ -86,7 +86,7 @@ func TestFlushHolders(t *testing.T) { func TestBlockDone(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) _, exists := th.blocks.Load(MonkeysCreationBlock + 500) c.Assert(exists, qt.IsFalse) @@ -99,7 +99,7 @@ func TestBlockDone(t *testing.T) { func TestHasBlock(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) c.Assert(th.HasBlock(MonkeysCreationBlock), qt.IsFalse) th.BlockDone(MonkeysCreationBlock) @@ -108,7 +108,7 @@ func TestHasBlock(t *testing.T) { func TestLastBlock(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) c.Assert(th.LastBlock(), qt.Equals, MonkeysCreationBlock) th.BlockDone(MonkeysCreationBlock + 1) @@ -119,7 +119,7 @@ func TestLastBlock(t *testing.T) { func TestSynced(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) c.Assert(th.synced.Load(), qt.IsFalse) th.Synced() @@ -128,7 +128,7 @@ func TestSynced(t *testing.T) { func TestIsSynced(t *testing.T) { c := qt.New(t) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) c.Assert(th.IsSynced(), qt.IsFalse) th.Synced() diff --git a/state/providers.go b/state/providers.go new file mode 100644 index 00000000..4662c283 --- /dev/null +++ b/state/providers.go @@ -0,0 +1,33 @@ +package state + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/ethclient" +) + +func CheckWeb3Providers(providersURIs []string) (map[int64]string, error) { + if len(providersURIs) == 0 { + return nil, fmt.Errorf("no URIs provided") + } + + providers := make(map[int64]string) + for _, uri := range providersURIs { + cli, err := ethclient.Dial(uri) + if err != nil { + return nil, fmt.Errorf("error dialing web3 provider uri '%s': %w", uri, err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // get the chainID from the web3 endpoint + chainID, err := cli.ChainID(ctx) + if err != nil { + return nil, fmt.Errorf("error getting the chainID from the web3 provider '%s': %w", uri, err) + } + providers[chainID.Int64()] = uri + } + return providers, nil +} diff --git a/state/web3.go b/state/web3.go index f24d3799..157b0785 100644 --- a/state/web3.go +++ b/state/web3.go @@ -343,9 +343,10 @@ func (w *Web3) UpdateTokenHolders(ctx context.Context, th *TokenHolders) (uint64 default: log.Debugw("analyzing blocks", "address", th.Address().Hex(), - "type", th.Type(), + "type", th.Type().String(), "from", fromBlockNumber, "to", fromBlockNumber+blocks, + "chainID", th.ChainID, ) // get transfer logs for the following n blocks diff --git a/state/web3_test.go b/state/web3_test.go index 78504531..c526fc69 100644 --- a/state/web3_test.go +++ b/state/web3_test.go @@ -218,7 +218,7 @@ func TestUpdateTokenHolders(t *testing.T) { c := qt.New(t) th := new(TokenHolders) - th = th.Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th = th.Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) w3 := Web3{} ctx, cancel := context.WithTimeout(context.Background(), 3000*time.Second) @@ -288,7 +288,7 @@ func Test_commitTokenHolders(t *testing.T) { c.Assert(w.Init(ctx, web3URI, MonkeysAddress, CONTRACT_TYPE_ERC20), qt.IsNil) hc := HoldersCandidates(MonkeysHolders) - th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock) + th := new(TokenHolders).Init(MonkeysAddress, CONTRACT_TYPE_ERC20, MonkeysCreationBlock, 5) c.Assert(w.commitTokenHolders(th, hc, MonkeysCreationBlock+1000), qt.IsNil) c.Assert(th.LastBlock(), qt.Equals, MonkeysCreationBlock)