feature: support for multiple web3 providers (#72)

Explained in #58

Changes:
* Update database schema and API endpoints to include the id of the chain of each token
* Delete chain id from service matadata on the database and remove the checks about the chain id
* Update the scanner and token related API endpoints to use the web3 provider associated to each token
* Handle a list of supported web3 providers as cmd/census3 entrypoint flag
* New endpoint (GET /api/info) to get api related information such as supported chain ids
* Include support for list of web3 providers on env file and docker-compose.yml

.env

@@ -1,6 +1,5 @@
 # A web3 endpoint provider
-# WEB3_PROVIDER="https://mainnet.infura.io/v3/..."
-WEB3_PROVIDER="https://web3.dappnode.net"
+WEB3_PROVIDERS=https://rpc-endoint.example1.com,https://rpc-endoint.example2.com
 
 # Internal port for the service (80 and 443 are used by traefik)
 PORT=7788

api/README.md

@@ -1,10 +1,31 @@
 # API endpoints
 
 Endpoints:
+ - [API info](#api-info)
  - [Tokens](#tokens)
  - [Strategies](#strategies)
  - [Censuses](#censuses)
 
+## API Info
+
+### GET `/info`
+
+Show information about the API service.
+
+- 📥 response:
+
+```json
+{
+    "chainIDs": [1, 5]
+}
+```
+
+- ⚠️ errors:
+
+| HTTP Status | Message | Internal error |
+|:---:|:---|:---:|
+| 500 | `error encoding API info` | 5023 | 
+
 ## Tokens
 
 ### GET `/token`

api/api.go

@@ -1,13 +1,9 @@
 package api
 
 import (
-	"context"
 	"database/sql"
-	"errors"
-	"fmt"
-	"time"
+	"encoding/json"
 
-	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/vocdoni/census3/census"
 	queries "github.com/vocdoni/census3/db/sqlc"
 	"go.vocdoni.io/dvote/httprouter"
@@ -16,37 +12,34 @@ import (
 )
 
 type Census3APIConf struct {
-	Hostname string
-	Port     int
-	DataDir  string
-	Web3URI  string
-	GroupKey string
+	Hostname      string
+	Port          int
+	DataDir       string
+	GroupKey      string
+	Web3Providers map[int64]string
 }
 
 type census3API struct {
 	conf     Census3APIConf
-	web3     string
 	db       *sql.DB
 	sqlc     *queries.Queries
 	endpoint *api.API
 	censusDB *census.CensusDB
+	w3p      map[int64]string
 }
 
 func Init(db *sql.DB, q *queries.Queries, conf Census3APIConf) error {
 	newAPI := &census3API{
 		conf: conf,
-		web3: conf.Web3URI,
 		db:   db,
 		sqlc: q,
+		w3p:  conf.Web3Providers,
 	}
 	// get the current chainID
-	chainID, err := newAPI.setupChainID()
-	if err != nil {
-		log.Fatal(err)
-	}
-	log.Infow("starting API", "chainID", chainID, "web3", conf.Web3URI)
+	log.Infow("starting API", "chainID-web3Providers", conf.Web3Providers)
 
 	// create a new http router with the hostname and port provided in the conf
+	var err error
 	r := httprouter.HTTProuter{}
 	if err = r.Init(conf.Hostname, conf.Port); err != nil {
 		return err
@@ -60,6 +53,9 @@ func Init(db *sql.DB, q *queries.Queries, conf Census3APIConf) error {
 		return err
 	}
 	// init handlers
+	if err := newAPI.initAPIHandlers(); err != nil {
+		return err
+	}
 	if err := newAPI.initTokenHandlers(); err != nil {
 		return err
 	}
@@ -76,38 +72,23 @@ func Init(db *sql.DB, q *queries.Queries, conf Census3APIConf) error {
 	return nil
 }
 
-// setup function gets the chainID from the web3 uri and checks if it is
-// registered in the database. If it is registered, the function compares both
-// values and panics if they are not the same. If it is not registered, the
-// function stores it.
-func (capi *census3API) setupChainID() (int64, error) {
-	web3client, err := ethclient.Dial(capi.web3)
-	if err != nil {
-		return -1, fmt.Errorf("error dialing to the web3 endpoint: %w", err)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	// get the chainID from the web3 endpoint
-	chainID, err := web3client.ChainID(ctx)
-	if err != nil {
-		return -1, fmt.Errorf("error getting the chainID from the web3 endpoint: %w", err)
+func (capi *census3API) initAPIHandlers() error {
+	return capi.endpoint.RegisterMethod("/info", "GET",
+		api.MethodAccessTypePublic, capi.getAPIInfo)
+}
+
+func (capi *census3API) getAPIInfo(msg *api.APIdata, ctx *httprouter.HTTPContext) error {
+	chainIDs := []int64{}
+	for chainID := range capi.w3p {
+		chainIDs = append(chainIDs, chainID)
 	}
-	// get the current chainID from the database
-	currentChainID, err := capi.sqlc.ChainID(ctx)
+
+	info := map[string]any{"chainIDs": chainIDs}
+	res, err := json.Marshal(info)
 	if err != nil {
-		// if it not exists register the value received from the web3 endpoint
-		if errors.Is(err, sql.ErrNoRows) {
-			_, err := capi.sqlc.SetChainID(ctx, chainID.Int64())
-			if err != nil {
-				return -1, fmt.Errorf("error setting the chainID in the database: %w", err)
-			}
-			return chainID.Int64(), nil
-		}
-		return -1, fmt.Errorf("error getting chainID from the database: %w", err)
-	}
-	// compare both values
-	if currentChainID != chainID.Int64() {
-		return -1, fmt.Errorf("received chainID is not the same that registered one: %w", err)
+		log.Errorw(err, "error encoding api info")
+		return ErrEncodeAPIInfo
 	}
-	return currentChainID, nil
+
+	return ctx.Send(res, api.HTTPstatusOK)
 }

api/censuses.go

@@ -57,18 +57,13 @@ func (capi *census3API) getCensus(msg *api.APIdata, ctx *httprouter.HTTPContext)
 		}
 		return ErrCantGetCensus
 	}
-	chainID, err := qtx.ChainID(internalCtx)
-	if err != nil {
-		return ErrCantGetCensus
-	}
 	res, err := json.Marshal(GetCensusResponse{
 		CensusID:   uint64(censusID),
 		StrategyID: uint64(currentCensus.StrategyID),
 		MerkleRoot: common.Bytes2Hex(currentCensus.MerkleRoot),
 		URI:        "ipfs://" + currentCensus.Uri.String,
 		Size:       int32(currentCensus.Size),
 		Weight:     new(big.Int).SetBytes(currentCensus.Weight).String(),
-		ChainID:    uint64(chainID),
 		Anonymous:  currentCensus.CensusType == int64(census.AnonymousCensusType),
 	})
 	if err != nil {

api/errors.go

@@ -58,6 +58,11 @@ var (
 		HTTPstatus: http.StatusConflict,
 		Err:        fmt.Errorf("token already created"),
 	}
+	ErrChainIDNotSupported = apirest.APIerror{
+		Code:       4013,
+		HTTPstatus: apirest.HTTPstatusBadRequest,
+		Err:        fmt.Errorf("chain ID provided not supported"),
+	}
 	ErrCantCreateToken = apirest.APIerror{
 		Code:       5000,
 		HTTPstatus: apirest.HTTPstatusInternalErr,
@@ -168,4 +173,9 @@ var (
 		HTTPstatus: apirest.HTTPstatusInternalErr,
 		Err:        fmt.Errorf("error getting last block number from web3 endpoint"),
 	}
+	ErrEncodeAPIInfo = apirest.APIerror{
+		Code:       5023,
+		HTTPstatus: apirest.HTTPstatusInternalErr,
+		Err:        fmt.Errorf("error encoding API info"),
+	}
 )

api/tokens.go

@@ -92,7 +92,12 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
 	w3 := state.Web3{}
 	internalCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	if err := w3.Init(internalCtx, capi.web3, addr, tokenType); err != nil {
+	// get correct web3 uri provider
+	w3uri, exists := capi.w3p[req.ChainID]
+	if !exists {
+		return ErrChainIDNotSupported.With("chain ID not supported")
+	}
+	if err := w3.Init(internalCtx, w3uri, addr, tokenType); err != nil {
 		log.Errorw(ErrInitializingWeb3, err.Error())
 		return ErrInitializingWeb3
 	}
@@ -139,6 +144,7 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
 		TypeID:        int64(tokenType),
 		Synced:        false,
 		Tag:           *tag,
+		ChainID:       req.ChainID,
 	})
 	if err != nil {
 		if strings.Contains(err.Error(), "UNIQUE constraint failed") {
@@ -194,9 +200,14 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
 	// calculate the current scan progress
 	tokenProgress := uint64(100)
 	if !tokenData.Synced {
+		// get correct web3 uri provider
+		w3uri, exists := capi.w3p[tokenData.ChainID]
+		if !exists {
+			return ErrChainIDNotSupported.With("chain ID not supported")
+		}
 		// get last block of the network, if something fails return progress 0
 		w3 := state.Web3{}
-		if err := w3.Init(internalCtx, capi.web3, address, state.TokenType(tokenData.TypeID)); err != nil {
+		if err := w3.Init(internalCtx, w3uri, address, state.TokenType(tokenData.TypeID)); err != nil {
 			return ErrInitializingWeb3.WithErr(err)
 		}
 		// fetch the last block header and calculate progress
@@ -232,6 +243,7 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
 		// TODO: Only for the MVP, consider to remove it
 		Tag:             tokenData.Tag.String,
 		DefaultStrategy: defaultStrategyID,
+		ChainID:         tokenData.ChainID,
 	}
 	if tokenData.CreationBlock.Valid {
 		tokenResponse.StartBlock = uint64(tokenData.CreationBlock.Int32)

api/types.go

@@ -1,9 +1,10 @@
 package api
 
 type CreateTokenRequest struct {
-	ID   string `json:"id"`
-	Type string `json:"type"`
-	Tag  string `json:"tag"`
+	ID      string `json:"id"`
+	Type    string `json:"type"`
+	Tag     string `json:"tag"`
+	ChainID int64  `json:"chainID"`
 }
 
 type GetTokenStatusResponse struct {
@@ -24,6 +25,7 @@ type GetTokenResponse struct {
 	Size            uint32                  `json:"size"`
 	DefaultStrategy uint64                  `json:"defaultStrategy,omitempty"`
 	Tag             string                  `json:"tag,omitempty"`
+	ChainID         int64                   `json:"chainID"`
 }
 
 type GetTokensItem struct {
@@ -33,6 +35,7 @@ type GetTokensItem struct {
 	Name       string `json:"name"`
 	Symbol     string `json:"symbol"`
 	Tag        string `json:"tag,omitempty"`
+	ChainID    int    `json:"chainID"`
 }
 
 type GetTokensResponse struct {
@@ -64,7 +67,6 @@ type GetCensusResponse struct {
 	URI        string `json:"uri"`
 	Size       int32  `json:"size"`
 	Weight     string `json:"weight"`
-	ChainID    uint64 `json:"chainId"`
 	Anonymous  bool   `json:"anonymous"`
 }
 

cmd/census3/main.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
 	flag "github.com/spf13/pflag"
 	"github.com/vocdoni/census3/api"
 	"github.com/vocdoni/census3/db"
 	"github.com/vocdoni/census3/service"
+	"github.com/vocdoni/census3/state"
 	"go.vocdoni.io/dvote/log"
 )
 
@@ -20,11 +22,11 @@ func main() {
 		panic(err)
 	}
 	home += "/.census3"
-	url := flag.String("url", "", "ethereum web3 url")
 	dataDir := flag.String("dataDir", home, "data directory for persistent storage")
 	logLevel := flag.String("logLevel", "info", "log level (debug, info, warn, error)")
 	port := flag.Int("port", 7788, "HTTP port for the API")
 	connectKey := flag.String("connectKey", "", "connect group key for IPFS connect")
+	listOfWeb3Providers := flag.String("web3Providers", "", "the list of URL's of available web3 providers (separated with commas)")
 	flag.Parse()
 	log.Init(*logLevel, "stdout", nil)
 
@@ -33,19 +35,26 @@ func main() {
 		log.Fatal(err)
 	}
 
+	web3Providers := strings.Split(*listOfWeb3Providers, ",")
+	w3p, err := state.CheckWeb3Providers(web3Providers)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Info(w3p)
+
 	// Start the holder scanner
-	hc, err := service.NewHoldersScanner(db, q, *url)
+	hc, err := service.NewHoldersScanner(db, q, w3p)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Start the API
 	err = api.Init(db, q, api.Census3APIConf{
-		Hostname: "0.0.0.0",
-		Port:     *port,
-		DataDir:  *dataDir,
-		Web3URI:  *url,
-		GroupKey: *connectKey,
+		Hostname:      "0.0.0.0",
+		Port:          *port,
+		DataDir:       *dataDir,
+		Web3Providers: w3p,
+		GroupKey:      *connectKey,
 	})
 	if err != nil {
 		log.Fatal(err)

db/migrations/0001_census3.sql

@@ -1,8 +1,4 @@
 -- +goose Up
-CREATE TABLE metadata (
-    chainID INTEGER PRIMARY KEY
-);
-
 CREATE TABLE strategies (
     id INTEGER PRIMARY KEY,
     predicate TEXT NOT NULL
@@ -30,6 +26,8 @@ CREATE TABLE tokens (
     type_id INTEGER NOT NULL,
     synced BOOLEAN NOT NULL,
     tag TEXT,
+    chain_id INTEGER NOT NULL,
+    UNIQUE (id, chain_id),
     FOREIGN KEY (type_id) REFERENCES token_types(id) ON DELETE CASCADE
 );
 CREATE INDEX idx_tokens_type_id ON tokens(type_id);

db/queries/tokens.sql

@@ -38,10 +38,11 @@ INSERT INTO tokens (
     creation_block,
     type_id,
     synced,
-    tag
+    tag,
+    chain_id
 )
 VALUES (
-    ?, ?, ?, ?, ?, ?, ?, ?, ?
+    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
 );
 
 -- name: UpdateToken :execresult