diff --git a/contract-watcher/cmd/backfiller/run.go b/contract-watcher/cmd/backfiller/run.go index 7a9539f9e..822d6ae33 100644 --- a/contract-watcher/cmd/backfiller/run.go +++ b/contract-watcher/cmd/backfiller/run.go @@ -3,6 +3,7 @@ package backfiller import ( "context" + "github.com/wormhole-foundation/wormhole-explorer/common/client/alert" "github.com/wormhole-foundation/wormhole-explorer/common/domain" "github.com/wormhole-foundation/wormhole-explorer/common/logger" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/builder" @@ -28,10 +29,14 @@ func Run(config *config.BackfillerConfiguration) { logger.Fatal("failed to connect MongoDB", zap.Error(err)) } + // create metrics client metrics := metrics.NewNoopMetrics() + // create alert client + alerts := alert.NewDummyClient() + // create repositories - repo := storage.NewRepository(db.Database, metrics, logger) + repo := storage.NewRepository(db.Database, metrics, alerts, logger) var watcher watcher.ContractWatcher diff --git a/contract-watcher/cmd/service/run.go b/contract-watcher/cmd/service/run.go index c451012cf..395a54b84 100644 --- a/contract-watcher/cmd/service/run.go +++ b/contract-watcher/cmd/service/run.go @@ -8,12 +8,14 @@ import ( "syscall" "time" + "github.com/wormhole-foundation/wormhole-explorer/common/client/alert" "github.com/wormhole-foundation/wormhole-explorer/common/domain" "github.com/wormhole-foundation/wormhole-explorer/common/health" "github.com/wormhole-foundation/wormhole-explorer/common/logger" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/builder" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/config" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/http/infrastructure" + cwAlert "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/alert" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/ankr" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/db" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/metrics" @@ -83,10 +85,14 @@ func Run() { logger.Fatal("failed to create health checks", zap.Error(err)) } + // create metrics client metrics := metrics.NewPrometheusMetrics(config.Environment) + // create alert client + alerts := newAlertClient(config, logger) + // create repositories - repo := storage.NewRepository(db.Database, metrics, logger) + repo := storage.NewRepository(db.Database, metrics, alerts, logger) // create watchers watchers := newWatchers(config, repo, metrics, logger) @@ -238,3 +244,21 @@ func newWatchersForTestnet() *watchersConfig { }, } } + +func newAlertClient(config *config.ServiceConfiguration, logger *zap.Logger) alert.AlertClient { + + if !config.AlertEnabled { + return alert.NewDummyClient() + } + alertConfig := alert.AlertConfig{ + Enviroment: config.Environment, + P2PNetwork: config.P2pNetwork, + ApiKey: config.AlertApiKey, + Enabled: config.AlertEnabled, + } + client, err := alert.NewAlertService(alertConfig, cwAlert.LoadAlerts) + if err != nil { + logger.Fatal("Error creating alert client", zap.Error(err)) + } + return client +} diff --git a/contract-watcher/config/config.go b/contract-watcher/config/config.go index 082040429..c15ae3700 100644 --- a/contract-watcher/config/config.go +++ b/contract-watcher/config/config.go @@ -23,6 +23,8 @@ type ServiceConfiguration struct { CeloUrl string `env:"CELO_URL,required"` PprofEnabled bool `env:"PPROF_ENABLED,default=false"` P2pNetwork string `env:"P2P_NETWORK,required"` + AlertEnabled bool `env:"ALERT_ENABLED,required"` + AlertApiKey string `env:"ALERT_API_KEY"` } // BackfillerConfiguration represents the application configuration when running as backfiller. diff --git a/contract-watcher/go.mod b/contract-watcher/go.mod index a4110d00e..1fbac4ccc 100644 --- a/contract-watcher/go.mod +++ b/contract-watcher/go.mod @@ -54,12 +54,15 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect + github.com/hashicorp/go-retryablehttp v0.5.1 // indirect github.com/holiman/uint256 v1.2.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.3 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -71,6 +74,7 @@ require ( github.com/montanaflynn/stats v0.7.0 // indirect github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.19 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -80,6 +84,7 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect + github.com/sirupsen/logrus v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 // indirect github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect diff --git a/contract-watcher/go.sum b/contract-watcher/go.sum index f64eb7d8c..37529aac9 100644 --- a/contract-watcher/go.sum +++ b/contract-watcher/go.sum @@ -245,10 +245,14 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.1 h1:Vsx5XKPqPs3M6sM4U4GWyUqFS8aBiL9U5gkgvpkg4SE= +github.com/hashicorp/go-retryablehttp v0.5.1/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -301,6 +305,7 @@ github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -365,6 +370,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.19 h1:JernwK3Bgd5x+UJPV6S2LPYoBF+DFOYBoQ5JeJPVBNc= +github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.19/go.mod h1:4OjcxgwdXzezqytxN534MooNmrxRD50geWZxTD7845s= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -431,6 +438,7 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -626,6 +634,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/contract-watcher/internal/alert/alert.go b/contract-watcher/internal/alert/alert.go new file mode 100644 index 000000000..8614c9a33 --- /dev/null +++ b/contract-watcher/internal/alert/alert.go @@ -0,0 +1,30 @@ +package alert + +import ( + "fmt" + + "github.com/wormhole-foundation/wormhole-explorer/common/client/alert" +) + +// alert key constants definition. +const ( + ErrorSaveDestinationTx = "ERROR_SAVE_DESTINATION_TX" +) + +func LoadAlerts(cfg alert.AlertConfig) map[string]alert.Alert { + alerts := make(map[string]alert.Alert) + + messagePrefix := alert.GetMessagePrefix(cfg.Enviroment, cfg.P2PNetwork) + // Alert error saving vaa. + alerts[ErrorSaveDestinationTx] = alert.Alert{ + Alias: ErrorSaveDestinationTx, + Message: fmt.Sprintf("%s %s", messagePrefix, "Error saving destination tx in globalTransactions collection"), + Description: "An error was found persisting the destination tx in globalTransactions collection.", + Actions: []string{"check globalTransactions collection"}, + Tags: []string{cfg.Enviroment, cfg.P2PNetwork, "contract-watcher", "destination tx", "mongo"}, + Entity: "contract-watcher", + Priority: alert.CRITICAL, + } + + return alerts +} diff --git a/contract-watcher/storage/model.go b/contract-watcher/storage/model.go index 4f237df7c..0a03b00f6 100644 --- a/contract-watcher/storage/model.go +++ b/contract-watcher/storage/model.go @@ -29,6 +29,19 @@ type TransactionUpdate struct { Destination DestinationTx `bson:"destinationTx"` } +func (t *TransactionUpdate) ToMap() map[string]string { + return map[string]string{ + "id": t.ID, + "destination.chainId": t.Destination.ChainID.String(), + "destination.status": t.Destination.Status, + "destination.method": t.Destination.Method, + "destination.txHash": t.Destination.TxHash, + "destination.from": t.Destination.From, + "destination.to": t.Destination.To, + "destination.blockNumber": t.Destination.BlockNumber, + } +} + type WatcherBlock struct { ID string `bson:"_id"` BlockNumber int64 `bson:"blockNumber"` diff --git a/contract-watcher/storage/repository.go b/contract-watcher/storage/repository.go index a3a37d152..048edd512 100644 --- a/contract-watcher/storage/repository.go +++ b/contract-watcher/storage/repository.go @@ -5,6 +5,8 @@ import ( "errors" "time" + "github.com/wormhole-foundation/wormhole-explorer/common/client/alert" + cwAlert "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/alert" "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/metrics" sdk "github.com/wormhole-foundation/wormhole/sdk/vaa" "go.mongodb.org/mongo-driver/bson" @@ -21,6 +23,7 @@ type Repository struct { db *mongo.Database log *zap.Logger metrics metrics.Metrics + alerts alert.AlertClient collections struct { watcherBlock *mongo.Collection globalTransactions *mongo.Collection @@ -28,8 +31,8 @@ type Repository struct { } // NewRepository create a new respository instance. -func NewRepository(db *mongo.Database, metrics metrics.Metrics, log *zap.Logger) *Repository { - return &Repository{db, log, metrics, struct { +func NewRepository(db *mongo.Database, metrics metrics.Metrics, alerts alert.AlertClient, log *zap.Logger) *Repository { + return &Repository{db, log, metrics, alerts, struct { watcherBlock *mongo.Collection globalTransactions *mongo.Collection }{ @@ -44,16 +47,22 @@ func indexedAt(t time.Time) IndexingTimestamps { } } -func (s *Repository) UpsertGlobalTransaction(ctx context.Context, chainID sdk.ChainID, globalTransactions TransactionUpdate) error { +func (s *Repository) UpsertGlobalTransaction(ctx context.Context, chainID sdk.ChainID, globalTx TransactionUpdate) error { update := bson.M{ - "$set": globalTransactions, + "$set": globalTx, "$setOnInsert": indexedAt(time.Now()), "$inc": bson.D{{Key: "revision", Value: 1}}, } - _, err := s.collections.globalTransactions.UpdateByID(ctx, globalTransactions.ID, update, options.Update().SetUpsert(true)) + _, err := s.collections.globalTransactions.UpdateByID(ctx, globalTx.ID, update, options.Update().SetUpsert(true)) if err != nil { s.log.Error("Error inserting global transaction", zap.Error(err)) + // send alert when exists an error saving ptth vaa. + alertContext := alert.AlertContext{ + Details: globalTx.ToMap(), + Error: err, + } + s.alerts.CreateAndSend(ctx, cwAlert.ErrorSaveDestinationTx, alertContext) return err } s.metrics.IncDestinationTrxSaved(chainID) diff --git a/deploy/contract-watcher/contract-watcher-service.yaml b/deploy/contract-watcher/contract-watcher-service.yaml index 97500cfdf..dac904d4a 100644 --- a/deploy/contract-watcher/contract-watcher-service.yaml +++ b/deploy/contract-watcher/contract-watcher-service.yaml @@ -95,6 +95,13 @@ spec: secretKeyRef: name: blockchain key: celo-url + - name: ALERT_API_KEY + valueFrom: + secretKeyRef: + name: opsgenie + key: api-key + - name: ALERT_ENABLED + value: "{{ .ALERT_ENABLED }}" resources: limits: memory: {{ .RESOURCES_LIMITS_MEMORY }} diff --git a/deploy/contract-watcher/env/production.env b/deploy/contract-watcher/env/production.env index 52dcb94d6..4c930c225 100644 --- a/deploy/contract-watcher/env/production.env +++ b/deploy/contract-watcher/env/production.env @@ -15,4 +15,5 @@ TERRA_URL= APTOS_URL= OASIS_URL= MOONBEAM_URL= -CELO_URL= \ No newline at end of file +CELO_URL= +ALERT_ENABLED=false \ No newline at end of file diff --git a/deploy/contract-watcher/env/staging.env b/deploy/contract-watcher/env/staging.env index 8457a1b55..c3986ab2e 100644 --- a/deploy/contract-watcher/env/staging.env +++ b/deploy/contract-watcher/env/staging.env @@ -15,4 +15,5 @@ TERRA_URL= APTOS_URL= OASIS_URL= MOONBEAM_URL= -CELO_URL= \ No newline at end of file +CELO_URL= +ALERT_ENABLED=false \ No newline at end of file diff --git a/deploy/contract-watcher/env/test.env b/deploy/contract-watcher/env/test.env index 2ab2dc67b..cd5bfb5ec 100644 --- a/deploy/contract-watcher/env/test.env +++ b/deploy/contract-watcher/env/test.env @@ -15,4 +15,5 @@ TERRA_URL= APTOS_URL= OASIS_URL= MOONBEAM_URL= -CELO_URL= \ No newline at end of file +CELO_URL= +ALERT_ENABLED=false \ No newline at end of file