Add alerts for contract-watcher

contract-watcher/cmd/backfiller/run.go

@@ -3,6 +3,7 @@ package backfiller
 import (
  "context"
 
+ "github.com/wormhole-foundation/wormhole-explorer/common/client/alert"
  "github.com/wormhole-foundation/wormhole-explorer/common/domain"
  "github.com/wormhole-foundation/wormhole-explorer/common/logger"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/builder"
@@ -28,10 +29,14 @@ func Run(config *config.BackfillerConfiguration) {
  logger.Fatal("failed to connect MongoDB", zap.Error(err))
  }
 
+ // create metrics client
  metrics := metrics.NewNoopMetrics()
 
+ // create alert client
+ alerts := alert.NewDummyClient()
+
  // create repositories
- repo := storage.NewRepository(db.Database, metrics, logger)
+ repo := storage.NewRepository(db.Database, metrics, alerts, logger)
 
  var watcher watcher.ContractWatcher
 

contract-watcher/cmd/service/run.go

@@ -8,12 +8,14 @@ import (
  "syscall"
  "time"
 
+ "github.com/wormhole-foundation/wormhole-explorer/common/client/alert"
  "github.com/wormhole-foundation/wormhole-explorer/common/domain"
  "github.com/wormhole-foundation/wormhole-explorer/common/health"
  "github.com/wormhole-foundation/wormhole-explorer/common/logger"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/builder"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/config"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/http/infrastructure"
+ cwAlert "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/alert"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/ankr"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/db"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/metrics"
@@ -83,10 +85,14 @@ func Run() {
  logger.Fatal("failed to create health checks", zap.Error(err))
  }
 
+ // create metrics client
  metrics := metrics.NewPrometheusMetrics(config.Environment)
 
+ // create alert client
+ alerts := newAlertClient(config, logger)
+
  // create repositories
- repo := storage.NewRepository(db.Database, metrics, logger)
+ repo := storage.NewRepository(db.Database, metrics, alerts, logger)
 
  // create watchers
  watchers := newWatchers(config, repo, metrics, logger)
@@ -238,3 +244,21 @@ func newWatchersForTestnet() *watchersConfig {
  },
  }
 }
+
+func newAlertClient(config *config.ServiceConfiguration, logger *zap.Logger) alert.AlertClient {
+
+ if !config.AlertEnabled {
+ return alert.NewDummyClient()
+ }
+ alertConfig := alert.AlertConfig{
+ Enviroment: config.Environment,
+ P2PNetwork: config.P2pNetwork,
+ ApiKey: config.AlertApiKey,
+ Enabled: config.AlertEnabled,
+ }
+ client, err := alert.NewAlertService(alertConfig, cwAlert.LoadAlerts)
+ if err != nil {
+ logger.Fatal("Error creating alert client", zap.Error(err))
+ }
+ return client
+}

contract-watcher/config/config.go

@@ -23,6 +23,8 @@ type ServiceConfiguration struct {
  CeloUrl string `env:"CELO_URL,required"`
  PprofEnabled bool `env:"PPROF_ENABLED,default=false"`
  P2pNetwork string `env:"P2P_NETWORK,required"`
+ AlertEnabled bool `env:"ALERT_ENABLED,required"`
+ AlertApiKey string `env:"ALERT_API_KEY"`
 }
 
 // BackfillerConfiguration represents the application configuration when running as backfiller.

contract-watcher/go.mod

@@ -54,12 +54,15 @@ require (
  github.com/golang/protobuf v1.5.2 // indirect
  github.com/golang/snappy v0.0.4 // indirect
  github.com/google/uuid v1.3.0 // indirect
+ github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+ github.com/hashicorp/go-retryablehttp v0.5.1 // indirect
  github.com/holiman/uint256 v1.2.1 // indirect
  github.com/inconshreveable/mousetrap v1.0.0 // indirect
  github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect
  github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect
  github.com/json-iterator/go v1.1.12 // indirect
  github.com/klauspost/compress v1.16.3 // indirect
+ github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
  github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
  github.com/mattn/go-colorable v0.1.13 // indirect
  github.com/mattn/go-isatty v0.0.19 // indirect
@@ -71,6 +74,7 @@ require (
  github.com/montanaflynn/stats v0.7.0 // indirect
  github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect
  github.com/mr-tron/base58 v1.2.0 // indirect
+ github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.19 // indirect
  github.com/philhofer/fwd v1.1.2 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -80,6 +84,7 @@ require (
  github.com/rivo/uniseg v0.4.4 // indirect
  github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
  github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
+ github.com/sirupsen/logrus v1.6.0 // indirect
  github.com/spf13/pflag v1.0.5 // indirect
  github.com/streamingfast/logging v0.0.0-20220405224725-2755dab2ce75 // indirect
  github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 // indirect

contract-watcher/go.sum

@@ -245,10 +245,14 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-retryablehttp v0.5.1 h1:Vsx5XKPqPs3M6sM4U4GWyUqFS8aBiL9U5gkgvpkg4SE=
+github.com/hashicorp/go-retryablehttp v0.5.1/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -301,6 +305,7 @@ github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU
 github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
 github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -365,6 +370,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.19 h1:JernwK3Bgd5x+UJPV6S2LPYoBF+DFOYBoQ5JeJPVBNc=
+github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.19/go.mod h1:4OjcxgwdXzezqytxN534MooNmrxRD50geWZxTD7845s=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@@ -431,6 +438,7 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -626,6 +634,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=

contract-watcher/internal/alert/alert.go

@@ -0,0 +1,30 @@
+package alert
+
+import (
+ "fmt"
+
+ "github.com/wormhole-foundation/wormhole-explorer/common/client/alert"
+)
+
+// alert key constants definition.
+const (
+ ErrorSaveDestinationTx = "ERROR_SAVE_DESTINATION_TX"
+)
+
+func LoadAlerts(cfg alert.AlertConfig) map[string]alert.Alert {
+ alerts := make(map[string]alert.Alert)
+
+ messagePrefix := alert.GetMessagePrefix(cfg.Enviroment, cfg.P2PNetwork)
+ // Alert error saving vaa.
+ alerts[ErrorSaveDestinationTx] = alert.Alert{
+ Alias: ErrorSaveDestinationTx,
+ Message: fmt.Sprintf("%s %s", messagePrefix, "Error saving destination tx in globalTransactions collection"),
+ Description: "An error was found persisting the destination tx in globalTransactions collection.",
+ Actions: []string{"check globalTransactions collection"},
+ Tags: []string{cfg.Enviroment, cfg.P2PNetwork, "contract-watcher", "destination tx", "mongo"},
+ Entity: "contract-watcher",
+ Priority: alert.CRITICAL,
+ }
+
+ return alerts
+}

contract-watcher/storage/model.go

@@ -29,6 +29,19 @@ type TransactionUpdate struct {
  Destination DestinationTx `bson:"destinationTx"`
 }
 
+func (t *TransactionUpdate) ToMap() map[string]string {
+ return map[string]string{
+ "id": t.ID,
+ "destination.chainId": t.Destination.ChainID.String(),
+ "destination.status": t.Destination.Status,
+ "destination.method": t.Destination.Method,
+ "destination.txHash": t.Destination.TxHash,
+ "destination.from": t.Destination.From,
+ "destination.to": t.Destination.To,
+ "destination.blockNumber": t.Destination.BlockNumber,
+ }
+}
+
 type WatcherBlock struct {
  ID string `bson:"_id"`
  BlockNumber int64 `bson:"blockNumber"`

contract-watcher/storage/repository.go

@@ -5,6 +5,8 @@ import (
  "errors"
  "time"
 
+ "github.com/wormhole-foundation/wormhole-explorer/common/client/alert"
+ cwAlert "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/alert"
  "github.com/wormhole-foundation/wormhole-explorer/contract-watcher/internal/metrics"
  sdk "github.com/wormhole-foundation/wormhole/sdk/vaa"
  "go.mongodb.org/mongo-driver/bson"
@@ -21,15 +23,16 @@ type Repository struct {
  db *mongo.Database
  log *zap.Logger
  metrics metrics.Metrics
+ alerts alert.AlertClient
  collections struct {
  watcherBlock *mongo.Collection
  globalTransactions *mongo.Collection
  }
 }
 
 // NewRepository create a new respository instance.
-func NewRepository(db *mongo.Database, metrics metrics.Metrics, log *zap.Logger) *Repository {
- return &Repository{db, log, metrics, struct {
+func NewRepository(db *mongo.Database, metrics metrics.Metrics, alerts alert.AlertClient, log *zap.Logger) *Repository {
+ return &Repository{db, log, metrics, alerts, struct {
  watcherBlock *mongo.Collection
  globalTransactions *mongo.Collection
  }{
@@ -44,16 +47,22 @@ func indexedAt(t time.Time) IndexingTimestamps {
  }
 }
 
-func (s *Repository) UpsertGlobalTransaction(ctx context.Context, chainID sdk.ChainID, globalTransactions TransactionUpdate) error {
+func (s *Repository) UpsertGlobalTransaction(ctx context.Context, chainID sdk.ChainID, globalTx TransactionUpdate) error {
  update := bson.M{
- "$set": globalTransactions,
+ "$set": globalTx,
  "$setOnInsert": indexedAt(time.Now()),
  "$inc": bson.D{{Key: "revision", Value: 1}},
  }
 
- _, err := s.collections.globalTransactions.UpdateByID(ctx, globalTransactions.ID, update, options.Update().SetUpsert(true))
+ _, err := s.collections.globalTransactions.UpdateByID(ctx, globalTx.ID, update, options.Update().SetUpsert(true))
  if err != nil {
  s.log.Error("Error inserting global transaction", zap.Error(err))
+ // send alert when exists an error saving ptth vaa.
+ alertContext := alert.AlertContext{
+ Details: globalTx.ToMap(),
+ Error: err,
+ }
+ s.alerts.CreateAndSend(ctx, cwAlert.ErrorSaveDestinationTx, alertContext)
  return err
  }
  s.metrics.IncDestinationTrxSaved(chainID)

deploy/contract-watcher/contract-watcher-service.yaml

@@ -95,6 +95,13 @@ spec:
  secretKeyRef:
  name: blockchain
  key: celo-url
+ - name: ALERT_API_KEY
+ valueFrom:
+ secretKeyRef:
+ name: opsgenie
+ key: api-key
+ - name: ALERT_ENABLED
+ value: "{{ .ALERT_ENABLED }}"
  resources:
  limits:
  memory: {{ .RESOURCES_LIMITS_MEMORY }}

deploy/contract-watcher/env/production.env

@@ -15,4 +15,5 @@ TERRA_URL=
 APTOS_URL=
 OASIS_URL=
 MOONBEAM_URL=
-CELO_URL=
+CELO_URL=
+ALERT_ENABLED=false

deploy/contract-watcher/env/staging.env

@@ -15,4 +15,5 @@ TERRA_URL=
 APTOS_URL=
 OASIS_URL=
 MOONBEAM_URL=
-CELO_URL=
+CELO_URL=
+ALERT_ENABLED=false

deploy/contract-watcher/env/test.env

@@ -15,4 +15,5 @@ TERRA_URL=
 APTOS_URL=
 OASIS_URL=
 MOONBEAM_URL=
-CELO_URL=
+CELO_URL=
+ALERT_ENABLED=false