From 356c70cb8079b1052faa45d0c53fa1d8212db355 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 11 Sep 2024 09:30:04 -0400 Subject: [PATCH] Productionize LLO transmitter (#14355) This mostly copies over similar patterns and structure from the battle-tested mercury transmitter, with a few modifications to generalize it to LLO --- .changeset/strange-swans-yawn.md | 21 ++ core/cmd/forwarders_commands_test.go | 2 +- core/scripts/go.mod | 2 +- core/scripts/go.sum | 4 +- .../llo/mercurytransmitter/helpers_test.go | 47 ++++ core/services/llo/mercurytransmitter/orm.go | 196 +++++++++++++ .../llo/mercurytransmitter/orm_test.go | 89 ++++++ .../mercurytransmitter/persistence_manager.go | 143 ++++++++++ .../persistence_manager_test.go | 127 +++++++++ core/services/llo/mercurytransmitter/queue.go | 250 +++++++++++++++++ .../llo/mercurytransmitter/queue_test.go | 105 +++++++ .../services/llo/mercurytransmitter/server.go | 240 ++++++++++++++++ .../llo/mercurytransmitter/transmitter.go | 254 +++++++++++++++++ .../mercurytransmitter/transmitter_test.go | 258 ++++++++++++++++++ core/services/llo/transmitter.go | 87 +++--- core/services/llo/transmitter_test.go | 7 - .../ocr2/plugins/llo/config/config.go | 65 +++-- .../ocr2/plugins/llo/config/config_test.go | 62 +++-- .../ocr2/plugins/llo/integration_test.go | 5 +- core/services/relay/dummy/llo_provider.go | 10 +- core/services/relay/evm/evm.go | 27 +- core/services/relay/evm/llo_provider.go | 11 +- .../relay/evm/mercury/config_poller.go | 1 - .../0252_add_llo_transmit_requests.sql | 21 ++ go.mod | 2 +- go.sum | 4 +- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- 30 files changed, 1934 insertions(+), 118 deletions(-) create mode 100644 .changeset/strange-swans-yawn.md create mode 100644 core/services/llo/mercurytransmitter/helpers_test.go create mode 100644 core/services/llo/mercurytransmitter/orm.go create mode 100644 core/services/llo/mercurytransmitter/orm_test.go create mode 100644 core/services/llo/mercurytransmitter/persistence_manager.go create mode 100644 core/services/llo/mercurytransmitter/persistence_manager_test.go create mode 100644 core/services/llo/mercurytransmitter/queue.go create mode 100644 core/services/llo/mercurytransmitter/queue_test.go create mode 100644 core/services/llo/mercurytransmitter/server.go create mode 100644 core/services/llo/mercurytransmitter/transmitter.go create mode 100644 core/services/llo/mercurytransmitter/transmitter_test.go delete mode 100644 core/services/llo/transmitter_test.go create mode 100644 core/store/migrate/migrations/0252_add_llo_transmit_requests.sql diff --git a/.changeset/strange-swans-yawn.md b/.changeset/strange-swans-yawn.md new file mode 100644 index 00000000000..d18c743a8ae --- /dev/null +++ b/.changeset/strange-swans-yawn.md @@ -0,0 +1,21 @@ +--- +"chainlink": patch +--- + +#changed + +Productionize transmitter for LLO + +Note that some minor changes to prometheus metrics will occur in the transition to LLO. Since feed IDs no longer apply, the metrics for transmissions change as follows: + +``` +"mercury_transmit_*" +[]string{"feedID", ...}, +``` + +Will change to: + +``` +"llo_mercury_transmit_*" +[]string{"donID", ...}, +``` diff --git a/core/cmd/forwarders_commands_test.go b/core/cmd/forwarders_commands_test.go index b5a96a73aa8..dfd8272d8a4 100644 --- a/core/cmd/forwarders_commands_test.go +++ b/core/cmd/forwarders_commands_test.go @@ -22,7 +22,7 @@ func TestEVMForwarderPresenter_RenderTable(t *testing.T) { t.Parallel() var ( - id = "1" + id = "ID:" address = utils.RandomAddress() evmChainID = big.NewI(4) createdAt = time.Now() diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 21687a7cc4a..f4f2e3a32fd 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -273,7 +273,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.21 // indirect github.com/smartcontractkit/chainlink-ccip v0.0.0-20240910072312-810030689426 // indirect github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 // indirect - github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 // indirect + github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc // indirect github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240910163253-2a5c9ab97de3 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index a65014a9d16..cac4c2cb3dc 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1085,8 +1085,8 @@ github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de732 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de7322/go.mod h1:D/qaCoq0SxXzg5NRN5FtBRv98VBf+D2NOC++RbvvuOc= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 h1:xow5cYrWxRmzTwhz2AjWOlnI9WRF4O5b84bXm1k292E= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227/go.mod h1:DUFantPYoBGwBSkNVt2k4ZJi0jPKRRrZVVlAzcZwreA= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 h1:0aZ3HiEz2bMM5ywHAyKlFMN95qTzpNDn7uvnHLrFX6s= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc h1:tRmTlaoAt+7FakMXXgeCuRPmzzBo5jsGpeCVvcU6KMc= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f h1:p4p3jBT91EQyLuAMvHD+zNJsuAYI/QjJbzuGUJ7wIgg= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f/go.mod h1:FLlWBt2hwiMVgt9AcSo6wBJYIRd/nsc8ENbV1Wir1bw= github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd h1:+RFI4mgXSGEcn847e7bavhQCqaBiW142g1XfGzijFoY= diff --git a/core/services/llo/mercurytransmitter/helpers_test.go b/core/services/llo/mercurytransmitter/helpers_test.go new file mode 100644 index 00000000000..060c07bef45 --- /dev/null +++ b/core/services/llo/mercurytransmitter/helpers_test.go @@ -0,0 +1,47 @@ +package mercurytransmitter + +import ( + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" +) + +func makeSampleReport() ocr3types.ReportWithInfo[llotypes.ReportInfo] { + return ocr3types.ReportWithInfo[llotypes.ReportInfo]{ + Report: ocrtypes.Report{1, 2, 3}, + Info: llotypes.ReportInfo{ + LifeCycleStage: llotypes.LifeCycleStage("production"), + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, + }, + } +} + +func makeSampleConfigDigest() ocrtypes.ConfigDigest { + return ocrtypes.ConfigDigest{1, 2, 3, 4, 5, 6} +} +func makeSampleTransmission(seqNr uint64) *Transmission { + // valid with seqnr of 3 + return &Transmission{ + ServerURL: "wss://example.com/mercury", + ConfigDigest: types.ConfigDigest{0x0, 0x9, 0x57, 0xdd, 0x2f, 0x63, 0x56, 0x69, 0x34, 0xfd, 0xc2, 0xe1, 0xcd, 0xc1, 0xe, 0x3e, 0x25, 0xb9, 0x26, 0x5a, 0x16, 0x23, 0x91, 0xa6, 0x53, 0x16, 0x66, 0x59, 0x51, 0x0, 0x28, 0x7c}, + SeqNr: seqNr, + Report: ocr3types.ReportWithInfo[llotypes.ReportInfo]{ + Report: ocrtypes.Report{0x0, 0x3, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1e, 0x8e, 0x95, 0xcf, 0xb5, 0xd8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1a, 0xd0, 0x1c, 0x67, 0xa9, 0xcf, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xdf, 0x3, 0xca, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x1c, 0x93, 0x6d, 0xa4, 0xf2, 0x17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x14, 0x8d, 0x9a, 0xc1, 0xd9, 0x6f, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x40, 0x5c, 0xcf, 0xa1, 0xbc, 0x63, 0xc0, 0x0}, + Info: llotypes.ReportInfo{ + LifeCycleStage: llotypes.LifeCycleStage("production"), + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, + }, + }, + Sigs: []types.AttributedOnchainSignature{types.AttributedOnchainSignature{Signature: []uint8{0x9d, 0xab, 0x8f, 0xa7, 0xca, 0x7, 0x62, 0x57, 0xf7, 0x11, 0x2c, 0xb7, 0xf3, 0x49, 0x37, 0x12, 0xbd, 0xe, 0x14, 0x27, 0xfc, 0x32, 0x5c, 0xec, 0xa6, 0xb9, 0x7f, 0xf9, 0xd7, 0x7b, 0xa6, 0x36, 0x30, 0x9d, 0x84, 0x29, 0xbf, 0xd4, 0xeb, 0xc5, 0xc9, 0x29, 0xef, 0xdd, 0xd3, 0x2f, 0xa6, 0x25, 0x63, 0xda, 0xd9, 0x2c, 0xa1, 0x4a, 0xba, 0x75, 0xb2, 0x85, 0x25, 0x8f, 0x2b, 0x84, 0xcd, 0x99, 0x1}, Signer: 0x1}, types.AttributedOnchainSignature{Signature: []uint8{0x9a, 0x47, 0x4a, 0x3, 0x1a, 0x95, 0xcf, 0x46, 0x10, 0xaf, 0xcc, 0x90, 0x49, 0xb2, 0xce, 0xbf, 0x63, 0xaa, 0xc7, 0x25, 0x4d, 0x2a, 0x8, 0x36, 0xda, 0xd5, 0x9f, 0x9d, 0x63, 0x69, 0x22, 0xb3, 0x36, 0xd9, 0x6e, 0xf, 0xae, 0x7b, 0xd1, 0x61, 0x59, 0xf, 0x36, 0x4a, 0x22, 0xec, 0xde, 0x45, 0x32, 0xe0, 0x5b, 0x5c, 0xe3, 0x14, 0x29, 0x4, 0x60, 0x7b, 0xce, 0xa3, 0x89, 0x6b, 0xbb, 0xe0, 0x0}, Signer: 0x3}}, + } +} + +func makeSampleTransmissions() []*Transmission { + return []*Transmission{ + makeSampleTransmission(1001), + makeSampleTransmission(1002), + makeSampleTransmission(1003), + } +} diff --git a/core/services/llo/mercurytransmitter/orm.go b/core/services/llo/mercurytransmitter/orm.go new file mode 100644 index 00000000000..d73722c82ab --- /dev/null +++ b/core/services/llo/mercurytransmitter/orm.go @@ -0,0 +1,196 @@ +package mercurytransmitter + +import ( + "context" + "errors" + "fmt" + "math" + + "github.com/lib/pq" + + "github.com/smartcontractkit/libocr/commontypes" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" +) + +// ORM is scoped to a single DON ID +type ORM interface { + DonID() uint32 + Insert(ctx context.Context, transmissions []*Transmission) error + Delete(ctx context.Context, hashes [][32]byte) error + Get(ctx context.Context, serverURL string) ([]*Transmission, error) + Prune(ctx context.Context, serverURL string, maxSize int) error +} + +type orm struct { + ds sqlutil.DataSource + donID uint32 +} + +func NewORM(ds sqlutil.DataSource, donID uint32) ORM { + return &orm{ds: ds, donID: donID} +} + +func (o *orm) DonID() uint32 { + return o.donID +} + +// Insert inserts the transmissions, ignoring duplicates +func (o *orm) Insert(ctx context.Context, transmissions []*Transmission) error { + if len(transmissions) == 0 { + return nil + } + + type transmission struct { + DonID uint32 `db:"don_id"` + ServerURL string `db:"server_url"` + ConfigDigest ocrtypes.ConfigDigest `db:"config_digest"` + SeqNr int64 `db:"seq_nr"` + Report []byte `db:"report"` + LifecycleStage string `db:"lifecycle_stage"` + ReportFormat uint32 `db:"report_format"` + Signatures [][]byte `db:"signatures"` + Signers []uint8 `db:"signers"` + TransmissionHash []byte `db:"transmission_hash"` + } + records := make([]transmission, len(transmissions)) + for i, t := range transmissions { + signatures := make([][]byte, len(t.Sigs)) + signers := make([]uint8, len(t.Sigs)) + for j, sig := range t.Sigs { + signatures[j] = sig.Signature + signers[j] = uint8(sig.Signer) + } + h := t.Hash() + if t.SeqNr > math.MaxInt64 { + // this is to appease the linter but shouldn't ever happen + return fmt.Errorf("seqNr is too large (got: %d, max: %d)", t.SeqNr, math.MaxInt64) + } + records[i] = transmission{ + DonID: o.donID, + ServerURL: t.ServerURL, + ConfigDigest: t.ConfigDigest, + SeqNr: int64(t.SeqNr), //nolint + Report: t.Report.Report, + LifecycleStage: string(t.Report.Info.LifeCycleStage), + ReportFormat: uint32(t.Report.Info.ReportFormat), + Signatures: signatures, + Signers: signers, + TransmissionHash: h[:], + } + } + + _, err := o.ds.NamedExecContext(ctx, ` + INSERT INTO llo_mercury_transmit_queue (don_id, server_url, config_digest, seq_nr, report, lifecycle_stage, report_format, signatures, signers, transmission_hash) + VALUES (:don_id, :server_url, :config_digest, :seq_nr, :report, :lifecycle_stage, :report_format, :signatures, :signers, :transmission_hash) + ON CONFLICT (transmission_hash) DO NOTHING + `, records) + + if err != nil { + return fmt.Errorf("llo orm: failed to insert transmissions: %w", err) + } + return nil +} + +// Delete deletes the given transmissions +func (o *orm) Delete(ctx context.Context, hashes [][32]byte) error { + if len(hashes) == 0 { + return nil + } + + var pqHashes pq.ByteaArray + for _, hash := range hashes { + pqHashes = append(pqHashes, hash[:]) + } + + _, err := o.ds.ExecContext(ctx, ` + DELETE FROM llo_mercury_transmit_queue + WHERE transmission_hash = ANY($1) + `, pqHashes) + if err != nil { + return fmt.Errorf("llo orm: failed to delete transmissions: %w", err) + } + return nil +} + +// Get returns all transmissions in chronologically descending order +func (o *orm) Get(ctx context.Context, serverURL string) ([]*Transmission, error) { + // The priority queue uses seqnr to sort transmissions so order by + // the same fields here for optimal insertion into the pq. + rows, err := o.ds.QueryContext(ctx, ` + SELECT config_digest, seq_nr, report, lifecycle_stage, report_format, signatures, signers + FROM llo_mercury_transmit_queue + WHERE don_id = $1 AND server_url = $2 + ORDER BY seq_nr DESC, transmission_hash DESC + `, o.donID, serverURL) + if err != nil { + return nil, fmt.Errorf("llo orm: failed to get transmissions: %w", err) + } + defer rows.Close() + + var transmissions []*Transmission + for rows.Next() { + transmission := Transmission{ + ServerURL: serverURL, + } + var digest []byte + var signatures pq.ByteaArray + var signers pq.Int32Array + + err := rows.Scan( + &digest, + &transmission.SeqNr, + &transmission.Report.Report, + &transmission.Report.Info.LifeCycleStage, + &transmission.Report.Info.ReportFormat, + &signatures, + &signers, + ) + if err != nil { + return nil, fmt.Errorf("llo orm: failed to scan transmission: %w", err) + } + transmission.ConfigDigest = ocrtypes.ConfigDigest(digest) + if len(signatures) != len(signers) { + return nil, errors.New("signatures and signers must have the same length") + } + for i, sig := range signatures { + if signers[i] > math.MaxUint8 { + // this is to appease the linter but shouldn't ever happen + return nil, fmt.Errorf("signer is too large (got: %d, max: %d)", signers[i], math.MaxUint8) + } + transmission.Sigs = append(transmission.Sigs, ocrtypes.AttributedOnchainSignature{ + Signature: sig, + Signer: commontypes.OracleID(signers[i]), //nolint + }) + } + + transmissions = append(transmissions, &transmission) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("llo orm: failed to scan transmissions: %w", err) + } + + return transmissions, nil +} + +// Prune keeps at most maxSize rows for the given job ID, +// deleting the oldest transactions. +func (o *orm) Prune(ctx context.Context, serverURL string, maxSize int) error { + // Prune the oldest requests by epoch and round. + _, err := o.ds.ExecContext(ctx, ` + DELETE FROM llo_mercury_transmit_queue + WHERE don_id = $1 AND server_url = $2 AND + transmission_hash NOT IN ( + SELECT transmission_hash + FROM llo_mercury_transmit_queue + WHERE don_id = $1 AND server_url = $2 + ORDER BY seq_nr DESC, transmission_hash DESC + LIMIT $3 + ) + `, o.donID, serverURL, maxSize) + if err != nil { + return fmt.Errorf("llo orm: failed to prune transmissions: %w", err) + } + return nil +} diff --git a/core/services/llo/mercurytransmitter/orm_test.go b/core/services/llo/mercurytransmitter/orm_test.go new file mode 100644 index 00000000000..73d5760a9a3 --- /dev/null +++ b/core/services/llo/mercurytransmitter/orm_test.go @@ -0,0 +1,89 @@ +package mercurytransmitter + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" +) + +var ( + sURL = "wss://example.com/mercury" + sURL2 = "wss://mercuryserver.test" + sURL3 = "wss://mercuryserver.example/foo" +) + +func TestORM(t *testing.T) { + ctx := testutils.Context(t) + db := pgtest.NewSqlxDB(t) + + donID := uint32(654321) + orm := NewORM(db, donID) + + t.Run("DonID", func(t *testing.T) { + assert.Equal(t, donID, orm.DonID()) + }) + + transmissions := makeSampleTransmissions()[:2] + + t.Run("Insert", func(t *testing.T) { + err := orm.Insert(ctx, transmissions) + require.NoError(t, err) + }) + t.Run("Get", func(t *testing.T) { + result, err := orm.Get(ctx, sURL) + require.NoError(t, err) + + assert.ElementsMatch(t, transmissions, result) + + result, err = orm.Get(ctx, "other server url") + require.NoError(t, err) + + assert.Empty(t, result) + }) + t.Run("Delete", func(t *testing.T) { + err := orm.Delete(ctx, [][32]byte{transmissions[0].Hash()}) + require.NoError(t, err) + + result, err := orm.Get(ctx, sURL) + require.NoError(t, err) + + require.Len(t, result, 1) + assert.Equal(t, transmissions[1], result[0]) + + err = orm.Delete(ctx, [][32]byte{transmissions[1].Hash()}) + require.NoError(t, err) + + result, err = orm.Get(ctx, sURL) + require.NoError(t, err) + require.Len(t, result, 0) + }) + t.Run("Prune", func(t *testing.T) { + err := orm.Insert(ctx, transmissions) + require.NoError(t, err) + + err = orm.Prune(ctx, sURL, 1) + require.NoError(t, err) + + result, err := orm.Get(ctx, sURL) + require.NoError(t, err) + require.Len(t, result, 1) + assert.Equal(t, transmissions[1], result[0]) + + err = orm.Prune(ctx, sURL, 1) + require.NoError(t, err) + result, err = orm.Get(ctx, sURL) + require.NoError(t, err) + require.Len(t, result, 1) + assert.Equal(t, transmissions[1], result[0]) + + err = orm.Prune(ctx, sURL, 0) + require.NoError(t, err) + result, err = orm.Get(ctx, sURL) + require.NoError(t, err) + require.Len(t, result, 0) + }) +} diff --git a/core/services/llo/mercurytransmitter/persistence_manager.go b/core/services/llo/mercurytransmitter/persistence_manager.go new file mode 100644 index 00000000000..eb36a7d1b80 --- /dev/null +++ b/core/services/llo/mercurytransmitter/persistence_manager.go @@ -0,0 +1,143 @@ +package mercurytransmitter + +import ( + "context" + "sync" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" +) + +var ( + flushDeletesFrequency = time.Second + pruneFrequency = time.Hour +) + +// persistenceManager scopes an ORM to a single serverURL and handles cleanup +// and asynchronous deletion +type persistenceManager struct { + lggr logger.Logger + orm ORM + serverURL string + + once services.StateMachine + stopCh services.StopChan + wg sync.WaitGroup + + deleteMu sync.Mutex + deleteQueue [][32]byte + + maxTransmitQueueSize int + flushDeletesFrequency time.Duration + pruneFrequency time.Duration +} + +func NewPersistenceManager(lggr logger.Logger, orm ORM, serverURL string, maxTransmitQueueSize int, flushDeletesFrequency, pruneFrequency time.Duration) *persistenceManager { + return &persistenceManager{ + orm: orm, + lggr: logger.Sugared(lggr).Named("LLOPersistenceManager").With("serverURL", serverURL), + serverURL: serverURL, + stopCh: make(services.StopChan), + maxTransmitQueueSize: maxTransmitQueueSize, + flushDeletesFrequency: flushDeletesFrequency, + pruneFrequency: pruneFrequency, + } +} + +func (pm *persistenceManager) Start(ctx context.Context) error { + return pm.once.StartOnce("LLOMercuryPersistenceManager", func() error { + pm.wg.Add(2) + go pm.runFlushDeletesLoop() + go pm.runPruneLoop() + return nil + }) +} + +func (pm *persistenceManager) Close() error { + return pm.once.StopOnce("LLOMercuryPersistenceManager", func() error { + close(pm.stopCh) + pm.wg.Wait() + return nil + }) +} + +func (pm *persistenceManager) DonID() uint32 { + return pm.orm.DonID() +} + +func (pm *persistenceManager) AsyncDelete(hash [32]byte) { + pm.addToDeleteQueue(hash) +} + +func (pm *persistenceManager) Load(ctx context.Context) ([]*Transmission, error) { + return pm.orm.Get(ctx, pm.serverURL) +} + +func (pm *persistenceManager) runFlushDeletesLoop() { + defer pm.wg.Done() + + ctx, cancel := pm.stopCh.Ctx(context.Background()) + defer cancel() + + ticker := services.NewTicker(pm.flushDeletesFrequency) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + queuedTransmissionHashes := pm.resetDeleteQueue() + if len(queuedTransmissionHashes) == 0 { + continue + } + if err := pm.orm.Delete(ctx, queuedTransmissionHashes); err != nil { + pm.lggr.Errorw("Failed to delete queued transmit requests", "err", err) + pm.addToDeleteQueue(queuedTransmissionHashes...) + } else { + pm.lggr.Debugw("Deleted queued transmit requests") + } + } + } +} + +func (pm *persistenceManager) runPruneLoop() { + defer pm.wg.Done() + + ctx, cancel := pm.stopCh.NewCtx() + defer cancel() + + ticker := services.NewTicker(pm.pruneFrequency) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + func(ctx context.Context) { + ctx, cancelPrune := context.WithTimeout(sqlutil.WithoutDefaultTimeout(ctx), time.Minute) + defer cancelPrune() + if err := pm.orm.Prune(ctx, pm.serverURL, pm.maxTransmitQueueSize); err != nil { + pm.lggr.Errorw("Failed to prune transmit requests table", "err", err) + } else { + pm.lggr.Debugw("Pruned transmit requests table") + } + }(ctx) + } + } +} + +func (pm *persistenceManager) addToDeleteQueue(hashes ...[32]byte) { + pm.deleteMu.Lock() + defer pm.deleteMu.Unlock() + pm.deleteQueue = append(pm.deleteQueue, hashes...) +} + +func (pm *persistenceManager) resetDeleteQueue() [][32]byte { + pm.deleteMu.Lock() + defer pm.deleteMu.Unlock() + queue := pm.deleteQueue + pm.deleteQueue = nil + return queue +} diff --git a/core/services/llo/mercurytransmitter/persistence_manager_test.go b/core/services/llo/mercurytransmitter/persistence_manager_test.go new file mode 100644 index 00000000000..b05de429ade --- /dev/null +++ b/core/services/llo/mercurytransmitter/persistence_manager_test.go @@ -0,0 +1,127 @@ +package mercurytransmitter + +import ( + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +func bootstrapPersistenceManager(t *testing.T, donID uint32, db *sqlx.DB) (*persistenceManager, *observer.ObservedLogs) { + t.Helper() + lggr, observedLogs := logger.TestLoggerObserved(t, zapcore.DebugLevel) + orm := NewORM(db, donID) + return NewPersistenceManager(lggr, orm, "wss://example.com/mercury", 2, 5*time.Millisecond, 5*time.Millisecond), observedLogs +} + +func TestPersistenceManager(t *testing.T) { + donID1 := uint32(1234) + donID2 := uint32(2345) + + ctx := testutils.Context(t) + db := pgtest.NewSqlxDB(t) + pm, _ := bootstrapPersistenceManager(t, donID1, db) + + transmissions := makeSampleTransmissions() + err := pm.orm.Insert(ctx, transmissions) + require.NoError(t, err) + + result, err := pm.Load(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, transmissions, result) + + err = pm.orm.Delete(ctx, [][32]byte{transmissions[0].Hash()}) + require.NoError(t, err) + + t.Run("scopes load to only transmissions with matching don ID", func(t *testing.T) { + pm2, _ := bootstrapPersistenceManager(t, donID2, db) + result, err = pm2.Load(ctx) + require.NoError(t, err) + + assert.Len(t, result, 0) + }) +} + +func TestPersistenceManagerAsyncDelete(t *testing.T) { + ctx := testutils.Context(t) + donID := uint32(1234) + db := pgtest.NewSqlxDB(t) + pm, observedLogs := bootstrapPersistenceManager(t, donID, db) + + transmissions := makeSampleTransmissions() + err := pm.orm.Insert(ctx, transmissions) + require.NoError(t, err) + + servicetest.Run(t, pm) + + pm.AsyncDelete(transmissions[0].Hash()) + + // Wait for next poll. + observedLogs.TakeAll() + testutils.WaitForLogMessage(t, observedLogs, "Deleted queued transmit requests") + + result, err := pm.Load(ctx) + require.NoError(t, err) + require.Len(t, result, 2) + assert.ElementsMatch(t, transmissions[1:], result) +} + +func TestPersistenceManagerPrune(t *testing.T) { + donID1 := uint32(123456) + donID2 := uint32(654321) + db := pgtest.NewSqlxDB(t) + + ctx := testutils.Context(t) + + transmissions := make([]*Transmission, 45) + for i := uint64(0); i < 45; i++ { + transmissions[i] = makeSampleTransmission(i) + } + + pm, _ := bootstrapPersistenceManager(t, donID1, db) + err := pm.orm.Insert(ctx, transmissions[:25]) + require.NoError(t, err) + + pm2, _ := bootstrapPersistenceManager(t, donID2, db) + err = pm2.orm.Insert(ctx, transmissions[25:]) + require.NoError(t, err) + + pm, observedLogs := bootstrapPersistenceManager(t, donID1, db) + + err = pm.Start(ctx) + require.NoError(t, err) + + // Wait for next poll. + observedLogs.TakeAll() + testutils.WaitForLogMessage(t, observedLogs, "Pruned transmit requests table") + + result, err := pm.Load(ctx) + require.NoError(t, err) + require.ElementsMatch(t, transmissions[23:25], result) + + // Test pruning stops after Close. + err = pm.Close() + require.NoError(t, err) + + err = pm.orm.Insert(ctx, transmissions) + require.NoError(t, err) + + result, err = pm.Load(ctx) + require.NoError(t, err) + require.Len(t, result, 25) + + t.Run("prune was scoped to don ID", func(t *testing.T) { + result, err = pm2.Load(ctx) + require.NoError(t, err) + assert.Len(t, result, 20) + }) +} diff --git a/core/services/llo/mercurytransmitter/queue.go b/core/services/llo/mercurytransmitter/queue.go new file mode 100644 index 00000000000..a5a606c5b32 --- /dev/null +++ b/core/services/llo/mercurytransmitter/queue.go @@ -0,0 +1,250 @@ +package mercurytransmitter + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + heap "github.com/esote/minmaxheap" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" +) + +type asyncDeleter interface { + AsyncDelete(hash [32]byte) + DonID() uint32 +} + +var _ services.Service = (*transmitQueue)(nil) + +var transmitQueueLoad = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "llo_transmit_queue_load", + Help: "Current count of items in the transmit queue", +}, + []string{"donID", "serverURL", "capacity"}, +) + +// Prometheus' default interval is 15s, set this to under 7.5s to avoid +// aliasing (see: https://en.wikipedia.org/wiki/Nyquist_frequency) +const promInterval = 6500 * time.Millisecond + +// TransmitQueue is the high-level package that everything outside of this file should be using +// It stores pending transmissions, yielding the latest (highest priority) first to the caller +type transmitQueue struct { + services.StateMachine + + cond sync.Cond + lggr logger.SugaredLogger + asyncDeleter asyncDeleter + mu *sync.RWMutex + + pq *priorityQueue + maxlen int + closed bool + + // monitor loop + stopMonitor func() + transmitQueueLoad prometheus.Gauge +} + +type TransmitQueue interface { + services.Service + + BlockingPop() (t *Transmission) + Push(t *Transmission) (ok bool) + Init(ts []*Transmission) + IsEmpty() bool +} + +// maxlen controls how many items will be stored in the queue +// 0 means unlimited - be careful, this can cause memory leaks +func NewTransmitQueue(lggr logger.Logger, serverURL string, maxlen int, asyncDeleter asyncDeleter) TransmitQueue { + mu := new(sync.RWMutex) + return &transmitQueue{ + services.StateMachine{}, + sync.Cond{L: mu}, + logger.Sugared(lggr).Named("TransmitQueue"), + asyncDeleter, + mu, + nil, // pq needs to be initialized by calling tq.Init before use + maxlen, + false, + nil, + transmitQueueLoad.WithLabelValues(fmt.Sprintf("%d", asyncDeleter.DonID()), serverURL, fmt.Sprintf("%d", maxlen)), + } +} + +func (tq *transmitQueue) Init(ts []*Transmission) { + pq := priorityQueue(ts) + heap.Init(&pq) // ensure the heap is ordered + tq.pq = &pq +} + +func (tq *transmitQueue) Push(t *Transmission) (ok bool) { + tq.cond.L.Lock() + defer tq.cond.L.Unlock() + + if tq.closed { + return false + } + + if tq.maxlen != 0 && tq.pq.Len() == tq.maxlen { + // evict oldest entry to make room + tq.lggr.Criticalf("Transmit queue is full; dropping oldest transmission (reached max length of %d)", tq.maxlen) + removed := heap.PopMax(tq.pq) + if removed, ok := removed.(*Transmission); ok { + tq.asyncDeleter.AsyncDelete(removed.Hash()) + } + } + + heap.Push(tq.pq, t) + tq.cond.Signal() + + return true +} + +// BlockingPop will block until at least one item is in the heap, and then return it +// If the queue is closed, it will immediately return nil +func (tq *transmitQueue) BlockingPop() (t *Transmission) { + tq.cond.L.Lock() + defer tq.cond.L.Unlock() + if tq.closed { + return nil + } + for t = tq.pop(); t == nil; t = tq.pop() { + tq.cond.Wait() + if tq.closed { + return nil + } + } + return t +} + +func (tq *transmitQueue) IsEmpty() bool { + tq.mu.RLock() + defer tq.mu.RUnlock() + return tq.pq.Len() == 0 +} + +func (tq *transmitQueue) Start(context.Context) error { + return tq.StartOnce("TransmitQueue", func() error { + t := services.NewTicker(promInterval) + wg := new(sync.WaitGroup) + chStop := make(chan struct{}) + tq.stopMonitor = func() { + t.Stop() + close(chStop) + wg.Wait() + } + wg.Add(1) + go tq.monitorLoop(t.C, chStop, wg) + return nil + }) +} + +func (tq *transmitQueue) Close() error { + return tq.StopOnce("TransmitQueue", func() error { + tq.cond.L.Lock() + tq.closed = true + tq.cond.L.Unlock() + tq.cond.Broadcast() + tq.stopMonitor() + return nil + }) +} + +func (tq *transmitQueue) monitorLoop(c <-chan time.Time, chStop <-chan struct{}, wg *sync.WaitGroup) { + defer wg.Done() + + for { + select { + case <-c: + tq.report() + case <-chStop: + return + } + } +} + +func (tq *transmitQueue) report() { + tq.mu.RLock() + length := tq.pq.Len() + tq.mu.RUnlock() + tq.transmitQueueLoad.Set(float64(length)) +} + +func (tq *transmitQueue) Ready() error { + return nil +} +func (tq *transmitQueue) Name() string { return tq.lggr.Name() } +func (tq *transmitQueue) HealthReport() map[string]error { + report := map[string]error{tq.Name(): errors.Join( + tq.status(), + )} + return report +} + +func (tq *transmitQueue) status() (merr error) { + tq.mu.RLock() + length := tq.pq.Len() + closed := tq.closed + tq.mu.RUnlock() + if tq.maxlen != 0 && length > (tq.maxlen/2) { + merr = errors.Join(merr, fmt.Errorf("transmit priority queue is greater than 50%% full (%d/%d)", length, tq.maxlen)) + } + if closed { + merr = errors.New("transmit queue is closed") + } + return merr +} + +// pop latest Transmission from the heap +// Not thread-safe +func (tq *transmitQueue) pop() *Transmission { + if tq.pq.Len() == 0 { + return nil + } + return heap.Pop(tq.pq).(*Transmission) +} + +// HEAP +// Adapted from https://pkg.go.dev/container/heap#example-package-PriorityQueue + +// WARNING: None of these methods are thread-safe, caller must synchronize + +var _ heap.Interface = &priorityQueue{} + +type priorityQueue []*Transmission + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + // We want Pop to give us the latest round, so we use greater than here + // i.e. a later seqNr is "less" than an earlier one + return pq[i].SeqNr > pq[j].SeqNr +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] +} + +func (pq *priorityQueue) Pop() any { + n := len(*pq) + if n == 0 { + return nil + } + old := *pq + item := old[n-1] + old[n-1] = nil // avoid memory leak + *pq = old[0 : n-1] + return item +} + +func (pq *priorityQueue) Push(x any) { + *pq = append(*pq, x.(*Transmission)) +} diff --git a/core/services/llo/mercurytransmitter/queue_test.go b/core/services/llo/mercurytransmitter/queue_test.go new file mode 100644 index 00000000000..d19d140d6be --- /dev/null +++ b/core/services/llo/mercurytransmitter/queue_test.go @@ -0,0 +1,105 @@ +package mercurytransmitter + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +var _ asyncDeleter = &mockAsyncDeleter{} + +type mockAsyncDeleter struct { + donID uint32 + hashes [][32]byte +} + +func (m *mockAsyncDeleter) AsyncDelete(hash [32]byte) { + m.hashes = append(m.hashes, hash) +} +func (m *mockAsyncDeleter) DonID() uint32 { + return m.donID +} + +func Test_Queue(t *testing.T) { + t.Parallel() + lggr, observedLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) + testTransmissions := makeSampleTransmissions() + deleter := &mockAsyncDeleter{} + transmitQueue := NewTransmitQueue(lggr, sURL, 7, deleter) + transmitQueue.Init([]*Transmission{}) + + t.Run("successfully add transmissions to transmit queue", func(t *testing.T) { + for _, tt := range testTransmissions { + ok := transmitQueue.Push(tt) + require.True(t, ok) + } + report := transmitQueue.HealthReport() + assert.Nil(t, report[transmitQueue.Name()]) + }) + + t.Run("transmit queue is more than 50% full", func(t *testing.T) { + transmitQueue.Push(testTransmissions[2]) + report := transmitQueue.HealthReport() + assert.Equal(t, report[transmitQueue.Name()].Error(), "transmit priority queue is greater than 50% full (4/7)") + }) + + t.Run("transmit queue pops the highest priority transmission", func(t *testing.T) { + tr := transmitQueue.BlockingPop() + assert.Equal(t, testTransmissions[2], tr) + }) + + t.Run("transmit queue is full and evicts the oldest transmission", func(t *testing.T) { + // add 5 more transmissions to overflow the queue by 1 + for i := 0; i < 5; i++ { + transmitQueue.Push(testTransmissions[1]) + } + + // expecting testTransmissions[0] to get evicted and not present in the queue anymore + testutils.WaitForLogMessage(t, observedLogs, "Transmit queue is full; dropping oldest transmission (reached max length of 7)") + var transmissions []*Transmission + for i := 0; i < 7; i++ { + tr := transmitQueue.BlockingPop() + transmissions = append(transmissions, tr) + } + + assert.NotContains(t, transmissions, testTransmissions[0]) + require.Len(t, deleter.hashes, 1) + assert.Equal(t, testTransmissions[0].Hash(), deleter.hashes[0]) + }) + + t.Run("transmit queue blocks when empty and resumes when tranmission available", func(t *testing.T) { + assert.True(t, transmitQueue.IsEmpty()) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + tr := transmitQueue.BlockingPop() + assert.Equal(t, tr, testTransmissions[0]) + }() + go func() { + defer wg.Done() + transmitQueue.Push(testTransmissions[0]) + }() + wg.Wait() + }) + + t.Run("initializes transmissions", func(t *testing.T) { + expected := makeSampleTransmission(1) + transmissions := []*Transmission{ + expected, + } + transmitQueue := NewTransmitQueue(lggr, sURL, 7, deleter) + transmitQueue.Init(transmissions) + + transmission := transmitQueue.BlockingPop() + assert.Equal(t, expected, transmission) + assert.True(t, transmitQueue.IsEmpty()) + }) +} diff --git a/core/services/llo/mercurytransmitter/server.go b/core/services/llo/mercurytransmitter/server.go new file mode 100644 index 00000000000..d025e65efa6 --- /dev/null +++ b/core/services/llo/mercurytransmitter/server.go @@ -0,0 +1,240 @@ +package mercurytransmitter + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/jpillora/backoff" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + + "github.com/smartcontractkit/chainlink/v2/core/services/llo/evm" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" + "github.com/smartcontractkit/chainlink/v2/core/utils" +) + +var ( + transmitQueueDeleteErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_queue_delete_error_count", + Help: "Running count of DB errors when trying to delete an item from the queue DB", + }, + []string{"donID", "serverURL"}, + ) + transmitQueueInsertErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_queue_insert_error_count", + Help: "Running count of DB errors when trying to insert an item into the queue DB", + }, + []string{"donID", "serverURL"}, + ) + transmitQueuePushErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_queue_push_error_count", + Help: "Running count of DB errors when trying to push an item onto the queue", + }, + []string{"donID", "serverURL"}, + ) + transmitServerErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_server_error_count", + Help: "Number of errored transmissions that failed due to an error returned by the mercury server", + }, + []string{"donID", "serverURL", "code"}, + ) +) + +// A server handles the queue for a given mercury server + +type server struct { + lggr logger.SugaredLogger + + transmitTimeout time.Duration + + c wsrpc.Client + pm *persistenceManager + q TransmitQueue + + deleteQueue chan [32]byte + + url string + + transmitSuccessCount prometheus.Counter + transmitDuplicateCount prometheus.Counter + transmitConnectionErrorCount prometheus.Counter + transmitQueueDeleteErrorCount prometheus.Counter + transmitQueueInsertErrorCount prometheus.Counter + transmitQueuePushErrorCount prometheus.Counter +} + +type QueueConfig interface { + TransmitQueueMaxSize() uint32 + TransmitTimeout() commonconfig.Duration +} + +func newServer(lggr logger.Logger, cfg QueueConfig, client wsrpc.Client, orm ORM, serverURL string) *server { + pm := NewPersistenceManager(lggr, orm, serverURL, int(cfg.TransmitQueueMaxSize()), flushDeletesFrequency, pruneFrequency) + donIDStr := fmt.Sprintf("%d", pm.DonID()) + return &server{ + logger.Sugared(lggr), + cfg.TransmitTimeout().Duration(), + client, + pm, + NewTransmitQueue(lggr, serverURL, int(cfg.TransmitQueueMaxSize()), pm), + make(chan [32]byte, int(cfg.TransmitQueueMaxSize())), + serverURL, + transmitSuccessCount.WithLabelValues(donIDStr, serverURL), + transmitDuplicateCount.WithLabelValues(donIDStr, serverURL), + transmitConnectionErrorCount.WithLabelValues(donIDStr, serverURL), + transmitQueueDeleteErrorCount.WithLabelValues(donIDStr, serverURL), + transmitQueueInsertErrorCount.WithLabelValues(donIDStr, serverURL), + transmitQueuePushErrorCount.WithLabelValues(donIDStr, serverURL), + } +} + +func (s *server) HealthReport() map[string]error { + report := map[string]error{} + services.CopyHealth(report, s.c.HealthReport()) + services.CopyHealth(report, s.q.HealthReport()) + return report +} + +func (s *server) runDeleteQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup) { + defer wg.Done() + runloopCtx, cancel := stopCh.Ctx(context.Background()) + defer cancel() + + // Exponential backoff for very rarely occurring errors (DB disconnect etc) + b := backoff.Backoff{ + Min: 1 * time.Second, + Max: 120 * time.Second, + Factor: 2, + Jitter: true, + } + + for { + select { + case hash := <-s.deleteQueue: + for { + if err := s.pm.orm.Delete(runloopCtx, [][32]byte{hash}); err != nil { + s.lggr.Errorw("Failed to delete transmission record", "err", err, "transmissionHash", hash) + s.transmitQueueDeleteErrorCount.Inc() + select { + case <-time.After(b.Duration()): + // Wait a backoff duration before trying to delete again + continue + case <-stopCh: + // abort and return immediately on stop even if items remain in queue + return + } + } + break + } + // success + b.Reset() + case <-stopCh: + // abort and return immediately on stop even if items remain in queue + return + } + } +} + +func (s *server) runQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup, donIDStr string) { + defer wg.Done() + // Exponential backoff with very short retry interval (since latency is a priority) + // 5ms, 10ms, 20ms, 40ms etc + b := backoff.Backoff{ + Min: 5 * time.Millisecond, + Max: 1 * time.Second, + Factor: 2, + Jitter: true, + } + runloopCtx, cancel := stopCh.Ctx(context.Background()) + defer cancel() + for { + t := s.q.BlockingPop() + if t == nil { + // queue was closed + return + } + ctx, cancel := context.WithTimeout(runloopCtx, utils.WithJitter(s.transmitTimeout)) + res, err := s.transmit(ctx, t) + cancel() + if runloopCtx.Err() != nil { + // runloop context is only canceled on transmitter close so we can + // exit the runloop here + return + } else if err != nil { + s.transmitConnectionErrorCount.Inc() + s.lggr.Errorw("Transmit report failed", "err", err, "transmission", t) + if ok := s.q.Push(t); !ok { + s.lggr.Error("Failed to push report to transmit queue; queue is closed") + return + } + // Wait a backoff duration before pulling the most recent transmission + // the heap + select { + case <-time.After(b.Duration()): + continue + case <-stopCh: + return + } + } + + b.Reset() + if res.Error == "" { + s.transmitSuccessCount.Inc() + s.lggr.Debugw("Transmit report success", "transmission", t, "response", res) + } else { + // We don't need to retry here because the mercury server + // has confirmed it received the report. We only need to retry + // on networking/unknown errors + switch res.Code { + case DuplicateReport: + s.transmitSuccessCount.Inc() + s.transmitDuplicateCount.Inc() + s.lggr.Debugw("Transmit report success; duplicate report", "transmission", t, "response", res) + default: + transmitServerErrorCount.WithLabelValues(donIDStr, s.url, fmt.Sprintf("%d", res.Code)).Inc() + s.lggr.Errorw("Transmit report failed; mercury server returned error", "response", res, "transmission", t, "err", res.Error, "code", res.Code) + } + } + + select { + case s.deleteQueue <- t.Hash(): + default: + s.lggr.Criticalw("Delete queue is full", "transmission", t) + } + } +} + +func (s *server) transmit(ctx context.Context, t *Transmission) (*pb.TransmitResponse, error) { + var payload []byte + var err error + + switch t.Report.Info.ReportFormat { + case llotypes.ReportFormatJSON: + // TODO: exactly how to handle JSON here? + // https://smartcontract-it.atlassian.net/browse/MERC-3659 + fallthrough + case llotypes.ReportFormatEVMPremiumLegacy: + payload, err = evm.ReportCodecPremiumLegacy{}.Pack(t.ConfigDigest, t.SeqNr, t.Report.Report, t.Sigs) + default: + return nil, fmt.Errorf("Transmit failed; unsupported report format: %q", t.Report.Info.ReportFormat) + } + + if err != nil { + return nil, fmt.Errorf("Transmit: encode failed; %w", err) + } + + req := &pb.TransmitRequest{ + Payload: payload, + ReportFormat: uint32(t.Report.Info.ReportFormat), + } + + return s.c.Transmit(ctx, req) +} diff --git a/core/services/llo/mercurytransmitter/transmitter.go b/core/services/llo/mercurytransmitter/transmitter.go new file mode 100644 index 00000000000..dc032a054dc --- /dev/null +++ b/core/services/llo/mercurytransmitter/transmitter.go @@ -0,0 +1,254 @@ +package mercurytransmitter + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "golang.org/x/sync/errgroup" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/services" + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" +) + +const ( + // Mercury server error codes + DuplicateReport = 2 +) + +var ( + transmitSuccessCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_success_count", + Help: "Number of successful transmissions (duplicates are counted as success)", + }, + []string{"donID", "serverURL"}, + ) + transmitDuplicateCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_duplicate_count", + Help: "Number of transmissions where the server told us it was a duplicate", + }, + []string{"donID", "serverURL"}, + ) + transmitConnectionErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_mercury_transmit_connection_error_count", + Help: "Number of errored transmissions that failed due to problem with the connection", + }, + []string{"donID", "serverURL"}, + ) +) + +type Transmission struct { + ServerURL string + ConfigDigest types.ConfigDigest + SeqNr uint64 + Report ocr3types.ReportWithInfo[llotypes.ReportInfo] + Sigs []types.AttributedOnchainSignature +} + +// Hash takes sha256 hash of all fields +func (t Transmission) Hash() [32]byte { + h := sha256.New() + h.Write([]byte(t.ServerURL)) + h.Write(t.ConfigDigest[:]) + if err := binary.Write(h, binary.BigEndian, t.SeqNr); err != nil { + // This should never happen + panic(err) + } + h.Write(t.Report.Report) + h.Write([]byte(t.Report.Info.LifeCycleStage)) + if err := binary.Write(h, binary.BigEndian, t.Report.Info.ReportFormat); err != nil { + // This should never happen + panic(err) + } + for _, sig := range t.Sigs { + h.Write(sig.Signature) + if err := binary.Write(h, binary.BigEndian, sig.Signer); err != nil { + // This should never happen + panic(err) + } + } + var result [32]byte + h.Sum(result[:0]) + return result +} + +type Transmitter interface { + llotypes.Transmitter + services.Service +} + +var _ Transmitter = (*transmitter)(nil) + +type Config interface { + TransmitQueueMaxSize() uint32 + TransmitTimeout() commonconfig.Duration +} + +type transmitter struct { + services.StateMachine + lggr logger.SugaredLogger + cfg Config + + orm ORM + servers map[string]*server + + donID uint32 + fromAccount string + + stopCh services.StopChan + wg *sync.WaitGroup +} + +type Opts struct { + Lggr logger.Logger + Cfg Config + Clients map[string]wsrpc.Client + FromAccount ed25519.PublicKey + DonID uint32 + ORM ORM +} + +func New(opts Opts) Transmitter { + return newTransmitter(opts) +} + +func newTransmitter(opts Opts) *transmitter { + sugared := logger.Sugared(opts.Lggr).Named("LLOMercuryTransmitter") + servers := make(map[string]*server, len(opts.Clients)) + for serverURL, client := range opts.Clients { + sLggr := sugared.Named(serverURL).With("serverURL", serverURL) + servers[serverURL] = newServer(sLggr, opts.Cfg, client, opts.ORM, serverURL) + } + return &transmitter{ + services.StateMachine{}, + sugared.Named("LLOMercuryTransmitter").With("donID", opts.ORM.DonID()), + opts.Cfg, + opts.ORM, + servers, + opts.DonID, + fmt.Sprintf("%x", opts.FromAccount), + make(services.StopChan), + &sync.WaitGroup{}, + } +} + +func (mt *transmitter) Start(ctx context.Context) (err error) { + return mt.StartOnce("LLOMercuryTransmitter", func() error { + mt.lggr.Debugw("Loading transmit requests from database") + + { + var startClosers []services.StartClose + for _, s := range mt.servers { + transmissions, err := s.pm.Load(ctx) + if err != nil { + return err + } + s.q.Init(transmissions) + // starting pm after loading from it is fine because it simply spawns some garbage collection/prune goroutines + startClosers = append(startClosers, s.c, s.q, s.pm) + + mt.wg.Add(2) + go s.runDeleteQueueLoop(mt.stopCh, mt.wg) + go s.runQueueLoop(mt.stopCh, mt.wg, fmt.Sprintf("%d", mt.donID)) + } + if err := (&services.MultiStart{}).Start(ctx, startClosers...); err != nil { + return err + } + } + + return nil + }) +} + +func (mt *transmitter) Close() error { + return mt.StopOnce("LLOMercuryTransmitter", func() error { + // Drain all the queues first + var qs []io.Closer + for _, s := range mt.servers { + qs = append(qs, s.q) + } + if err := services.CloseAll(qs...); err != nil { + return err + } + + close(mt.stopCh) + mt.wg.Wait() + + // Close all the persistence managers + // Close all the clients + var closers []io.Closer + for _, s := range mt.servers { + closers = append(closers, s.pm) + closers = append(closers, s.c) + } + return services.CloseAll(closers...) + }) +} + +func (mt *transmitter) Name() string { return mt.lggr.Name() } + +func (mt *transmitter) HealthReport() map[string]error { + report := map[string]error{mt.Name(): mt.Healthy()} + for _, s := range mt.servers { + services.CopyHealth(report, s.HealthReport()) + } + return report +} + +// Transmit enqueues the report for transmission to the Mercury servers +func (mt *transmitter) Transmit( + ctx context.Context, + digest types.ConfigDigest, + seqNr uint64, + report ocr3types.ReportWithInfo[llotypes.ReportInfo], + sigs []types.AttributedOnchainSignature) error { + transmissions := make([]*Transmission, 0, len(mt.servers)) + for serverURL := range mt.servers { + transmissions = append(transmissions, &Transmission{ + ServerURL: serverURL, + ConfigDigest: digest, + SeqNr: seqNr, + Report: report, + Sigs: sigs, + }) + } + if err := mt.orm.Insert(ctx, transmissions); err != nil { + return err + } + + g := new(errgroup.Group) + for i := range transmissions { + t := transmissions[i] + mt.lggr.Debugw("LLOMercuryTransmit", "digest", digest.Hex(), "seqNr", seqNr, "reportFormat", report.Info.ReportFormat, "reportLifeCycleStage", report.Info.LifeCycleStage, "transmissionHash", t.Hash()) + g.Go(func() error { + s := mt.servers[t.ServerURL] + if ok := s.q.Push(t); !ok { + s.transmitQueuePushErrorCount.Inc() + return errors.New("transmit queue is closed") + } + return nil + }) + } + + return g.Wait() +} + +// FromAccount returns the stringified (hex) CSA public key +func (mt *transmitter) FromAccount() (ocrtypes.Account, error) { + return ocrtypes.Account(mt.fromAccount), nil +} diff --git a/core/services/llo/mercurytransmitter/transmitter_test.go b/core/services/llo/mercurytransmitter/transmitter_test.go new file mode 100644 index 00000000000..db3d0d2e584 --- /dev/null +++ b/core/services/llo/mercurytransmitter/transmitter_test.go @@ -0,0 +1,258 @@ +package mercurytransmitter + +import ( + "context" + "crypto/ed25519" + "sync" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/mocks" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" +) + +type mockCfg struct{} + +func (m mockCfg) TransmitQueueMaxSize() uint32 { + return 10_000 +} + +func (m mockCfg) TransmitTimeout() commonconfig.Duration { + return *commonconfig.MustNewDuration(1 * time.Hour) +} + +func Test_Transmitter_Transmit(t *testing.T) { + lggr := logger.TestLogger(t) + db := pgtest.NewSqlxDB(t) + donID := uint32(123456) + orm := NewORM(db, donID) + clients := map[string]wsrpc.Client{} + + t.Run("with multiple mercury servers", func(t *testing.T) { + t.Run("transmission successfully enqueued", func(t *testing.T) { + c := &mocks.MockWSRPCClient{} + clients[sURL] = c + clients[sURL2] = c + clients[sURL3] = c + + mt := newTransmitter(Opts{ + Lggr: lggr, + Cfg: mockCfg{}, + Clients: clients, + FromAccount: ed25519.PublicKey{}, + DonID: donID, + ORM: orm, + }) + // init the queue since we skipped starting transmitter + mt.servers[sURL].q.Init([]*Transmission{}) + mt.servers[sURL2].q.Init([]*Transmission{}) + mt.servers[sURL3].q.Init([]*Transmission{}) + + seqNr := uint64(55) + report := makeSampleReport() + digest := makeSampleConfigDigest() + sigs := []types.AttributedOnchainSignature{{ + Signature: []byte{22}, + Signer: commontypes.OracleID(43), + }} + err := mt.Transmit(testutils.Context(t), digest, seqNr, report, sigs) + require.NoError(t, err) + + // ensure it was added to the queue + require.Equal(t, mt.servers[sURL].q.(*transmitQueue).pq.Len(), 1) + assert.Equal(t, &Transmission{ + ServerURL: sURL, + ConfigDigest: digest, + SeqNr: seqNr, + Report: report, + Sigs: sigs, + }, mt.servers[sURL].q.(*transmitQueue).pq.Pop().(*Transmission)) + require.Equal(t, mt.servers[sURL2].q.(*transmitQueue).pq.Len(), 1) + assert.Equal(t, &Transmission{ + ServerURL: sURL2, + ConfigDigest: digest, + SeqNr: seqNr, + Report: report, + Sigs: sigs, + }, mt.servers[sURL2].q.(*transmitQueue).pq.Pop().(*Transmission)) + require.Equal(t, mt.servers[sURL3].q.(*transmitQueue).pq.Len(), 1) + assert.Equal(t, &Transmission{ + ServerURL: sURL3, + ConfigDigest: digest, + SeqNr: seqNr, + Report: report, + Sigs: sigs, + }, mt.servers[sURL3].q.(*transmitQueue).pq.Pop().(*Transmission)) + }) + }) +} + +type mockQ struct { + ch chan *Transmission +} + +func newMockQ() *mockQ { + return &mockQ{make(chan *Transmission, 100)} +} + +func (m *mockQ) Start(context.Context) error { return nil } +func (m *mockQ) Close() error { + m.ch <- nil + return nil +} +func (m *mockQ) Ready() error { return nil } +func (m *mockQ) HealthReport() map[string]error { return nil } +func (m *mockQ) Name() string { return "" } +func (m *mockQ) BlockingPop() (t *Transmission) { + val := <-m.ch + return val +} +func (m *mockQ) Push(t *Transmission) (ok bool) { + m.ch <- t + return true +} +func (m *mockQ) Init(transmissions []*Transmission) {} +func (m *mockQ) IsEmpty() bool { return false } + +func Test_Transmitter_runQueueLoop(t *testing.T) { + donIDStr := "555" + lggr := logger.TestLogger(t) + c := &mocks.MockWSRPCClient{} + db := pgtest.NewSqlxDB(t) + donID := uint32(123456) + orm := NewORM(db, donID) + cfg := mockCfg{} + + s := newServer(lggr, cfg, c, orm, sURL) + + t.Run("pulls from queue and transmits successfully", func(t *testing.T) { + transmit := make(chan *pb.TransmitRequest, 1) + c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { + transmit <- in + return &pb.TransmitResponse{Code: 0, Error: ""}, nil + } + q := newMockQ() + s.q = q + wg := &sync.WaitGroup{} + wg.Add(1) + + go s.runQueueLoop(nil, wg, donIDStr) + + transmission := makeSampleTransmission(1) + q.Push(transmission) + + select { + case tr := <-transmit: + assert.Equal(t, []byte{0x0, 0x9, 0x57, 0xdd, 0x2f, 0x63, 0x56, 0x69, 0x34, 0xfd, 0xc2, 0xe1, 0xcd, 0xc1, 0xe, 0x3e, 0x25, 0xb9, 0x26, 0x5a, 0x16, 0x23, 0x91, 0xa6, 0x53, 0x16, 0x66, 0x59, 0x51, 0x0, 0x28, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x80, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x20, 0x0, 0x3, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1e, 0x8e, 0x95, 0xcf, 0xb5, 0xd8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1a, 0xd0, 0x1c, 0x67, 0xa9, 0xcf, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xdf, 0x3, 0xca, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x1c, 0x93, 0x6d, 0xa4, 0xf2, 0x17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x14, 0x8d, 0x9a, 0xc1, 0xd9, 0x6f, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x40, 0x5c, 0xcf, 0xa1, 0xbc, 0x63, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9d, 0xab, 0x8f, 0xa7, 0xca, 0x7, 0x62, 0x57, 0xf7, 0x11, 0x2c, 0xb7, 0xf3, 0x49, 0x37, 0x12, 0xbd, 0xe, 0x14, 0x27, 0xfc, 0x32, 0x5c, 0xec, 0xa6, 0xb9, 0x7f, 0xf9, 0xd7, 0x7b, 0xa6, 0x36, 0x9a, 0x47, 0x4a, 0x3, 0x1a, 0x95, 0xcf, 0x46, 0x10, 0xaf, 0xcc, 0x90, 0x49, 0xb2, 0xce, 0xbf, 0x63, 0xaa, 0xc7, 0x25, 0x4d, 0x2a, 0x8, 0x36, 0xda, 0xd5, 0x9f, 0x9d, 0x63, 0x69, 0x22, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x30, 0x9d, 0x84, 0x29, 0xbf, 0xd4, 0xeb, 0xc5, 0xc9, 0x29, 0xef, 0xdd, 0xd3, 0x2f, 0xa6, 0x25, 0x63, 0xda, 0xd9, 0x2c, 0xa1, 0x4a, 0xba, 0x75, 0xb2, 0x85, 0x25, 0x8f, 0x2b, 0x84, 0xcd, 0x99, 0x36, 0xd9, 0x6e, 0xf, 0xae, 0x7b, 0xd1, 0x61, 0x59, 0xf, 0x36, 0x4a, 0x22, 0xec, 0xde, 0x45, 0x32, 0xe0, 0x5b, 0x5c, 0xe3, 0x14, 0x29, 0x4, 0x60, 0x7b, 0xce, 0xa3, 0x89, 0x6b, 0xbb, 0xe0}, tr.Payload) + assert.Equal(t, int(transmission.Report.Info.ReportFormat), int(tr.ReportFormat)) + case <-time.After(testutils.WaitTimeout(t)): + t.Fatal("expected a transmit request to be sent") + } + + q.Close() + wg.Wait() + }) + + t.Run("on duplicate, success", func(t *testing.T) { + transmit := make(chan *pb.TransmitRequest, 1) + c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { + transmit <- in + return &pb.TransmitResponse{Code: DuplicateReport, Error: ""}, nil + } + q := newMockQ() + s.q = q + wg := &sync.WaitGroup{} + wg.Add(1) + + go s.runQueueLoop(nil, wg, donIDStr) + + transmission := makeSampleTransmission(1) + q.Push(transmission) + + select { + case tr := <-transmit: + assert.Equal(t, []byte{0x0, 0x9, 0x57, 0xdd, 0x2f, 0x63, 0x56, 0x69, 0x34, 0xfd, 0xc2, 0xe1, 0xcd, 0xc1, 0xe, 0x3e, 0x25, 0xb9, 0x26, 0x5a, 0x16, 0x23, 0x91, 0xa6, 0x53, 0x16, 0x66, 0x59, 0x51, 0x0, 0x28, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x80, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x20, 0x0, 0x3, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1e, 0x8e, 0x95, 0xcf, 0xb5, 0xd8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1a, 0xd0, 0x1c, 0x67, 0xa9, 0xcf, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xdf, 0x3, 0xca, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x1c, 0x93, 0x6d, 0xa4, 0xf2, 0x17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x14, 0x8d, 0x9a, 0xc1, 0xd9, 0x6f, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x40, 0x5c, 0xcf, 0xa1, 0xbc, 0x63, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9d, 0xab, 0x8f, 0xa7, 0xca, 0x7, 0x62, 0x57, 0xf7, 0x11, 0x2c, 0xb7, 0xf3, 0x49, 0x37, 0x12, 0xbd, 0xe, 0x14, 0x27, 0xfc, 0x32, 0x5c, 0xec, 0xa6, 0xb9, 0x7f, 0xf9, 0xd7, 0x7b, 0xa6, 0x36, 0x9a, 0x47, 0x4a, 0x3, 0x1a, 0x95, 0xcf, 0x46, 0x10, 0xaf, 0xcc, 0x90, 0x49, 0xb2, 0xce, 0xbf, 0x63, 0xaa, 0xc7, 0x25, 0x4d, 0x2a, 0x8, 0x36, 0xda, 0xd5, 0x9f, 0x9d, 0x63, 0x69, 0x22, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x30, 0x9d, 0x84, 0x29, 0xbf, 0xd4, 0xeb, 0xc5, 0xc9, 0x29, 0xef, 0xdd, 0xd3, 0x2f, 0xa6, 0x25, 0x63, 0xda, 0xd9, 0x2c, 0xa1, 0x4a, 0xba, 0x75, 0xb2, 0x85, 0x25, 0x8f, 0x2b, 0x84, 0xcd, 0x99, 0x36, 0xd9, 0x6e, 0xf, 0xae, 0x7b, 0xd1, 0x61, 0x59, 0xf, 0x36, 0x4a, 0x22, 0xec, 0xde, 0x45, 0x32, 0xe0, 0x5b, 0x5c, 0xe3, 0x14, 0x29, 0x4, 0x60, 0x7b, 0xce, 0xa3, 0x89, 0x6b, 0xbb, 0xe0}, tr.Payload) + assert.Equal(t, int(transmission.Report.Info.ReportFormat), int(tr.ReportFormat)) + case <-time.After(testutils.WaitTimeout(t)): + t.Fatal("expected a transmit request to be sent") + } + + q.Close() + wg.Wait() + }) + t.Run("on server-side error, does not retry", func(t *testing.T) { + transmit := make(chan *pb.TransmitRequest, 1) + c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { + transmit <- in + return &pb.TransmitResponse{Code: DuplicateReport, Error: ""}, nil + } + q := newMockQ() + s.q = q + wg := &sync.WaitGroup{} + wg.Add(1) + + go s.runQueueLoop(nil, wg, donIDStr) + + transmission := makeSampleTransmission(1) + q.Push(transmission) + + select { + case tr := <-transmit: + assert.Equal(t, []byte{0x0, 0x9, 0x57, 0xdd, 0x2f, 0x63, 0x56, 0x69, 0x34, 0xfd, 0xc2, 0xe1, 0xcd, 0xc1, 0xe, 0x3e, 0x25, 0xb9, 0x26, 0x5a, 0x16, 0x23, 0x91, 0xa6, 0x53, 0x16, 0x66, 0x59, 0x51, 0x0, 0x28, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x80, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x20, 0x0, 0x3, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1e, 0x8e, 0x95, 0xcf, 0xb5, 0xd8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1a, 0xd0, 0x1c, 0x67, 0xa9, 0xcf, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xdf, 0x3, 0xca, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x1c, 0x93, 0x6d, 0xa4, 0xf2, 0x17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x14, 0x8d, 0x9a, 0xc1, 0xd9, 0x6f, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x40, 0x5c, 0xcf, 0xa1, 0xbc, 0x63, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9d, 0xab, 0x8f, 0xa7, 0xca, 0x7, 0x62, 0x57, 0xf7, 0x11, 0x2c, 0xb7, 0xf3, 0x49, 0x37, 0x12, 0xbd, 0xe, 0x14, 0x27, 0xfc, 0x32, 0x5c, 0xec, 0xa6, 0xb9, 0x7f, 0xf9, 0xd7, 0x7b, 0xa6, 0x36, 0x9a, 0x47, 0x4a, 0x3, 0x1a, 0x95, 0xcf, 0x46, 0x10, 0xaf, 0xcc, 0x90, 0x49, 0xb2, 0xce, 0xbf, 0x63, 0xaa, 0xc7, 0x25, 0x4d, 0x2a, 0x8, 0x36, 0xda, 0xd5, 0x9f, 0x9d, 0x63, 0x69, 0x22, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x30, 0x9d, 0x84, 0x29, 0xbf, 0xd4, 0xeb, 0xc5, 0xc9, 0x29, 0xef, 0xdd, 0xd3, 0x2f, 0xa6, 0x25, 0x63, 0xda, 0xd9, 0x2c, 0xa1, 0x4a, 0xba, 0x75, 0xb2, 0x85, 0x25, 0x8f, 0x2b, 0x84, 0xcd, 0x99, 0x36, 0xd9, 0x6e, 0xf, 0xae, 0x7b, 0xd1, 0x61, 0x59, 0xf, 0x36, 0x4a, 0x22, 0xec, 0xde, 0x45, 0x32, 0xe0, 0x5b, 0x5c, 0xe3, 0x14, 0x29, 0x4, 0x60, 0x7b, 0xce, 0xa3, 0x89, 0x6b, 0xbb, 0xe0}, tr.Payload) + assert.Equal(t, int(transmission.Report.Info.ReportFormat), int(tr.ReportFormat)) + case <-time.After(testutils.WaitTimeout(t)): + t.Fatal("expected a transmit request to be sent") + } + + q.Close() + wg.Wait() + }) + t.Run("on transmit error, retries", func(t *testing.T) { + transmit := make(chan *pb.TransmitRequest, 1) + c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { + transmit <- in + return &pb.TransmitResponse{}, errors.New("transmission error") + } + q := newMockQ() + s.q = q + wg := &sync.WaitGroup{} + wg.Add(1) + stopCh := make(chan struct{}, 1) + + go s.runQueueLoop(stopCh, wg, donIDStr) + + transmission := makeSampleTransmission(1) + q.Push(transmission) + + cnt := 0 + Loop: + for { + select { + case tr := <-transmit: + assert.Equal(t, []byte{0x0, 0x9, 0x57, 0xdd, 0x2f, 0x63, 0x56, 0x69, 0x34, 0xfd, 0xc2, 0xe1, 0xcd, 0xc1, 0xe, 0x3e, 0x25, 0xb9, 0x26, 0x5a, 0x16, 0x23, 0x91, 0xa6, 0x53, 0x16, 0x66, 0x59, 0x51, 0x0, 0x28, 0x7c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x80, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x20, 0x0, 0x3, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xde, 0xf5, 0xba, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1e, 0x8e, 0x95, 0xcf, 0xb5, 0xd8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1a, 0xd0, 0x1c, 0x67, 0xa9, 0xcf, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x66, 0xdf, 0x3, 0xca, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x1c, 0x93, 0x6d, 0xa4, 0xf2, 0x17, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x14, 0x8d, 0x9a, 0xc1, 0xd9, 0x6f, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1b, 0x40, 0x5c, 0xcf, 0xa1, 0xbc, 0x63, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x9d, 0xab, 0x8f, 0xa7, 0xca, 0x7, 0x62, 0x57, 0xf7, 0x11, 0x2c, 0xb7, 0xf3, 0x49, 0x37, 0x12, 0xbd, 0xe, 0x14, 0x27, 0xfc, 0x32, 0x5c, 0xec, 0xa6, 0xb9, 0x7f, 0xf9, 0xd7, 0x7b, 0xa6, 0x36, 0x9a, 0x47, 0x4a, 0x3, 0x1a, 0x95, 0xcf, 0x46, 0x10, 0xaf, 0xcc, 0x90, 0x49, 0xb2, 0xce, 0xbf, 0x63, 0xaa, 0xc7, 0x25, 0x4d, 0x2a, 0x8, 0x36, 0xda, 0xd5, 0x9f, 0x9d, 0x63, 0x69, 0x22, 0xb3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x30, 0x9d, 0x84, 0x29, 0xbf, 0xd4, 0xeb, 0xc5, 0xc9, 0x29, 0xef, 0xdd, 0xd3, 0x2f, 0xa6, 0x25, 0x63, 0xda, 0xd9, 0x2c, 0xa1, 0x4a, 0xba, 0x75, 0xb2, 0x85, 0x25, 0x8f, 0x2b, 0x84, 0xcd, 0x99, 0x36, 0xd9, 0x6e, 0xf, 0xae, 0x7b, 0xd1, 0x61, 0x59, 0xf, 0x36, 0x4a, 0x22, 0xec, 0xde, 0x45, 0x32, 0xe0, 0x5b, 0x5c, 0xe3, 0x14, 0x29, 0x4, 0x60, 0x7b, 0xce, 0xa3, 0x89, 0x6b, 0xbb, 0xe0}, tr.Payload) + assert.Equal(t, int(transmission.Report.Info.ReportFormat), int(tr.ReportFormat)) + if cnt > 2 { + break Loop + } + cnt++ + case <-time.After(testutils.WaitTimeout(t)): + t.Fatal("expected 3 transmit requests to be sent") + } + } + + close(stopCh) + wg.Wait() + }) +} diff --git a/core/services/llo/transmitter.go b/core/services/llo/transmitter.go index fe6f26c3174..3b7ef66e6a5 100644 --- a/core/services/llo/transmitter.go +++ b/core/services/llo/transmitter.go @@ -2,24 +2,24 @@ package llo import ( "context" - "crypto/ed25519" - "fmt" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" "github.com/smartcontractkit/libocr/offchainreporting2plus/types" ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + "golang.org/x/sync/errgroup" + + "github.com/smartcontractkit/chainlink/v2/core/services/llo/mercurytransmitter" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" - - "github.com/smartcontractkit/chainlink/v2/core/services/llo/evm" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" ) // LLO Transmitter implementation, based on // core/services/relay/evm/mercury/transmitter.go +// +// If you need to "fan-out" transmits and send reports to a new destination, +// add a new subTransmitter // TODO: prom metrics (common with mercury/transmitter.go?) // https://smartcontract-it.atlassian.net/browse/MERC-3659 @@ -42,30 +42,57 @@ type Transmitter interface { type transmitter struct { services.StateMachine lggr logger.Logger - rpcClient wsrpc.Client fromAccount string + + subTransmitters []Transmitter } -func NewTransmitter(lggr logger.Logger, rpcClient wsrpc.Client, fromAccount ed25519.PublicKey) Transmitter { +type TransmitterOpts struct { + Lggr logger.Logger + FromAccount string + MercuryTransmitterOpts mercurytransmitter.Opts +} + +// The transmitter will handle starting and stopping the subtransmitters +func NewTransmitter(opts TransmitterOpts) Transmitter { + subTransmitters := []Transmitter{ + mercurytransmitter.New(opts.MercuryTransmitterOpts), + } return &transmitter{ services.StateMachine{}, - lggr, - rpcClient, - fmt.Sprintf("%x", fromAccount), + opts.Lggr, + opts.FromAccount, + subTransmitters, } } func (t *transmitter) Start(ctx context.Context) error { - return nil + return t.StartOnce("llo.Transmitter", func() error { + for _, st := range t.subTransmitters { + if err := st.Start(ctx); err != nil { + return err + } + } + return nil + }) } func (t *transmitter) Close() error { - return nil + return t.StopOnce("llo.Transmitter", func() error { + for _, st := range t.subTransmitters { + if err := st.Close(); err != nil { + return err + } + } + return nil + }) } func (t *transmitter) HealthReport() map[string]error { report := map[string]error{t.Name(): t.Healthy()} - services.CopyHealth(report, t.rpcClient.HealthReport()) + for _, st := range t.subTransmitters { + services.CopyHealth(report, st.HealthReport()) + } return report } @@ -78,32 +105,14 @@ func (t *transmitter) Transmit( report ocr3types.ReportWithInfo[llotypes.ReportInfo], sigs []types.AttributedOnchainSignature, ) (err error) { - var payload []byte - - switch report.Info.ReportFormat { - case llotypes.ReportFormatJSON: - // TODO: exactly how to handle JSON here? - // https://smartcontract-it.atlassian.net/browse/MERC-3659 - fallthrough - case llotypes.ReportFormatEVMPremiumLegacy: - payload, err = evm.ReportCodecPremiumLegacy{}.Pack(digest, seqNr, report.Report, sigs) - default: - return fmt.Errorf("Transmit failed; unsupported report format: %q", report.Info.ReportFormat) + g := new(errgroup.Group) + for _, st := range t.subTransmitters { + st := st + g.Go(func() error { + return st.Transmit(ctx, digest, seqNr, report, sigs) + }) } - - if err != nil { - return fmt.Errorf("Transmit: encode failed; %w", err) - } - - req := &pb.TransmitRequest{ - Payload: payload, - ReportFormat: uint32(report.Info.ReportFormat), - } - - // TODO: persistenceManager and queueing, error handling, retry etc - // https://smartcontract-it.atlassian.net/browse/MERC-3659 - _, err = t.rpcClient.Transmit(ctx, req) - return err + return g.Wait() } // FromAccount returns the stringified (hex) CSA public key diff --git a/core/services/llo/transmitter_test.go b/core/services/llo/transmitter_test.go deleted file mode 100644 index eb231494c0b..00000000000 --- a/core/services/llo/transmitter_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package llo - -import "testing" - -func Test_Transmitter(t *testing.T) { - // TODO: https://smartcontract-it.atlassian.net/browse/MERC-3659 -} diff --git a/core/services/ocr2/plugins/llo/config/config.go b/core/services/ocr2/plugins/llo/config/config.go index 2460fecd72d..7bd36b4ff8b 100644 --- a/core/services/ocr2/plugins/llo/config/config.go +++ b/core/services/ocr2/plugins/llo/config/config.go @@ -9,19 +9,18 @@ import ( "fmt" "net/url" "regexp" + "sort" "github.com/ethereum/go-ethereum/common" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" + mercuryconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" "github.com/smartcontractkit/chainlink/v2/core/utils" ) type PluginConfig struct { - RawServerURL string `json:"serverURL" toml:"serverURL"` - ServerPubKey utils.PlainHexBytes `json:"serverPubKey" toml:"serverPubKey"` - ChannelDefinitionsContractAddress common.Address `json:"channelDefinitionsContractAddress" toml:"channelDefinitionsContractAddress"` ChannelDefinitionsContractFromBlock int64 `json:"channelDefinitionsContractFromBlock" toml:"channelDefinitionsContractFromBlock"` @@ -41,31 +40,40 @@ type PluginConfig struct { KeyBundleIDs map[string]string `json:"keyBundleIDs" toml:"keyBundleIDs"` DonID uint32 `json:"donID" toml:"donID"` + + // Mercury servers + Servers map[string]utils.PlainHexBytes `json:"servers" toml:"servers"` } func (p *PluginConfig) Unmarshal(data []byte) error { return json.Unmarshal(data, p) } +func (p PluginConfig) GetServers() (servers []mercuryconfig.Server) { + for url, pubKey := range p.Servers { + servers = append(servers, mercuryconfig.Server{URL: wssRegexp.ReplaceAllString(url, ""), PubKey: pubKey}) + } + sort.Slice(servers, func(i, j int) bool { + return servers[i].URL < servers[j].URL + }) + return +} + func (p PluginConfig) Validate() (merr error) { if p.DonID == 0 { merr = errors.Join(merr, errors.New("llo: DonID must be specified and not zero")) } - if p.RawServerURL == "" { - merr = errors.Join(merr, errors.New("llo: ServerURL must be specified")) + if len(p.Servers) == 0 { + merr = errors.Join(merr, errors.New("llo: At least one Mercury server must be specified")) } else { - var normalizedURI string - if schemeRegexp.MatchString(p.RawServerURL) { - normalizedURI = p.RawServerURL - } else { - normalizedURI = fmt.Sprintf("wss://%s", p.RawServerURL) - } - uri, err := url.ParseRequestURI(normalizedURI) - if err != nil { - merr = errors.Join(merr, fmt.Errorf("llo: invalid value for ServerURL: %w", err)) - } else if uri.Scheme != "wss" { - merr = errors.Join(merr, fmt.Errorf(`llo: invalid scheme specified for MercuryServer, got: %q (scheme: %q) but expected a websocket url e.g. "192.0.2.2:4242" or "wss://192.0.2.2:4242"`, p.RawServerURL, uri.Scheme)) + for serverName, serverPubKey := range p.Servers { + if err := validateURL(serverName); err != nil { + merr = errors.Join(merr, fmt.Errorf("llo: invalid value for ServerURL: %w", err)) + } + if len(serverPubKey) != 32 { + merr = errors.Join(merr, errors.New("llo: ServerPubKey must be a 32-byte hex string")) + } } } @@ -88,15 +96,28 @@ func (p PluginConfig) Validate() (merr error) { } } - if len(p.ServerPubKey) != 32 { - merr = errors.Join(merr, errors.New("llo: ServerPubKey is required and must be a 32-byte hex string")) - } - merr = errors.Join(merr, validateKeyBundleIDs(p.KeyBundleIDs)) return merr } +func validateURL(rawServerURL string) error { + var normalizedURI string + if schemeRegexp.MatchString(rawServerURL) { + normalizedURI = rawServerURL + } else { + normalizedURI = fmt.Sprintf("wss://%s", rawServerURL) + } + uri, err := url.ParseRequestURI(normalizedURI) + if err != nil { + return fmt.Errorf(`llo: invalid value for ServerURL, got: %q`, rawServerURL) + } + if uri.Scheme != "wss" { + return fmt.Errorf(`llo: invalid scheme specified for MercuryServer, got: %q (scheme: %q) but expected a websocket url e.g. "192.0.2.2:4242" or "wss://192.0.2.2:4242"`, rawServerURL, uri.Scheme) + } + return nil +} + func validateKeyBundleIDs(keyBundleIDs map[string]string) error { for k, v := range keyBundleIDs { if k == "" { @@ -117,7 +138,3 @@ func validateKeyBundleIDs(keyBundleIDs map[string]string) error { var schemeRegexp = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`) var wssRegexp = regexp.MustCompile(`^wss://`) - -func (p PluginConfig) ServerURL() string { - return wssRegexp.ReplaceAllString(p.RawServerURL, "") -} diff --git a/core/services/ocr2/plugins/llo/config/config_test.go b/core/services/ocr2/plugins/llo/config/config_test.go index 5152dc839be..096543b2524 100644 --- a/core/services/ocr2/plugins/llo/config/config_test.go +++ b/core/services/ocr2/plugins/llo/config/config_test.go @@ -7,6 +7,8 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/v2/core/utils" ) func Test_Config(t *testing.T) { @@ -31,8 +33,7 @@ func Test_Config(t *testing.T) { t.Run("with all possible values set", func(t *testing.T) { rawToml := fmt.Sprintf(` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" + Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", "example2.invalid:1234" = "524ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } BenchmarkMode = true ChannelDefinitionsContractAddress = "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" ChannelDefinitionsContractFromBlock = 1234 @@ -44,8 +45,8 @@ func Test_Config(t *testing.T) { err := toml.Unmarshal([]byte(rawToml), &mc) require.NoError(t, err) - assert.Equal(t, "example.com:80", mc.RawServerURL) - assert.Equal(t, "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.ServerPubKey.String()) + assert.Len(t, mc.Servers, 2) + assert.Equal(t, map[string]utils.PlainHexBytes{"example.com:80": utils.PlainHexBytes{0x72, 0x4f, 0xf6, 0xea, 0xe9, 0xe9, 0x0, 0x27, 0xe, 0xdf, 0xff, 0x23, 0x3e, 0x16, 0x32, 0x2a, 0x70, 0xec, 0x6, 0xe1, 0xa6, 0xe6, 0x2a, 0x81, 0xef, 0x13, 0x92, 0x1f, 0x39, 0x8f, 0x6c, 0x93}, "example2.invalid:1234": utils.PlainHexBytes{0x52, 0x4f, 0xf6, 0xea, 0xe9, 0xe9, 0x0, 0x27, 0xe, 0xdf, 0xff, 0x23, 0x3e, 0x16, 0x32, 0x2a, 0x70, 0xec, 0x6, 0xe1, 0xa6, 0xe6, 0x2a, 0x81, 0xef, 0x13, 0x92, 0x1f, 0x39, 0x8f, 0x6c, 0x93}}, mc.Servers) assert.Equal(t, "0xDeaDbeefdEAdbeefdEadbEEFdeadbeEFdEaDbeeF", mc.ChannelDefinitionsContractAddress.Hex()) assert.Equal(t, int64(1234), mc.ChannelDefinitionsContractFromBlock) assert.JSONEq(t, cdjson, mc.ChannelDefinitions) @@ -60,8 +61,7 @@ func Test_Config(t *testing.T) { t.Run("with only channelDefinitions", func(t *testing.T) { rawToml := fmt.Sprintf(` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" + Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } DonID = 12345 ChannelDefinitions = """ %s @@ -71,8 +71,7 @@ func Test_Config(t *testing.T) { err := toml.Unmarshal([]byte(rawToml), &mc) require.NoError(t, err) - assert.Equal(t, "example.com:80", mc.RawServerURL) - assert.Equal(t, "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.ServerPubKey.String()) + assert.Len(t, mc.Servers, 1) assert.JSONEq(t, cdjson, mc.ChannelDefinitions) assert.Equal(t, uint32(12345), mc.DonID) assert.False(t, mc.BenchmarkMode) @@ -82,17 +81,15 @@ func Test_Config(t *testing.T) { }) t.Run("with only channelDefinitions contract details", func(t *testing.T) { rawToml := ` + Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } DonID = 12345 - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" ChannelDefinitionsContractAddress = "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"` var mc PluginConfig err := toml.Unmarshal([]byte(rawToml), &mc) require.NoError(t, err) - assert.Equal(t, "example.com:80", mc.RawServerURL) - assert.Equal(t, "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.ServerPubKey.String()) + assert.Len(t, mc.Servers, 1) assert.Equal(t, "0xDeaDbeefdEAdbeefdEadbEEFdeadbeEFdEaDbeeF", mc.ChannelDefinitionsContractAddress.Hex()) assert.Equal(t, uint32(12345), mc.DonID) assert.False(t, mc.BenchmarkMode) @@ -102,17 +99,15 @@ func Test_Config(t *testing.T) { }) t.Run("with missing ChannelDefinitionsContractAddress", func(t *testing.T) { rawToml := ` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" DonID = 12345 + Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } ` var mc PluginConfig err := toml.Unmarshal([]byte(rawToml), &mc) require.NoError(t, err) - assert.Equal(t, "example.com:80", mc.RawServerURL) - assert.Equal(t, "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.ServerPubKey.String()) + assert.Len(t, mc.Servers, 1) assert.Equal(t, uint32(12345), mc.DonID) assert.False(t, mc.BenchmarkMode) @@ -142,8 +137,39 @@ func Test_Config(t *testing.T) { err = mc.Validate() require.Error(t, err) - assert.Contains(t, err.Error(), `invalid scheme specified for MercuryServer, got: "http://example.com" (scheme: "http") but expected a websocket url e.g. "192.0.2.2:4242" or "wss://192.0.2.2:4242"`) - assert.Contains(t, err.Error(), `ServerPubKey is required and must be a 32-byte hex string`) + assert.Contains(t, err.Error(), `DonID must be specified and not zero`) + assert.Contains(t, err.Error(), `At least one Mercury server must be specified`) + assert.Contains(t, err.Error(), `ChannelDefinitionsContractAddress is required if ChannelDefinitions is not specified`) }) }) } + +func Test_PluginConfig_Validate(t *testing.T) { + t.Run("with invalid URLs or keys", func(t *testing.T) { + servers := map[string]utils.PlainHexBytes{ + "not a valid url": utils.PlainHexBytes([]byte{1, 2, 3}), + "mercuryserver.invalid:1234/foo": nil, + } + pc := PluginConfig{Servers: servers} + + err := pc.Validate() + assert.Contains(t, err.Error(), "ServerPubKey must be a 32-byte hex string") + assert.Contains(t, err.Error(), "invalid value for ServerURL: llo: invalid value for ServerURL, got: \"not a valid url\"") + }) +} + +func Test_PluginConfig_GetServers(t *testing.T) { + t.Run("with multiple servers", func(t *testing.T) { + servers := map[string]utils.PlainHexBytes{ + "example.com:80": utils.PlainHexBytes([]byte{1, 2, 3}), + "mercuryserver.invalid:1234/foo": utils.PlainHexBytes([]byte{4, 5, 6}), + } + pc := PluginConfig{Servers: servers} + + require.Len(t, pc.GetServers(), 2) + assert.Equal(t, "example.com:80", pc.GetServers()[0].URL) + assert.Equal(t, utils.PlainHexBytes{1, 2, 3}, pc.GetServers()[0].PubKey) + assert.Equal(t, "mercuryserver.invalid:1234/foo", pc.GetServers()[1].URL) + assert.Equal(t, utils.PlainHexBytes{4, 5, 6}, pc.GetServers()[1].PubKey) + }) +} diff --git a/core/services/ocr2/plugins/llo/integration_test.go b/core/services/ocr2/plugins/llo/integration_test.go index 50d580fe258..62c43eaf4da 100644 --- a/core/services/ocr2/plugins/llo/integration_test.go +++ b/core/services/ocr2/plugins/llo/integration_test.go @@ -321,11 +321,10 @@ lloDonID = %d require.NoError(t, err) backend.Commit() - pluginConfig := fmt.Sprintf(`serverURL = "%s" + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } donID = %d -serverPubKey = "%x" channelDefinitionsContractAddress = "0x%x" -channelDefinitionsContractFromBlock = %d`, serverURL, donID, serverPubKey, configStoreAddress, fromBlock) +channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, configuratorAddress, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) // Set config on configurator diff --git a/core/services/relay/dummy/llo_provider.go b/core/services/relay/dummy/llo_provider.go index 887fa134309..88f72588157 100644 --- a/core/services/relay/dummy/llo_provider.go +++ b/core/services/relay/dummy/llo_provider.go @@ -11,14 +11,18 @@ import ( relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/llo" ) var _ commontypes.LLOProvider = (*lloProvider)(nil) +type Transmitter interface { + services.Service + llotypes.Transmitter +} + type lloProvider struct { cp commontypes.ConfigProvider - transmitter llo.Transmitter + transmitter Transmitter logger logger.Logger channelDefinitionCache llotypes.ChannelDefinitionCache @@ -28,7 +32,7 @@ type lloProvider struct { func NewLLOProvider( lggr logger.Logger, cp commontypes.ConfigProvider, - transmitter llo.Transmitter, + transmitter Transmitter, channelDefinitionCache llotypes.ChannelDefinitionCache, ) relaytypes.LLOProvider { return &lloProvider{ diff --git a/core/services/relay/evm/evm.go b/core/services/relay/evm/evm.go index 273926136ef..579e8b67881 100644 --- a/core/services/relay/evm/evm.go +++ b/core/services/relay/evm/evm.go @@ -41,6 +41,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/keystore" "github.com/smartcontractkit/chainlink/v2/core/services/llo" "github.com/smartcontractkit/chainlink/v2/core/services/llo/bm" + "github.com/smartcontractkit/chainlink/v2/core/services/llo/mercurytransmitter" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/ccipcommit" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/ccipexec" @@ -437,17 +438,31 @@ func (r *Relayer) NewLLOProvider(rargs commontypes.RelayArgs, pargs commontypes. // FIXME: Remove after benchmarking is done // https://smartcontract-it.atlassian.net/browse/MERC-3487 - var transmitter llo.Transmitter + var transmitter LLOTransmitter if lloCfg.BenchmarkMode { r.lggr.Info("Benchmark mode enabled, using dummy transmitter. NOTE: THIS WILL NOT TRANSMIT ANYTHING") transmitter = bm.NewTransmitter(r.lggr, fmt.Sprintf("%x", privKey.PublicKey)) } else { - var client wsrpc.Client - client, err = r.mercuryPool.Checkout(context.Background(), privKey, lloCfg.ServerPubKey, lloCfg.ServerURL()) - if err != nil { - return nil, err + clients := make(map[string]wsrpc.Client) + for _, server := range lloCfg.GetServers() { + client, err2 := r.mercuryPool.Checkout(context.Background(), privKey, server.PubKey, server.URL) + if err2 != nil { + return nil, err2 + } + clients[server.URL] = client } - transmitter = llo.NewTransmitter(r.lggr, client, privKey.PublicKey) + transmitter = llo.NewTransmitter(llo.TransmitterOpts{ + Lggr: r.lggr, + FromAccount: fmt.Sprintf("%x", privKey.PublicKey), // NOTE: This may need to change if we support e.g. multiple tranmsmitters, to be a composite of all keys + MercuryTransmitterOpts: mercurytransmitter.Opts{ + Lggr: r.lggr, + Cfg: r.transmitterCfg, + Clients: clients, + FromAccount: privKey.PublicKey, + DonID: relayConfig.LLODONID, + ORM: mercurytransmitter.NewORM(r.ds, relayConfig.LLODONID), + }, + }) } cdcFactory, err := r.cdcFactory() diff --git a/core/services/relay/evm/llo_provider.go b/core/services/relay/evm/llo_provider.go index 55b80bd58e0..2cf6163047f 100644 --- a/core/services/relay/evm/llo_provider.go +++ b/core/services/relay/evm/llo_provider.go @@ -11,15 +11,18 @@ import ( commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" - - "github.com/smartcontractkit/chainlink/v2/core/services/llo" ) var _ commontypes.LLOProvider = (*lloProvider)(nil) +type LLOTransmitter interface { + services.Service + llotypes.Transmitter +} + type lloProvider struct { cp commontypes.ConfigProvider - transmitter llo.Transmitter + transmitter LLOTransmitter logger logger.Logger channelDefinitionCache llotypes.ChannelDefinitionCache @@ -28,7 +31,7 @@ type lloProvider struct { func NewLLOProvider( cp commontypes.ConfigProvider, - transmitter llo.Transmitter, + transmitter LLOTransmitter, lggr logger.Logger, channelDefinitionCache llotypes.ChannelDefinitionCache, ) relaytypes.LLOProvider { diff --git a/core/services/relay/evm/mercury/config_poller.go b/core/services/relay/evm/mercury/config_poller.go index 111c432e6fd..7bb5d2af7b5 100644 --- a/core/services/relay/evm/mercury/config_poller.go +++ b/core/services/relay/evm/mercury/config_poller.go @@ -99,7 +99,6 @@ func FilterName(addr common.Address, feedID common.Hash) string { // NewConfigPoller creates a new Mercury ConfigPoller func NewConfigPoller(ctx context.Context, lggr logger.Logger, destChainPoller logpoller.LogPoller, addr common.Address, feedId common.Hash) (*ConfigPoller, error) { - fmt.Println("TRASH RegisterFilter", FilterName(addr, feedId)) err := destChainPoller.RegisterFilter(ctx, logpoller.Filter{Name: FilterName(addr, feedId), EventSigs: []common.Hash{FeedScopedConfigSet}, Addresses: []common.Address{addr}}) if err != nil { return nil, err diff --git a/core/store/migrate/migrations/0252_add_llo_transmit_requests.sql b/core/store/migrate/migrations/0252_add_llo_transmit_requests.sql new file mode 100644 index 00000000000..d08ae0c921a --- /dev/null +++ b/core/store/migrate/migrations/0252_add_llo_transmit_requests.sql @@ -0,0 +1,21 @@ +-- +goose Up + +CREATE TABLE llo_mercury_transmit_queue ( + don_id BIGINT NOT NULL, + server_url TEXT NOT NULL, + config_digest BYTEA NOT NULL, + seq_nr BIGINT NOT NULL, + report BYTEA NOT NULL, + lifecycle_stage TEXT NOT NULL, + report_format BIGINT NOT NULL, + signatures BYTEA[] NOT NULL, + signers SMALLINT[] NOT NULL, + transmission_hash BYTEA NOT NULL, + PRIMARY KEY (transmission_hash) +); + + CREATE INDEX idx_llo_mercury_transmit_queue_don_id_server_url_seq_nr ON llo_mercury_transmit_queue (don_id, server_url, seq_nr DESC); + +-- +goose Down + +DROP TABLE llo_mercury_transmit_queue; diff --git a/go.mod b/go.mod index 32959272ea0..4803a35b9d8 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/smartcontractkit/chainlink-ccip v0.0.0-20240910072312-810030689426 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de7322 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 - github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 + github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240910163253-2a5c9ab97de3 diff --git a/go.sum b/go.sum index adc46fa7236..aea2168da83 100644 --- a/go.sum +++ b/go.sum @@ -1046,8 +1046,8 @@ github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de732 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de7322/go.mod h1:D/qaCoq0SxXzg5NRN5FtBRv98VBf+D2NOC++RbvvuOc= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 h1:xow5cYrWxRmzTwhz2AjWOlnI9WRF4O5b84bXm1k292E= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227/go.mod h1:DUFantPYoBGwBSkNVt2k4ZJi0jPKRRrZVVlAzcZwreA= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 h1:0aZ3HiEz2bMM5ywHAyKlFMN95qTzpNDn7uvnHLrFX6s= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc h1:tRmTlaoAt+7FakMXXgeCuRPmzzBo5jsGpeCVvcU6KMc= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f h1:p4p3jBT91EQyLuAMvHD+zNJsuAYI/QjJbzuGUJ7wIgg= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f/go.mod h1:FLlWBt2hwiMVgt9AcSo6wBJYIRd/nsc8ENbV1Wir1bw= github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd h1:+RFI4mgXSGEcn847e7bavhQCqaBiW142g1XfGzijFoY= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 4effcf9f2f5..53aa0f475b3 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -404,7 +404,7 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 // indirect - github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 // indirect + github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc // indirect github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240910163253-2a5c9ab97de3 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 18538f1b984..02208bcf8a4 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1429,8 +1429,8 @@ github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de732 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de7322/go.mod h1:D/qaCoq0SxXzg5NRN5FtBRv98VBf+D2NOC++RbvvuOc= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 h1:xow5cYrWxRmzTwhz2AjWOlnI9WRF4O5b84bXm1k292E= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227/go.mod h1:DUFantPYoBGwBSkNVt2k4ZJi0jPKRRrZVVlAzcZwreA= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 h1:0aZ3HiEz2bMM5ywHAyKlFMN95qTzpNDn7uvnHLrFX6s= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc h1:tRmTlaoAt+7FakMXXgeCuRPmzzBo5jsGpeCVvcU6KMc= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f h1:p4p3jBT91EQyLuAMvHD+zNJsuAYI/QjJbzuGUJ7wIgg= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f/go.mod h1:FLlWBt2hwiMVgt9AcSo6wBJYIRd/nsc8ENbV1Wir1bw= github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd h1:+RFI4mgXSGEcn847e7bavhQCqaBiW142g1XfGzijFoY= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index d3d6d8452a1..6d2658d24af 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -395,7 +395,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.21 // indirect github.com/smartcontractkit/chainlink-ccip v0.0.0-20240910072312-810030689426 // indirect github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 // indirect - github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 // indirect + github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc // indirect github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f // indirect github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd // indirect github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240910163253-2a5c9ab97de3 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index e781103a2d0..592984aca4b 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1403,8 +1403,8 @@ github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de732 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240910154010-ed9f50de7322/go.mod h1:D/qaCoq0SxXzg5NRN5FtBRv98VBf+D2NOC++RbvvuOc= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227 h1:xow5cYrWxRmzTwhz2AjWOlnI9WRF4O5b84bXm1k292E= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20240910210931-638ba8a76227/go.mod h1:DUFantPYoBGwBSkNVt2k4ZJi0jPKRRrZVVlAzcZwreA= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652 h1:0aZ3HiEz2bMM5ywHAyKlFMN95qTzpNDn7uvnHLrFX6s= -github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240904093355-e40169857652/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc h1:tRmTlaoAt+7FakMXXgeCuRPmzzBo5jsGpeCVvcU6KMc= +github.com/smartcontractkit/chainlink-data-streams v0.0.0-20240906125718-9f0a98d32fbc/go.mod h1:PwPcmQNAzVmU8r8JWKrDRgvXesDwxnqbMD6DvYt/Z7M= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f h1:p4p3jBT91EQyLuAMvHD+zNJsuAYI/QjJbzuGUJ7wIgg= github.com/smartcontractkit/chainlink-feeds v0.0.0-20240910155501-42f20443189f/go.mod h1:FLlWBt2hwiMVgt9AcSo6wBJYIRd/nsc8ENbV1Wir1bw= github.com/smartcontractkit/chainlink-solana v1.1.1-0.20240910161529-a7050b5193cd h1:+RFI4mgXSGEcn847e7bavhQCqaBiW142g1XfGzijFoY=