From c53bb4c0d5426068d546024147f648a9f2af79a5 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 31 Oct 2023 16:38:04 -0400 Subject: [PATCH 01/74] initial commit for inmemory implementation --- common/txmgr/inmemory_store.go | 393 ++++++++++++++++++++++++++++ common/txmgr/inmemory_store_test.go | 152 +++++++++++ 2 files changed, 545 insertions(+) create mode 100644 common/txmgr/inmemory_store.go create mode 100644 common/txmgr/inmemory_store_test.go diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go new file mode 100644 index 00000000000..c84d9b738a4 --- /dev/null +++ b/common/txmgr/inmemory_store.go @@ -0,0 +1,393 @@ +package txmgr + +import ( + "context" + "fmt" + "sync" + "time" + + feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/common/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/label" + "gopkg.in/guregu/null.v4" +) + +var ( + // ErrInvalidChainID is returned when the chain ID is invalid + ErrInvalidChainID = fmt.Errorf("invalid chain ID") + // ErrTxnNotFound is returned when a transaction is not found + ErrTxnNotFound = fmt.Errorf("transaction not found") + // ErrExistingIdempotencyKey is returned when a transaction with the same idempotency key already exists + ErrExistingIdempotencyKey = fmt.Errorf("transaction with idempotency key already exists") + // ErrExistingPilelineTaskRunId is returned when a transaction with the same pipeline task run id already exists + ErrExistingPilelineTaskRunId = fmt.Errorf("transaction with pipeline task run id already exists") +) + +// Store and update all transaction state as files +// Read from the files to restore state at startup +// Delete files when transactions are completed or reaped + +// Life of a Transaction +// 1. Transaction Request is created +// 2. Transaction Request is submitted to the Transaction Manager +// 3. Transaction Manager creates and persists a new transaction (unstarted) from the transaction request (not persisted) +// 4. Transaction Manager sends the transaction (unstarted) to the Broadcaster Unstarted Queue +// 4. Transaction Manager prunes the Unstarted Queue based on the transaction prune strategy + +// NOTE(jtw): Gets triggered by postgres Events +// NOTE(jtw): Only one transaction per address can be in_progress at a time +// NOTE(jtw): Only one broadcasted attempt exists per transaction the rest are errored or abandoned +// 1. Broadcaster assigns a sequence number to the transaction +// 2. Broadcaster creates and persists a new transaction attempt (in_progress) from the transaction (in_progress) +// 3. Broadcaster asks the Checker to check if the transaction should not be sent +// 4. Broadcaster asks the Attempt builder to figure out gas fee for the transaction +// 5. Broadcaster attempts to send the Transaction to TransactionClient to be published on-chain +// 6. Broadcaster updates the transaction attempt (broadcast) and transaction (unconfirmed) +// 7. Broadcaster increments global sequence number for address for next transaction attempt + +// NOTE(jtw): Only one receipt should exist per confirmed transaction +// 1. Confirmer listens and reads new Head events from the Chain +// 2. Confirmer sets the last known block number for the transaction attempts that have been broadcast +// 3. Confirmer checks for missing receipts for transactions that have been broadcast +// 4. Confirmer sets transactions that have failed to (unconfirmed) which will be retried by the resender +// 5. Confirmer sets transactions that have been confirmed to (confirmed) and creates a new receipt which is persisted + +type InMemoryStore[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] struct { + // TODO(jtw): Change this to non exported and figure it out via configs or other settings + LegacyEnabled bool + + chainID CHAIN_ID + + keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] + // EventRecorder is used to persist events which can be replayed later to restore the state of the system + // rename to txStore + eventRecorder txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + + pendingLock sync.Mutex + // NOTE(jtw): we might need to watch out for txns that finish and are removed from the pending map + pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + pendingPipelineTaskRunIds map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + + unstarted map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + inprogress map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] +} + +// NewInMemoryStore returns a new InMemoryStore +func NewInMemoryStore[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +]( + chainID CHAIN_ID, + keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], + eventRecorder txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], +) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { + tm := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + LegacyEnabled: true, + chainID: chainID, + keyStore: keyStore, + eventRecorder: eventRecorder, + + pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + pendingPipelineTaskRunIds: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + + unstarted: map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + inprogress: map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + } + + addresses, err := keyStore.EnabledAddressesForChain(chainID) + if err != nil { + return nil, err + } + for _, fromAddr := range addresses { + // Channel Buffer is set to something high to prevent blocking and allow the pruning to happen + tm.unstarted[fromAddr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) + } + + return &tm, nil +} + +// Close closes the InMemoryStore +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { + // Close the event recorder + ms.eventRecorder.Close() + + // Close all channels + for _, ch := range ms.unstarted { + close(ch) + } + + // Clear all pending requests + ms.pendingLock.Lock() + clear(ms.pendingIdempotencyKeys) + clear(ms.pendingPipelineTaskRunIds) + ms.pendingLock.Unlock() +} + +// Abandon removes all transactions for a given address +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Abandon(ctx context.Context, chainID CHAIN_ID, addr ADDR) error { + if ms.chainID.String() != chainID.String() { + return ErrInvalidChainID + } + + // Mark all persisted transactions as abandoned + if err := ms.eventRecorder.Abandon(ctx, chainID, addr); err != nil { + return err + } + + // Mark all unstarted transactions as abandoned + close(ms.unstarted[addr]) + for tx := range ms.unstarted[addr] { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + // reset the unstarted channel + ms.unstarted[addr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) + + // Mark all inprogress transactions as abandoned + if tx, ok := ms.inprogress[addr]; ok { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + ms.inprogress[addr] = nil + + // TODO(jtw): Mark all unconfirmed transactions as abandoned + + // Mark all pending transactions as abandoned + for _, tx := range ms.pendingIdempotencyKeys { + if tx.FromAddress == addr { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + } + for _, tx := range ms.pendingPipelineTaskRunIds { + if tx.FromAddress == addr { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + } + // TODO(jtw): SHOULD THE REAPER BE RESPONSIBLE FOR CLEARING THE PENDING MAPS? + + return nil +} + +// CreateTransaction creates a new transaction for a given txRequest. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + // Persist Transaction to persistent storage + if ms.LegacyEnabled { + tx, err := ms.eventRecorder.CreateTransaction(ctx, txRequest, chainID) + if err != nil { + return tx, err + } + return tx, ms.sendTxToBroadcaster(tx) + } else { + // HANDLE NEW EVENT RECORDER FOR PERSISTENCE + } + + // Check if PipelineTaskRunId already exists + if txRequest.PipelineTaskRunID != nil { + ms.pendingLock.Lock() + if tx, ok := ms.pendingPipelineTaskRunIds[txRequest.PipelineTaskRunID.String()]; ok { + ms.pendingLock.Unlock() + return *tx, ErrExistingPilelineTaskRunId + } + ms.pendingLock.Unlock() + } + + tx := txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{ + CreatedAt: time.Now().UTC(), + State: TxUnstarted, + FromAddress: txRequest.FromAddress, + ToAddress: txRequest.ToAddress, + EncodedPayload: txRequest.EncodedPayload, + Value: txRequest.Value, + FeeLimit: txRequest.FeeLimit, + // TODO(jtw): this needs to be implemented + // Meta: txRequest.Meta, + // TODO(jtw): this needs to be implemented + // Subject: txRequest.Strategy.Subject(), + ChainID: chainID, + // TODO(jtw): this needs to be implemented + // PipelineTaskRunID: txRequest.PipelineTaskRunID, + IdempotencyKey: txRequest.IdempotencyKey, + // TODO(jtw): this needs to be implemented + // TransmitChecker: txRequest.Checker, + MinConfirmations: txRequest.MinConfirmations, + } + + return tx, ms.sendTxToBroadcaster(tx) +} + +// TODO(jtw): change naming to something more appropriate +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendTxToBroadcaster(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + // TODO(jtw); HANDLE PRUNING STEP + + select { + // Add the request to the Unstarted channel to be processed by the Broadcaster + case ms.unstarted[tx.FromAddress] <- &tx: + // Persist to persistent storage + + ms.pendingLock.Lock() + if tx.IdempotencyKey != nil { + ms.pendingIdempotencyKeys[*tx.IdempotencyKey] = &tx + } + if tx.PipelineTaskRunID.UUID.String() != "" { + ms.pendingPipelineTaskRunIds[tx.PipelineTaskRunID.UUID.String()] = &tx + } + ms.pendingLock.Unlock() + + return nil + default: + // Return an error if the Manager Queue Capacity has been reached + return fmt.Errorf("transaction manager queue capacity has been reached") + } +} + +// FindTxWithIdempotencyKey returns a transaction with the given idempotency key +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + // TODO(jtw): this is a change from current functionality... it returns nil, nil if nothing found in other implementations + if ms.chainID.String() != chainID.String() { + return nil, ErrInvalidChainID + } + if idempotencyKey == "" { + return nil, fmt.Errorf("FindTxWithIdempotencyKey: idempotency key cannot be empty") + } + + ms.pendingLock.Lock() + defer ms.pendingLock.Unlock() + + tx, ok := ms.pendingIdempotencyKeys[idempotencyKey] + if !ok { + return nil, fmt.Errorf("FindTxWithIdempotencyKey: transaction not found") + } + + return tx, nil +} + +// CheckTxQueueCapacity checks if the queue capacity has been reached for a given address +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckTxQueueCapacity(ctx context.Context, fromAddress ADDR, maxQueuedTransactions uint64, chainID CHAIN_ID) (err error) { + if maxQueuedTransactions == 0 { + return nil + } + if ms.chainID.String() != chainID.String() { + return ErrInvalidChainID + } + if _, ok := ms.unstarted[fromAddress]; !ok { + return fmt.Errorf("CheckTxQueueCapacity: address not found") + } + + count := uint64(len(ms.unstarted[fromAddress])) + if count >= maxQueuedTransactions { + return fmt.Errorf("CheckTxQueueCapacity: cannot create transaction; too many unstarted transactions in the queue (%v/%v). %s", count, maxQueuedTransactions, label.MaxQueuedTransactionsWarning) + } + + return nil +} + +/* +// BROADCASTER FUNCTIONS +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) CountUnconfirmedTransactions(_ context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) { + if ms.chainID != chainID { + return 0, ErrInvalidChainID + } + // TODO(jtw): NEED TO COMPLETE + return 0, nil +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) CountUnstartedTransactions(_ context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) { + if ms.chainID != chainID { + return 0, ErrInvalidChainID + } + + return uint32(len(ms.unstarted[fromAddress])), nil +} +func (ms *InMemoryStore) FindNextUnstartedTransactionFromAddress(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID) error { + +} +func (ms *InMemoryStore) GetTxInProgress(ctx context.Context, fromAddress ADDR) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +} + +func (ms *InMemoryStore) UpdateTxAttemptInProgressToBroadcast(etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState TxAttemptState, incrNextSequenceCallback QueryerFunc, qopts ...pg.QOpt) error { +} +func (ms *InMemoryStore) UpdateTxUnstartedToInProgress(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +} +func (ms *InMemoryStore) UpdateTxFatalError(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +} +func (ms *InMemoryStore) SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +} + +// ProcessUnstartedTxs processes unstarted transactions +// TODO(jtw): SHOULD THIS BE CALLED THE BROADCASTER? +func (tm *TransactionManager) ProcessUnstartedTxs(ctx context.Context, fromAddress string) { + // if there are more in flight transactions than the max then throttle using the InFlightTransactionRecheckInterval + for { + select { + // NOTE: There can be at most one in_progress transaction per address. + case txReq := <-tm.Unstarted[fromAddress]: + // check number of in flight transactions to see if we can process more... MaxInFlight is for total inflight + tm.inFlightWG.Wait() + // Reserve a spot in the in flight transactions + tm.inFlightWG.Done() + + // TODO(jtw): THERE ARE SOME CHANGES AROUND ERROR FUNCTIONALITY + // EXAMPLE NO LONGER WILL ERROR IF THE NUMBER OF IN FLIGHT TRANSACTIONS IS EXCEEDED + if err := tm.PublishToChain(txReq); err != nil { + // TODO(jtw): Handle error properly + fmt.Println(err) + } + + // Free up a spot in the in flight transactions + tm.inFlightWG.Add(1) + case <-ctx.Done(): + return + } + } + +} + +// PublishToChain attempts to publish a transaction to the chain +// TODO(jtw): NO LONGER RETURNS AN ERROR IF FULL OF IN PROGRESS TRANSACTIONS... not sure if okay +func (tm *TransactionManager) PublishToChain(txReq TxRequest) error { + // Handle an unstarted request + // Get next sequence number from the KeyStore + // ks.NextSequence(fromAddress, tm.ChainID) + // Create a new transaction attempt to be put on chain + // eb.NewTxAttempt(ctx, txReq, logger) + + // IT BLOCKS UNTIL THERE IS A SPOT IN THE IN PROGRESS TRANSACTIONS + tm.Inprogress[txReq.FromAddress] <- txReq + + return nil +} +*/ + +/* +// Close closes the InMemoryStore +func (ms *InMemoryStore) Close() { + // Close all channels + for _, ch := range ms.Unstarted { + close(ch) + } + for _, ch := range ms.Inprogress { + close(ch) + } + + // Clear all pending requests + ms.pendingLock.Lock() + clear(ms.pendingIdempotencyKeys) + clear(ms.pendingPipelineTaskRunIds) + ms.pendingLock.Unlock() +} +*/ diff --git a/common/txmgr/inmemory_store_test.go b/common/txmgr/inmemory_store_test.go new file mode 100644 index 00000000000..b0e2d778ffa --- /dev/null +++ b/common/txmgr/inmemory_store_test.go @@ -0,0 +1,152 @@ +package txmgr_test + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/cometbft/cometbft/libs/rand" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" + "github.com/smartcontractkit/chainlink/v2/common/txmgr" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/common/txmgr/types/mocks" + types "github.com/smartcontractkit/chainlink/v2/common/types" +) + +func TestInMemoryStore_CreateTransaction(t *testing.T) { + chainID := big.NewInt(1) + idempotencyKey := "11" + fromAddress := common.BytesToAddress(rand.Bytes(20)) + mockKeyStore := mocks.NewKeyStore[common.Address, *big.Int, types.Sequence](t) + mockKeyStore.Mock.On("EnabledAddressesForChain", chainID).Return([]common.Address{fromAddress}, nil) + mockEventRecorder := mocks.NewTxStore[ + common.Address, *big.Int, common.Hash, common.Hash, txmgrtypes.ChainReceipt[common.Hash, common.Hash], types.Sequence, feetypes.Fee](t) + mockEventRecorder.Mock.On("CreateTransaction", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil, nil) + ctx := context.Background() + + ims, err := txmgr.NewInMemoryStore[ + *big.Int, common.Address, common.Hash, common.Hash, txmgrtypes.ChainReceipt[common.Hash, common.Hash], types.Sequence, feetypes.Fee, + ](chainID, mockKeyStore, mockEventRecorder) + ims.LegacyEnabled = false // TODO(jtw): this is just for initial testing, remove this + require.NoError(t, err) + + tts := []struct { + scenario string + createTransactionInput createTransactionInput + createTransactionOutput createTransactionOutput + findTxWithIdempotencyKeyInput findTxWithIdempotencyKeyInput + findTxWithIdempotencyKeyOutput findTxWithIdempotencyKeyOutput + checkTxQueueCapacityInput checkTxQueueCapacityInput + checkTxQueueCapacityOutput checkTxQueueCapacityOutput + }{ + { + scenario: "success", + createTransactionInput: createTransactionInput{ + txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + ToAddress: common.BytesToAddress([]byte("test")), + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: uint32(1000), + Meta: nil, + Strategy: nil, //TODO + }, + chainID: chainID, + }, + createTransactionOutput: createTransactionOutput{ + tx: txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee]{ + IdempotencyKey: &idempotencyKey, + CreatedAt: time.Now().UTC(), + State: txmgr.TxUnstarted, + ChainID: chainID, + FromAddress: fromAddress, + ToAddress: common.BytesToAddress([]byte("test")), + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: uint32(1000), + Meta: nil, + }, + err: nil, + }, + findTxWithIdempotencyKeyInput: findTxWithIdempotencyKeyInput{ + idempotencyKey: "11", + chainID: chainID, + }, + findTxWithIdempotencyKeyOutput: findTxWithIdempotencyKeyOutput{ + tx: txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee]{ + IdempotencyKey: &idempotencyKey, + CreatedAt: time.Now().UTC(), + State: txmgr.TxUnstarted, + ChainID: chainID, + FromAddress: fromAddress, + ToAddress: common.BytesToAddress([]byte("test")), + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: uint32(1000), + Meta: nil, + }, + }, + checkTxQueueCapacityInput: checkTxQueueCapacityInput{ + fromAddress: fromAddress, + maxQueued: uint64(16), + chainID: chainID, + }, + checkTxQueueCapacityOutput: checkTxQueueCapacityOutput{ + err: nil, + }, + }, + } + + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actTx, actErr := ims.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) + require.Equal(t, tt.createTransactionOutput.err, actErr, "CreateTransaction: expected err to match actual err") + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, tt.createTransactionOutput.tx.CreatedAt, actTx.CreatedAt, time.Second, "CreateTransaction: expected time to be within 1 second of actual time") + // Reset CreatedAt to avoid flaky test + tt.createTransactionOutput.tx.CreatedAt = actTx.CreatedAt + assert.Equal(t, tt.createTransactionOutput.tx, actTx, "CreateTransaction: expected tx to match actual tx") + + actTxPtr, actErr := ims.FindTxWithIdempotencyKey(ctx, tt.findTxWithIdempotencyKeyInput.idempotencyKey, tt.findTxWithIdempotencyKeyInput.chainID) + require.Equal(t, tt.findTxWithIdempotencyKeyOutput.err, actErr, "FindTxWithIdempotencyKey: expected err to match actual err") + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, tt.findTxWithIdempotencyKeyOutput.tx.CreatedAt, actTx.CreatedAt, time.Second, "FindTxWithIdempotencyKey: expected time to be within 1 second of actual time") + // Reset CreatedAt to avoid flaky test + tt.findTxWithIdempotencyKeyOutput.tx.CreatedAt = actTxPtr.CreatedAt + assert.Equal(t, tt.findTxWithIdempotencyKeyOutput.tx, actTx, "FindTxWithIdempotencyKey: expected tx to match actual tx") + + actErr = ims.CheckTxQueueCapacity(ctx, tt.checkTxQueueCapacityInput.fromAddress, tt.checkTxQueueCapacityInput.maxQueued, tt.checkTxQueueCapacityInput.chainID) + require.Equal(t, tt.checkTxQueueCapacityOutput.err, actErr, "CheckTxQueueCapacity: expected err to match actual err") + }) + } + +} + +type createTransactionInput struct { + txRequest txmgrtypes.TxRequest[common.Address, common.Hash] + chainID *big.Int +} +type createTransactionOutput struct { + tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee] + err error +} +type findTxWithIdempotencyKeyInput struct { + idempotencyKey string + chainID *big.Int +} +type findTxWithIdempotencyKeyOutput struct { + tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee] + err error +} +type checkTxQueueCapacityInput struct { + fromAddress common.Address + maxQueued uint64 + chainID *big.Int +} +type checkTxQueueCapacityOutput struct { + err error +} From b75d260e345bc9f0a091e521cb0b90faa3d587ca Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 1 Nov 2023 11:09:27 -0400 Subject: [PATCH 02/74] some cleanup --- common/txmgr/inmemory_store.go | 48 ++--------------------------- common/txmgr/inmemory_store_test.go | 1 - 2 files changed, 3 insertions(+), 46 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index c84d9b738a4..694bf64a631 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "sync" - "time" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" @@ -60,9 +59,6 @@ type InMemoryStore[ SEQ types.Sequence, FEE feetypes.Fee, ] struct { - // TODO(jtw): Change this to non exported and figure it out via configs or other settings - LegacyEnabled bool - chainID CHAIN_ID keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] @@ -92,7 +88,6 @@ func NewInMemoryStore[ eventRecorder txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], ) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { tm := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - LegacyEnabled: true, chainID: chainID, keyStore: keyStore, eventRecorder: eventRecorder, @@ -187,47 +182,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband // CreateTransaction creates a new transaction for a given txRequest. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { // Persist Transaction to persistent storage - if ms.LegacyEnabled { - tx, err := ms.eventRecorder.CreateTransaction(ctx, txRequest, chainID) - if err != nil { - return tx, err - } - return tx, ms.sendTxToBroadcaster(tx) - } else { - // HANDLE NEW EVENT RECORDER FOR PERSISTENCE - } - - // Check if PipelineTaskRunId already exists - if txRequest.PipelineTaskRunID != nil { - ms.pendingLock.Lock() - if tx, ok := ms.pendingPipelineTaskRunIds[txRequest.PipelineTaskRunID.String()]; ok { - ms.pendingLock.Unlock() - return *tx, ErrExistingPilelineTaskRunId - } - ms.pendingLock.Unlock() - } - - tx := txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{ - CreatedAt: time.Now().UTC(), - State: TxUnstarted, - FromAddress: txRequest.FromAddress, - ToAddress: txRequest.ToAddress, - EncodedPayload: txRequest.EncodedPayload, - Value: txRequest.Value, - FeeLimit: txRequest.FeeLimit, - // TODO(jtw): this needs to be implemented - // Meta: txRequest.Meta, - // TODO(jtw): this needs to be implemented - // Subject: txRequest.Strategy.Subject(), - ChainID: chainID, - // TODO(jtw): this needs to be implemented - // PipelineTaskRunID: txRequest.PipelineTaskRunID, - IdempotencyKey: txRequest.IdempotencyKey, - // TODO(jtw): this needs to be implemented - // TransmitChecker: txRequest.Checker, - MinConfirmations: txRequest.MinConfirmations, + tx, err := ms.eventRecorder.CreateTransaction(ctx, txRequest, chainID) + if err != nil { + return tx, err } - return tx, ms.sendTxToBroadcaster(tx) } diff --git a/common/txmgr/inmemory_store_test.go b/common/txmgr/inmemory_store_test.go index b0e2d778ffa..a9f5fb8c06e 100644 --- a/common/txmgr/inmemory_store_test.go +++ b/common/txmgr/inmemory_store_test.go @@ -33,7 +33,6 @@ func TestInMemoryStore_CreateTransaction(t *testing.T) { ims, err := txmgr.NewInMemoryStore[ *big.Int, common.Address, common.Hash, common.Hash, txmgrtypes.ChainReceipt[common.Hash, common.Hash], types.Sequence, feetypes.Fee, ](chainID, mockKeyStore, mockEventRecorder) - ims.LegacyEnabled = false // TODO(jtw): this is just for initial testing, remove this require.NoError(t, err) tts := []struct { From 3dde323beae0fb2bb2e2b581e856e8bc3f56fb8e Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 2 Nov 2023 18:15:24 -0400 Subject: [PATCH 03/74] fix tests --- common/txmgr/inmemory_store.go | 18 ++-- common/txmgr/inmemory_store_test.go | 151 --------------------------- core/chains/inmemory_store_test.go | 153 ++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 161 deletions(-) delete mode 100644 common/txmgr/inmemory_store_test.go create mode 100644 core/chains/inmemory_store_test.go diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 694bf64a631..4bf07bad474 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -62,9 +62,7 @@ type InMemoryStore[ chainID CHAIN_ID keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] - // EventRecorder is used to persist events which can be replayed later to restore the state of the system - // rename to txStore - eventRecorder txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] pendingLock sync.Mutex // NOTE(jtw): we might need to watch out for txns that finish and are removed from the pending map @@ -85,12 +83,12 @@ func NewInMemoryStore[ ]( chainID CHAIN_ID, keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], - eventRecorder txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], ) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { tm := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - chainID: chainID, - keyStore: keyStore, - eventRecorder: eventRecorder, + chainID: chainID, + keyStore: keyStore, + txStore: txStore, pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, pendingPipelineTaskRunIds: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, @@ -114,7 +112,7 @@ func NewInMemoryStore[ // Close closes the InMemoryStore func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { // Close the event recorder - ms.eventRecorder.Close() + ms.txStore.Close() // Close all channels for _, ch := range ms.unstarted { @@ -135,7 +133,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband } // Mark all persisted transactions as abandoned - if err := ms.eventRecorder.Abandon(ctx, chainID, addr); err != nil { + if err := ms.txStore.Abandon(ctx, chainID, addr); err != nil { return err } @@ -182,7 +180,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband // CreateTransaction creates a new transaction for a given txRequest. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { // Persist Transaction to persistent storage - tx, err := ms.eventRecorder.CreateTransaction(ctx, txRequest, chainID) + tx, err := ms.txStore.CreateTransaction(ctx, txRequest, chainID) if err != nil { return tx, err } diff --git a/common/txmgr/inmemory_store_test.go b/common/txmgr/inmemory_store_test.go deleted file mode 100644 index a9f5fb8c06e..00000000000 --- a/common/txmgr/inmemory_store_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package txmgr_test - -import ( - "context" - "math/big" - "testing" - "time" - - "github.com/cometbft/cometbft/libs/rand" - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" - "github.com/smartcontractkit/chainlink/v2/common/txmgr" - txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" - "github.com/smartcontractkit/chainlink/v2/common/txmgr/types/mocks" - types "github.com/smartcontractkit/chainlink/v2/common/types" -) - -func TestInMemoryStore_CreateTransaction(t *testing.T) { - chainID := big.NewInt(1) - idempotencyKey := "11" - fromAddress := common.BytesToAddress(rand.Bytes(20)) - mockKeyStore := mocks.NewKeyStore[common.Address, *big.Int, types.Sequence](t) - mockKeyStore.Mock.On("EnabledAddressesForChain", chainID).Return([]common.Address{fromAddress}, nil) - mockEventRecorder := mocks.NewTxStore[ - common.Address, *big.Int, common.Hash, common.Hash, txmgrtypes.ChainReceipt[common.Hash, common.Hash], types.Sequence, feetypes.Fee](t) - mockEventRecorder.Mock.On("CreateTransaction", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil, nil) - ctx := context.Background() - - ims, err := txmgr.NewInMemoryStore[ - *big.Int, common.Address, common.Hash, common.Hash, txmgrtypes.ChainReceipt[common.Hash, common.Hash], types.Sequence, feetypes.Fee, - ](chainID, mockKeyStore, mockEventRecorder) - require.NoError(t, err) - - tts := []struct { - scenario string - createTransactionInput createTransactionInput - createTransactionOutput createTransactionOutput - findTxWithIdempotencyKeyInput findTxWithIdempotencyKeyInput - findTxWithIdempotencyKeyOutput findTxWithIdempotencyKeyOutput - checkTxQueueCapacityInput checkTxQueueCapacityInput - checkTxQueueCapacityOutput checkTxQueueCapacityOutput - }{ - { - scenario: "success", - createTransactionInput: createTransactionInput{ - txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ - IdempotencyKey: &idempotencyKey, - FromAddress: fromAddress, - ToAddress: common.BytesToAddress([]byte("test")), - EncodedPayload: []byte{1, 2, 3}, - FeeLimit: uint32(1000), - Meta: nil, - Strategy: nil, //TODO - }, - chainID: chainID, - }, - createTransactionOutput: createTransactionOutput{ - tx: txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee]{ - IdempotencyKey: &idempotencyKey, - CreatedAt: time.Now().UTC(), - State: txmgr.TxUnstarted, - ChainID: chainID, - FromAddress: fromAddress, - ToAddress: common.BytesToAddress([]byte("test")), - EncodedPayload: []byte{1, 2, 3}, - FeeLimit: uint32(1000), - Meta: nil, - }, - err: nil, - }, - findTxWithIdempotencyKeyInput: findTxWithIdempotencyKeyInput{ - idempotencyKey: "11", - chainID: chainID, - }, - findTxWithIdempotencyKeyOutput: findTxWithIdempotencyKeyOutput{ - tx: txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee]{ - IdempotencyKey: &idempotencyKey, - CreatedAt: time.Now().UTC(), - State: txmgr.TxUnstarted, - ChainID: chainID, - FromAddress: fromAddress, - ToAddress: common.BytesToAddress([]byte("test")), - EncodedPayload: []byte{1, 2, 3}, - FeeLimit: uint32(1000), - Meta: nil, - }, - }, - checkTxQueueCapacityInput: checkTxQueueCapacityInput{ - fromAddress: fromAddress, - maxQueued: uint64(16), - chainID: chainID, - }, - checkTxQueueCapacityOutput: checkTxQueueCapacityOutput{ - err: nil, - }, - }, - } - - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actTx, actErr := ims.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) - require.Equal(t, tt.createTransactionOutput.err, actErr, "CreateTransaction: expected err to match actual err") - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, tt.createTransactionOutput.tx.CreatedAt, actTx.CreatedAt, time.Second, "CreateTransaction: expected time to be within 1 second of actual time") - // Reset CreatedAt to avoid flaky test - tt.createTransactionOutput.tx.CreatedAt = actTx.CreatedAt - assert.Equal(t, tt.createTransactionOutput.tx, actTx, "CreateTransaction: expected tx to match actual tx") - - actTxPtr, actErr := ims.FindTxWithIdempotencyKey(ctx, tt.findTxWithIdempotencyKeyInput.idempotencyKey, tt.findTxWithIdempotencyKeyInput.chainID) - require.Equal(t, tt.findTxWithIdempotencyKeyOutput.err, actErr, "FindTxWithIdempotencyKey: expected err to match actual err") - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, tt.findTxWithIdempotencyKeyOutput.tx.CreatedAt, actTx.CreatedAt, time.Second, "FindTxWithIdempotencyKey: expected time to be within 1 second of actual time") - // Reset CreatedAt to avoid flaky test - tt.findTxWithIdempotencyKeyOutput.tx.CreatedAt = actTxPtr.CreatedAt - assert.Equal(t, tt.findTxWithIdempotencyKeyOutput.tx, actTx, "FindTxWithIdempotencyKey: expected tx to match actual tx") - - actErr = ims.CheckTxQueueCapacity(ctx, tt.checkTxQueueCapacityInput.fromAddress, tt.checkTxQueueCapacityInput.maxQueued, tt.checkTxQueueCapacityInput.chainID) - require.Equal(t, tt.checkTxQueueCapacityOutput.err, actErr, "CheckTxQueueCapacity: expected err to match actual err") - }) - } - -} - -type createTransactionInput struct { - txRequest txmgrtypes.TxRequest[common.Address, common.Hash] - chainID *big.Int -} -type createTransactionOutput struct { - tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee] - err error -} -type findTxWithIdempotencyKeyInput struct { - idempotencyKey string - chainID *big.Int -} -type findTxWithIdempotencyKeyOutput struct { - tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, types.Sequence, feetypes.Fee] - err error -} -type checkTxQueueCapacityInput struct { - fromAddress common.Address - maxQueued uint64 - chainID *big.Int -} -type checkTxQueueCapacityOutput struct { - err error -} diff --git a/core/chains/inmemory_store_test.go b/core/chains/inmemory_store_test.go new file mode 100644 index 00000000000..9c269761efa --- /dev/null +++ b/core/chains/inmemory_store_test.go @@ -0,0 +1,153 @@ +package chains_test + +import ( + "context" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink/v2/common/txmgr" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + commontxmmocks "github.com/smartcontractkit/chainlink/v2/common/txmgr/types/mocks" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + evmtxmgr "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/pg/datatypes" +) + +func TestInMemoryStore_CreateTransaction(t *testing.T) { + db := pgtest.NewSqlxDB(t) + cfg := configtest.NewGeneralConfig(t, nil) + idempotencyKey := "11" + lggr := logger.TestLogger(t) + txStore := evmtxmgr.NewTxStore(db, lggr, cfg.Database()) + keyStore := cltest.NewKeyStore(t, db, cfg.Database()) + _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) + + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + chainID := ethClient.ConfiguredChainID() + + subject := uuid.New() + strategy := commontxmmocks.NewTxStrategy(t) + strategy.On("Subject").Return(uuid.NullUUID{UUID: subject, Valid: true}) + strategy.On("PruneQueue", mock.Anything, mock.AnythingOfType("*txmgr.evmTxStore")).Return(int64(0), nil) + ctx := context.Background() + + ims, err := txmgr.NewInMemoryStore[ + *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, + ](chainID, keyStore.Eth(), txStore) + require.NoError(t, err) + + tts := []struct { + scenario string + createTransactionInput createTransactionInput + createTransactionOutputCheck func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) + findTxWithIdempotencyKeyInput findTxWithIdempotencyKeyInput + findTxWithIdempotencyKeyOutput func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) + checkTxQueueCapacityInput checkTxQueueCapacityInput + checkTxQueueCapacityOutput checkTxQueueCapacityOutput + }{ + { + scenario: "success", + createTransactionInput: createTransactionInput{ + txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + ToAddress: common.BytesToAddress([]byte("test")), + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: uint32(1000), + Meta: nil, + Strategy: strategy, + }, + chainID: chainID, + }, + createTransactionOutputCheck: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + funcName := "CreateTransaction" + require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) + assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) + assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) + assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) + assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) + assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) + assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) + assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) + var expMeta *datatypes.JSON + assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) + assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) + }, + findTxWithIdempotencyKeyInput: findTxWithIdempotencyKeyInput{ + idempotencyKey: "11", + chainID: chainID, + }, + findTxWithIdempotencyKeyOutput: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + funcName := "FindTxWithIdempotencyKey" + require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) + assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) + assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) + assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) + assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) + assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) + assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) + assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) + var expMeta *datatypes.JSON + assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) + assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) + }, + checkTxQueueCapacityInput: checkTxQueueCapacityInput{ + fromAddress: fromAddress, + maxQueued: uint64(16), + chainID: chainID, + }, + checkTxQueueCapacityOutput: checkTxQueueCapacityOutput{ + err: nil, + }, + }, + } + + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actTx, actErr := ims.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) + tt.createTransactionOutputCheck(t, actTx, actErr) + + actTxPtr, actErr := ims.FindTxWithIdempotencyKey(ctx, tt.findTxWithIdempotencyKeyInput.idempotencyKey, tt.findTxWithIdempotencyKeyInput.chainID) + tt.findTxWithIdempotencyKeyOutput(t, *actTxPtr, actErr) + + actErr = ims.CheckTxQueueCapacity(ctx, tt.checkTxQueueCapacityInput.fromAddress, tt.checkTxQueueCapacityInput.maxQueued, tt.checkTxQueueCapacityInput.chainID) + require.Equal(t, tt.checkTxQueueCapacityOutput.err, actErr, "CheckTxQueueCapacity: expected err to match actual err") + }) + } + +} + +type createTransactionInput struct { + txRequest txmgrtypes.TxRequest[common.Address, common.Hash] + chainID *big.Int +} +type findTxWithIdempotencyKeyInput struct { + idempotencyKey string + chainID *big.Int +} +type checkTxQueueCapacityInput struct { + fromAddress common.Address + maxQueued uint64 + chainID *big.Int +} +type checkTxQueueCapacityOutput struct { + err error +} From 201a2ddb83a70ca64828eeb587d6a10bd66ab391 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 6 Nov 2023 10:57:27 -0500 Subject: [PATCH 04/74] add some functions for the in memory store --- common/txmgr/inmemory_store.go | 360 ++++++++++++++++++--------------- 1 file changed, 194 insertions(+), 166 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 4bf07bad474..0861703919c 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -19,8 +19,8 @@ var ( ErrTxnNotFound = fmt.Errorf("transaction not found") // ErrExistingIdempotencyKey is returned when a transaction with the same idempotency key already exists ErrExistingIdempotencyKey = fmt.Errorf("transaction with idempotency key already exists") - // ErrExistingPilelineTaskRunId is returned when a transaction with the same pipeline task run id already exists - ErrExistingPilelineTaskRunId = fmt.Errorf("transaction with pipeline task run id already exists") + // ErrAddressNotFound is returned when an address is not found + ErrAddressNotFound = fmt.Errorf("address not found") ) // Store and update all transaction state as files @@ -34,7 +34,6 @@ var ( // 4. Transaction Manager sends the transaction (unstarted) to the Broadcaster Unstarted Queue // 4. Transaction Manager prunes the Unstarted Queue based on the transaction prune strategy -// NOTE(jtw): Gets triggered by postgres Events // NOTE(jtw): Only one transaction per address can be in_progress at a time // NOTE(jtw): Only one broadcasted attempt exists per transaction the rest are errored or abandoned // 1. Broadcaster assigns a sequence number to the transaction @@ -66,8 +65,7 @@ type InMemoryStore[ pendingLock sync.Mutex // NOTE(jtw): we might need to watch out for txns that finish and are removed from the pending map - pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - pendingPipelineTaskRunIds map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] unstarted map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] inprogress map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] @@ -90,8 +88,7 @@ func NewInMemoryStore[ keyStore: keyStore, txStore: txStore, - pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - pendingPipelineTaskRunIds: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, unstarted: map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, inprogress: map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, @@ -99,7 +96,7 @@ func NewInMemoryStore[ addresses, err := keyStore.EnabledAddressesForChain(chainID) if err != nil { - return nil, err + return nil, fmt.Errorf("new_in_memory_store: %w", err) } for _, fromAddr := range addresses { // Channel Buffer is set to something high to prevent blocking and allow the pruning to happen @@ -109,6 +106,193 @@ func NewInMemoryStore[ return &tm, nil } +// CreateTransaction creates a new transaction for a given txRequest. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + // TODO(jtw): do generic checks + + // Persist Transaction to persistent storage + tx, err := ms.txStore.CreateTransaction(ctx, txRequest, chainID) + if err != nil { + return tx, fmt.Errorf("create_transaction: %w", err) + } + if err := ms.sendTxToBroadcaster(tx); err != nil { + return tx, fmt.Errorf("create_transaction: %w", err) + } + + return tx, nil +} + +// FindTxWithIdempotencyKey returns a transaction with the given idempotency key +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + // TODO(jtw): this is a change from current functionality... it returns nil, nil if nothing found in other implementations + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_tx_with_idempotency_key: %w", ErrInvalidChainID) + } + if idempotencyKey == "" { + return nil, fmt.Errorf("find_tx_with_idempotency_key: idempotency key cannot be empty") + } + + ms.pendingLock.Lock() + defer ms.pendingLock.Unlock() + + tx, ok := ms.pendingIdempotencyKeys[idempotencyKey] + if !ok { + return nil, fmt.Errorf("find_tx_with_idempotency_key: %w", ErrTxnNotFound) + } + + return tx, nil +} + +// CheckTxQueueCapacity checks if the queue capacity has been reached for a given address +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckTxQueueCapacity(ctx context.Context, fromAddress ADDR, maxQueuedTransactions uint64, chainID CHAIN_ID) error { + if maxQueuedTransactions == 0 { + return nil + } + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("check_tx_queue_capacity: %w", ErrInvalidChainID) + } + if _, ok := ms.unstarted[fromAddress]; !ok { + return fmt.Errorf("check_tx_queue_capacity: %w", ErrAddressNotFound) + } + + count := uint64(len(ms.unstarted[fromAddress])) + if count >= maxQueuedTransactions { + return fmt.Errorf("check_tx_queue_capacity: cannot create transaction; too many unstarted transactions in the queue (%v/%v). %s", count, maxQueuedTransactions, label.MaxQueuedTransactionsWarning) + } + + return nil +} + +///// +// BROADCASTER FUNCTIONS +///// + +// FindLatestSequence returns the latest sequence number for a given address +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) { + // query the persistent storage since this method only gets called when the broadcaster is starting up. + // It is used to initialize the in-memory sequence map in the broadcaster + // NOTE(jtw): should the nextSequenceMap be moved to the in-memory store? + + // TODO(jtw): do generic checks + // TODO(jtw): maybe this should be handled now + seq, err := ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) + if err != nil { + return seq, fmt.Errorf("find_latest_sequence: %w", err) + } + + return seq, nil +} + +// CountUnconfirmedTransactions returns the number of unconfirmed transactions for a given address. +// Unconfirmed transactions are transactions that have been broadcast but not confirmed on-chain. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountUnconfirmedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (uint32, error) { + // NOTE(jtw): used to calculate total inflight transactions + if ms.chainID.String() != chainID.String() { + return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) + } + if _, ok := ms.unstarted[fromAddress]; !ok { + return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) + } + + // TODO(jtw): NEEDS TO BE UPDATED TO USE IN MEMORY STORE + return ms.txStore.CountUnconfirmedTransactions(ctx, fromAddress, chainID) +} + +// CountUnstartedTransactions returns the number of unstarted transactions for a given address. +// Unstarted transactions are transactions that have not been broadcast yet. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountUnstartedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (uint32, error) { + // NOTE(jtw): used to calculate total inflight transactions + if ms.chainID.String() != chainID.String() { + return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) + } + if _, ok := ms.unstarted[fromAddress]; !ok { + return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) + } + + return uint32(len(ms.unstarted[fromAddress])), nil +} + +// UpdateTxUnstartedToInProgress updates a transaction from unstarted to in_progress. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxUnstartedToInProgress( + ctx context.Context, + tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) error { + if tx.Sequence == nil { + return fmt.Errorf("update_tx_unstarted_to_in_progress: in_progress transaction must have a sequence number") + } + if tx.State != TxUnstarted { + return fmt.Errorf("update_tx_unstarted_to_in_progress: can only transition to in_progress from unstarted, transaction is currently %s", tx.State) + } + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("update_tx_unstarted_to_in_progress: attempt state must be in_progress") + } + + // Persist to persistent storage + if err := ms.txStore.UpdateTxUnstartedToInProgress(ctx, tx, attempt); err != nil { + return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) + } + tx.TxAttempts = append(tx.TxAttempts, *attempt) + + // Update in memory store + ms.inprogress[tx.FromAddress] = tx + + return nil +} + +// GetTxInProgress returns the in_progress transaction for a given address. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxInProgress(ctx context.Context, fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + tx, ok := ms.inprogress[fromAddress] + if !ok { + return nil, nil + } + if len(tx.TxAttempts) != 1 || tx.TxAttempts[0].State != txmgrtypes.TxAttemptInProgress { + return nil, fmt.Errorf("get_tx_in_progress: expected in_progress transaction %v to have exactly one unsent attempt. "+ + "Your database is in an inconsistent state and this node will not function correctly until the problem is resolved", tx.ID) + } + + return tx, nil +} + +// UpdateTxAttemptInProgressToBroadcast updates a transaction attempt from in_progress to broadcast. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxAttemptInProgressToBroadcast( + ctx context.Context, + tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + newAttemptState txmgrtypes.TxAttemptState, +) error { + // TODO(jtw) + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: not implemented") +} + +// FindNextUnstartedTransactionFromAddress returns the next unstarted transaction for a given address. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindNextUnstartedTransactionFromAddress(ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID) error { + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) + } + if _, ok := ms.unstarted[fromAddress]; !ok { + return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) + } + + return fmt.Errorf("find_next_unstarted_transaction_from_address: not implemented") +} + +// SaveReplacementInProgressAttempt saves a replacement attempt for a transaction that is in_progress. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveReplacementInProgressAttempt( + ctx context.Context, + oldAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + replacementAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) error { + // TODO(jtw) + return fmt.Errorf("save_replacement_in_progress_attempt: not implemented") +} + +// UpdateTxFatalError updates a transaction to fatal_error. +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + // TODO(jtw) + return fmt.Errorf("update_tx_fatal_error: not implemented") +} + // Close closes the InMemoryStore func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { // Close the event recorder @@ -122,16 +306,16 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close // Clear all pending requests ms.pendingLock.Lock() clear(ms.pendingIdempotencyKeys) - clear(ms.pendingPipelineTaskRunIds) ms.pendingLock.Unlock() } // Abandon removes all transactions for a given address func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Abandon(ctx context.Context, chainID CHAIN_ID, addr ADDR) error { if ms.chainID.String() != chainID.String() { - return ErrInvalidChainID + return fmt.Errorf("abandon: %w", ErrInvalidChainID) } + // TODO(jtw): do generic checks // Mark all persisted transactions as abandoned if err := ms.txStore.Abandon(ctx, chainID, addr); err != nil { return err @@ -165,28 +349,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband tx.Error = null.NewString("abandoned", true) } } - for _, tx := range ms.pendingPipelineTaskRunIds { - if tx.FromAddress == addr { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) - } - } // TODO(jtw): SHOULD THE REAPER BE RESPONSIBLE FOR CLEARING THE PENDING MAPS? return nil } -// CreateTransaction creates a new transaction for a given txRequest. -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - // Persist Transaction to persistent storage - tx, err := ms.txStore.CreateTransaction(ctx, txRequest, chainID) - if err != nil { - return tx, err - } - return tx, ms.sendTxToBroadcaster(tx) -} - // TODO(jtw): change naming to something more appropriate func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendTxToBroadcaster(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { // TODO(jtw); HANDLE PRUNING STEP @@ -200,9 +367,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendT if tx.IdempotencyKey != nil { ms.pendingIdempotencyKeys[*tx.IdempotencyKey] = &tx } - if tx.PipelineTaskRunID.UUID.String() != "" { - ms.pendingPipelineTaskRunIds[tx.PipelineTaskRunID.UUID.String()] = &tx - } ms.pendingLock.Unlock() return nil @@ -211,139 +375,3 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendT return fmt.Errorf("transaction manager queue capacity has been reached") } } - -// FindTxWithIdempotencyKey returns a transaction with the given idempotency key -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - // TODO(jtw): this is a change from current functionality... it returns nil, nil if nothing found in other implementations - if ms.chainID.String() != chainID.String() { - return nil, ErrInvalidChainID - } - if idempotencyKey == "" { - return nil, fmt.Errorf("FindTxWithIdempotencyKey: idempotency key cannot be empty") - } - - ms.pendingLock.Lock() - defer ms.pendingLock.Unlock() - - tx, ok := ms.pendingIdempotencyKeys[idempotencyKey] - if !ok { - return nil, fmt.Errorf("FindTxWithIdempotencyKey: transaction not found") - } - - return tx, nil -} - -// CheckTxQueueCapacity checks if the queue capacity has been reached for a given address -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckTxQueueCapacity(ctx context.Context, fromAddress ADDR, maxQueuedTransactions uint64, chainID CHAIN_ID) (err error) { - if maxQueuedTransactions == 0 { - return nil - } - if ms.chainID.String() != chainID.String() { - return ErrInvalidChainID - } - if _, ok := ms.unstarted[fromAddress]; !ok { - return fmt.Errorf("CheckTxQueueCapacity: address not found") - } - - count := uint64(len(ms.unstarted[fromAddress])) - if count >= maxQueuedTransactions { - return fmt.Errorf("CheckTxQueueCapacity: cannot create transaction; too many unstarted transactions in the queue (%v/%v). %s", count, maxQueuedTransactions, label.MaxQueuedTransactionsWarning) - } - - return nil -} - -/* -// BROADCASTER FUNCTIONS -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) CountUnconfirmedTransactions(_ context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) { - if ms.chainID != chainID { - return 0, ErrInvalidChainID - } - // TODO(jtw): NEED TO COMPLETE - return 0, nil -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) CountUnstartedTransactions(_ context.Context, fromAddress ADDR, chainID CHAIN_ID) (count uint32, err error) { - if ms.chainID != chainID { - return 0, ErrInvalidChainID - } - - return uint32(len(ms.unstarted[fromAddress])), nil -} -func (ms *InMemoryStore) FindNextUnstartedTransactionFromAddress(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID) error { - -} -func (ms *InMemoryStore) GetTxInProgress(ctx context.Context, fromAddress ADDR) (etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { -} - -func (ms *InMemoryStore) UpdateTxAttemptInProgressToBroadcast(etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState TxAttemptState, incrNextSequenceCallback QueryerFunc, qopts ...pg.QOpt) error { -} -func (ms *InMemoryStore) UpdateTxUnstartedToInProgress(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { -} -func (ms *InMemoryStore) UpdateTxFatalError(ctx context.Context, etx *Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { -} -func (ms *InMemoryStore) SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { -} - -// ProcessUnstartedTxs processes unstarted transactions -// TODO(jtw): SHOULD THIS BE CALLED THE BROADCASTER? -func (tm *TransactionManager) ProcessUnstartedTxs(ctx context.Context, fromAddress string) { - // if there are more in flight transactions than the max then throttle using the InFlightTransactionRecheckInterval - for { - select { - // NOTE: There can be at most one in_progress transaction per address. - case txReq := <-tm.Unstarted[fromAddress]: - // check number of in flight transactions to see if we can process more... MaxInFlight is for total inflight - tm.inFlightWG.Wait() - // Reserve a spot in the in flight transactions - tm.inFlightWG.Done() - - // TODO(jtw): THERE ARE SOME CHANGES AROUND ERROR FUNCTIONALITY - // EXAMPLE NO LONGER WILL ERROR IF THE NUMBER OF IN FLIGHT TRANSACTIONS IS EXCEEDED - if err := tm.PublishToChain(txReq); err != nil { - // TODO(jtw): Handle error properly - fmt.Println(err) - } - - // Free up a spot in the in flight transactions - tm.inFlightWG.Add(1) - case <-ctx.Done(): - return - } - } - -} - -// PublishToChain attempts to publish a transaction to the chain -// TODO(jtw): NO LONGER RETURNS AN ERROR IF FULL OF IN PROGRESS TRANSACTIONS... not sure if okay -func (tm *TransactionManager) PublishToChain(txReq TxRequest) error { - // Handle an unstarted request - // Get next sequence number from the KeyStore - // ks.NextSequence(fromAddress, tm.ChainID) - // Create a new transaction attempt to be put on chain - // eb.NewTxAttempt(ctx, txReq, logger) - - // IT BLOCKS UNTIL THERE IS A SPOT IN THE IN PROGRESS TRANSACTIONS - tm.Inprogress[txReq.FromAddress] <- txReq - - return nil -} -*/ - -/* -// Close closes the InMemoryStore -func (ms *InMemoryStore) Close() { - // Close all channels - for _, ch := range ms.Unstarted { - close(ch) - } - for _, ch := range ms.Inprogress { - close(ch) - } - - // Clear all pending requests - ms.pendingLock.Lock() - clear(ms.pendingIdempotencyKeys) - clear(ms.pendingPipelineTaskRunIds) - ms.pendingLock.Unlock() -} -*/ From b565d4507a14ccaaeb3455d8fceaa6d425da0598 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 6 Nov 2023 15:12:31 -0500 Subject: [PATCH 05/74] implement a few more methods for the in memory store --- common/txmgr/inmemory_store.go | 84 +++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 0861703919c..8a064db8c82 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -51,6 +51,8 @@ var ( // 4. Confirmer sets transactions that have failed to (unconfirmed) which will be retried by the resender // 5. Confirmer sets transactions that have been confirmed to (confirmed) and creates a new receipt which is persisted +// TODO(jtw): WHAT DO WE WANT TO DO WITH TX_ATTEMPTS? + type InMemoryStore[ CHAIN_ID types.ID, ADDR, TX_HASH, BLOCK_HASH types.Hashable, @@ -168,14 +170,16 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check ///// // FindLatestSequence returns the latest sequence number for a given address -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (seq SEQ, err error) { // query the persistent storage since this method only gets called when the broadcaster is starting up. // It is used to initialize the in-memory sequence map in the broadcaster // NOTE(jtw): should the nextSequenceMap be moved to the in-memory store? - // TODO(jtw): do generic checks - // TODO(jtw): maybe this should be handled now - seq, err := ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) + if ms.chainID.String() != chainID.String() { + return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) + } + + seq, err = ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) if err != nil { return seq, fmt.Errorf("find_latest_sequence: %w", err) } @@ -258,15 +262,39 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxAttemptInProgressToBroadcast( ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], newAttemptState txmgrtypes.TxAttemptState, ) error { - // TODO(jtw) - return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: not implemented") + if tx.BroadcastAt == nil { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: unconfirmed transaction must have broadcast)at time") + } + if tx.InitialBroadcastAt == nil { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: unconfirmed transaction must have initial_broadcast_at time") + } + if tx.State != TxInProgress { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: can only transition to unconfirmed from in_progress, transaction is currently %s", tx.State) + } + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: attempt must be in in_progress state") + } + if newAttemptState != txmgrtypes.TxAttemptBroadcast { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: new attempt state must be broadcast, got: %s", newAttemptState) + } + + // Persist to persistent storage + if err := ms.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, tx, attempt, newAttemptState); err != nil { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) + } + // Ensure that the tx state is updated to unconfirmed since this is a chain agnostic operation + tx.State = TxUnconfirmed + // NOTE(jtw): attempt is not getting updated in memory yet... only in persistent storage + + return nil } // FindNextUnstartedTransactionFromAddress returns the next unstarted transaction for a given address. -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindNextUnstartedTransactionFromAddress(ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID) error { +// NOTE(jtw): method signature is different from most other signatures where the tx is passed in and updated +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindNextUnstartedTransactionFromAddress(_ context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID) error { if ms.chainID.String() != chainID.String() { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } @@ -274,22 +302,54 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) } - return fmt.Errorf("find_next_unstarted_transaction_from_address: not implemented") + select { + case tx = <-ms.unstarted[fromAddress]: + return nil + default: + return fmt.Errorf("find_next_unstarted_transaction_from_address: failed to FindNextUnstartedTransactionFromAddress") + } } // SaveReplacementInProgressAttempt saves a replacement attempt for a transaction that is in_progress. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveReplacementInProgressAttempt( ctx context.Context, oldAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - replacementAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + replacementAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) error { - // TODO(jtw) + if oldAttempt.State != txmgrtypes.TxAttemptInProgress || replacementAttempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("save_replacement_in_progress_attempt: expected attempts to be in_progress") + } + if oldAttempt.ID == 0 { + return fmt.Errorf("save_replacement_in_progress_attempt: expected oldattempt to have an ID") + } + + // Persist to persistent storage + if err := ms.txStore.SaveReplacementInProgressAttempt(ctx, oldAttempt, replacementAttempt); err != nil { + return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) + } + + // TODO(jtw): finish implementing return fmt.Errorf("save_replacement_in_progress_attempt: not implemented") } // UpdateTxFatalError updates a transaction to fatal_error. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - // TODO(jtw) + if tx.State != TxInProgress { + return fmt.Errorf("update_tx_fatal_error: can only transition to fatal_error from in_progress, transaction is currently %s", tx.State) + } + if !tx.Error.Valid { + return fmt.Errorf("update_tx_fatal_error: expected error field to be set") + } + + // Persist to persistent storage + if err := ms.txStore.UpdateTxFatalError(ctx, tx); err != nil { + return fmt.Errorf("update_tx_fatal_error: %w", err) + } + + // Ensure that the tx state is updated to fatal_error since this is a chain agnostic operation + tx.Sequence = nil + tx.State = TxFatalError + return fmt.Errorf("update_tx_fatal_error: not implemented") } From faf670f7220aac5a4d3c17f986a1b8702965f638 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 6 Nov 2023 19:58:53 -0500 Subject: [PATCH 06/74] some clean up --- common/txmgr/inmemory_store.go | 143 ++++++++++++++++++++++++++------- common/txmgr/txmgr.go | 2 +- 2 files changed, 114 insertions(+), 31 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 8a064db8c82..b1c791b4bcf 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -51,8 +51,6 @@ var ( // 4. Confirmer sets transactions that have failed to (unconfirmed) which will be retried by the resender // 5. Confirmer sets transactions that have been confirmed to (confirmed) and creates a new receipt which is persisted -// TODO(jtw): WHAT DO WE WANT TO DO WITH TX_ATTEMPTS? - type InMemoryStore[ CHAIN_ID types.ID, ADDR, TX_HASH, BLOCK_HASH types.Hashable, @@ -65,12 +63,18 @@ type InMemoryStore[ keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - pendingLock sync.Mutex + pendingLock sync.RWMutex // NOTE(jtw): we might need to watch out for txns that finish and are removed from the pending map pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - unstarted map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - inprogress map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // unstarted is a map of addresses to a channel of unstarted transactions + unstarted map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // inprogress is a map of addresses to inprogress transactions + inprogressLock sync.RWMutex + inprogress map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // unconfirmed is a map of addresses to a map of transaction IDs to unconfirmed transactions + unconfirmedLock sync.RWMutex + unconfirmed map[ADDR]map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } // NewInMemoryStore returns a new InMemoryStore @@ -85,15 +89,16 @@ func NewInMemoryStore[ keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], ) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { - tm := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + ms := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ chainID: chainID, keyStore: keyStore, txStore: txStore, pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - unstarted: map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - inprogress: map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + unstarted: map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + inprogress: map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + unconfirmed: map[ADDR]map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } addresses, err := keyStore.EnabledAddressesForChain(chainID) @@ -102,22 +107,28 @@ func NewInMemoryStore[ } for _, fromAddr := range addresses { // Channel Buffer is set to something high to prevent blocking and allow the pruning to happen - tm.unstarted[fromAddr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) + ms.unstarted[fromAddr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) + ms.unconfirmed[fromAddr] = map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} } - return &tm, nil + return &ms, nil } // CreateTransaction creates a new transaction for a given txRequest. -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - // TODO(jtw): do generic checks +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + if ms.chainID.String() != chainID.String() { + return tx, fmt.Errorf("create_transaction: %w", ErrInvalidChainID) + } + if _, ok := ms.unstarted[txRequest.FromAddress]; !ok { + return tx, fmt.Errorf("create_transaction: %w", ErrAddressNotFound) + } // Persist Transaction to persistent storage - tx, err := ms.txStore.CreateTransaction(ctx, txRequest, chainID) + tx, err = ms.txStore.CreateTransaction(ctx, txRequest, chainID) if err != nil { return tx, fmt.Errorf("create_transaction: %w", err) } - if err := ms.sendTxToBroadcaster(tx); err != nil { + if err := ms.sendTxToUnstartedQueue(tx); err != nil { return tx, fmt.Errorf("create_transaction: %w", err) } @@ -126,13 +137,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat // FindTxWithIdempotencyKey returns a transaction with the given idempotency key func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithIdempotencyKey(ctx context.Context, idempotencyKey string, chainID CHAIN_ID) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - // TODO(jtw): this is a change from current functionality... it returns nil, nil if nothing found in other implementations if ms.chainID.String() != chainID.String() { return nil, fmt.Errorf("find_tx_with_idempotency_key: %w", ErrInvalidChainID) } - if idempotencyKey == "" { - return nil, fmt.Errorf("find_tx_with_idempotency_key: idempotency key cannot be empty") - } ms.pendingLock.Lock() defer ms.pendingLock.Unlock() @@ -178,6 +185,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL if ms.chainID.String() != chainID.String() { return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) } + if _, ok := ms.unstarted[fromAddress]; !ok { + return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) + } seq, err = ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) if err != nil { @@ -189,23 +199,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL // CountUnconfirmedTransactions returns the number of unconfirmed transactions for a given address. // Unconfirmed transactions are transactions that have been broadcast but not confirmed on-chain. +// NOTE(jtw): used to calculate total inflight transactions func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountUnconfirmedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (uint32, error) { - // NOTE(jtw): used to calculate total inflight transactions if ms.chainID.String() != chainID.String() { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } - if _, ok := ms.unstarted[fromAddress]; !ok { + u, ok := ms.unconfirmed[fromAddress] + if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - // TODO(jtw): NEEDS TO BE UPDATED TO USE IN MEMORY STORE - return ms.txStore.CountUnconfirmedTransactions(ctx, fromAddress, chainID) + return uint32(len(u)), nil } // CountUnstartedTransactions returns the number of unstarted transactions for a given address. // Unstarted transactions are transactions that have not been broadcast yet. +// NOTE(jtw): used to calculate total inflight transactions func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountUnstartedTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (uint32, error) { - // NOTE(jtw): used to calculate total inflight transactions if ms.chainID.String() != chainID.String() { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } @@ -239,13 +249,17 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat tx.TxAttempts = append(tx.TxAttempts, *attempt) // Update in memory store + ms.inprogressLock.Lock() ms.inprogress[tx.FromAddress] = tx + ms.inprogressLock.Unlock() return nil } // GetTxInProgress returns the in_progress transaction for a given address. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxInProgress(ctx context.Context, fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + ms.inprogressLock.RLock() + defer ms.inprogressLock.RUnlock() tx, ok := ms.inprogress[fromAddress] if !ok { return nil, nil @@ -287,7 +301,28 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Ensure that the tx state is updated to unconfirmed since this is a chain agnostic operation tx.State = TxUnconfirmed - // NOTE(jtw): attempt is not getting updated in memory yet... only in persistent storage + attempt.State = newAttemptState + var found bool + for i := 0; i < len(tx.TxAttempts); i++ { + if tx.TxAttempts[i].ID == attempt.ID { + tx.TxAttempts[i] = attempt + found = true + } + } + if !found { + tx.TxAttempts = append(tx.TxAttempts, attempt) + // NOTE(jtw): should this log a warning? + } + + // remove the transaction from the inprogress map + ms.inprogressLock.Lock() + ms.inprogress[tx.FromAddress] = nil + ms.inprogressLock.Unlock() + + // add the transaction to the unconfirmed map + ms.unconfirmedLock.Lock() + ms.unconfirmed[tx.FromAddress][tx.ID] = tx + ms.unconfirmedLock.Unlock() return nil } @@ -328,7 +363,26 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) } - // TODO(jtw): finish implementing + // Update in memory store + ms.inprogressLock.Lock() + tx, ok := ms.inprogress[oldAttempt.Tx.FromAddress] + if tx == nil || !ok { + ms.inprogressLock.Unlock() + return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) + } + var found bool + for i := 0; i < len(tx.TxAttempts); i++ { + if tx.TxAttempts[i].ID == oldAttempt.ID { + tx.TxAttempts[i] = *replacementAttempt + found = true + } + } + if !found { + tx.TxAttempts = append(tx.TxAttempts, *replacementAttempt) + // NOTE(jtw): should this log a warning? + } + ms.inprogressLock.Unlock() + return fmt.Errorf("save_replacement_in_progress_attempt: not implemented") } @@ -367,6 +421,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close ms.pendingLock.Lock() clear(ms.pendingIdempotencyKeys) ms.pendingLock.Unlock() + // Clear all unstarted transactions + ms.inprogressLock.Lock() + clear(ms.inprogress) + ms.inprogressLock.Unlock() + // Clear all unconfirmed transactions + ms.unconfirmedLock.Lock() + clear(ms.unconfirmed) + ms.unconfirmedLock.Unlock() } // Abandon removes all transactions for a given address @@ -375,12 +437,15 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband return fmt.Errorf("abandon: %w", ErrInvalidChainID) } - // TODO(jtw): do generic checks // Mark all persisted transactions as abandoned if err := ms.txStore.Abandon(ctx, chainID, addr); err != nil { return err } + // check that the address exists in the unstarted transactions + if _, ok := ms.unstarted[addr]; !ok { + return fmt.Errorf("abandon: %w", ErrAddressNotFound) + } // Mark all unstarted transactions as abandoned close(ms.unstarted[addr]) for tx := range ms.unstarted[addr] { @@ -391,6 +456,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband // reset the unstarted channel ms.unstarted[addr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) + ms.inprogressLock.Lock() + if _, ok := ms.inprogress[addr]; !ok { + ms.inprogressLock.Unlock() + return fmt.Errorf("abandon: %w", ErrAddressNotFound) + } // Mark all inprogress transactions as abandoned if tx, ok := ms.inprogress[addr]; ok { tx.State = TxFatalError @@ -398,9 +468,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband tx.Error = null.NewString("abandoned", true) } ms.inprogress[addr] = nil + ms.inprogressLock.Unlock() - // TODO(jtw): Mark all unconfirmed transactions as abandoned + ms.unconfirmedLock.Lock() + if _, ok := ms.unconfirmed[addr]; !ok { + ms.unconfirmedLock.Unlock() + return fmt.Errorf("abandon: %w", ErrAddressNotFound) + } + // Mark all unconfirmed transactions as abandoned + for _, tx := range ms.unconfirmed[addr] { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + ms.unconfirmed[addr] = nil + ms.unconfirmedLock.Unlock() + ms.pendingLock.Lock() // Mark all pending transactions as abandoned for _, tx := range ms.pendingIdempotencyKeys { if tx.FromAddress == addr { @@ -409,13 +493,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband tx.Error = null.NewString("abandoned", true) } } - // TODO(jtw): SHOULD THE REAPER BE RESPONSIBLE FOR CLEARING THE PENDING MAPS? + ms.pendingLock.Unlock() return nil } -// TODO(jtw): change naming to something more appropriate -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendTxToBroadcaster(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendTxToUnstartedQueue(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { // TODO(jtw); HANDLE PRUNING STEP select { diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index 5b7afd32242..46db9e08767 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -452,7 +452,7 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTran if txRequest.IdempotencyKey != nil { var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) - if err != nil && !errors.Is(err, sql.ErrNoRows) { + if err != nil && !errors.Is(err, sql.ErrNoRows) && !errors.Is(err, ErrTxnNotFound) { return tx, fmt.Errorf("Failed to search for transaction with IdempotencyKey: %w", err) } if existingTx != nil { From 178b0f03aef87e41078a30af4f1f9646b7cdfe8c Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 6 Nov 2023 20:12:22 -0500 Subject: [PATCH 07/74] add check for inprogress txn --- common/txmgr/broadcaster.go | 2 +- common/txmgr/inmemory_store.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/common/txmgr/broadcaster.go b/common/txmgr/broadcaster.go index 4f6ffae2ad8..3936014584f 100644 --- a/common/txmgr/broadcaster.go +++ b/common/txmgr/broadcaster.go @@ -695,7 +695,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) next defer cancel() etx := &txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} if err := eb.txStore.FindNextUnstartedTransactionFromAddress(ctx, etx, fromAddress, eb.chainID); err != nil { - if errors.Is(err, sql.ErrNoRows) { + if errors.Is(err, sql.ErrNoRows) || errors.Is(err, ErrTxnNotFound) { // Finish. No more transactions left to process. Hoorah! return nil, nil } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index b1c791b4bcf..504d5ea35a9 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -336,12 +336,19 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN if _, ok := ms.unstarted[fromAddress]; !ok { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) } + // ensure that the address is not already busy with a transaction in progress + ms.inprogressLock.RLock() + if ms.inprogress[fromAddress] != nil { + ms.inprogressLock.RUnlock() + return fmt.Errorf("find_next_unstarted_transaction_from_address: address %s is already busy with a transaction in progress", fromAddress) + } + ms.inprogressLock.RUnlock() select { case tx = <-ms.unstarted[fromAddress]: return nil default: - return fmt.Errorf("find_next_unstarted_transaction_from_address: failed to FindNextUnstartedTransactionFromAddress") + return ErrTxnNotFound } } From fe25a2649a5cafa94df20340fe8a3740e5c43e46 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 6 Nov 2023 22:08:01 -0500 Subject: [PATCH 08/74] make some changes to testing --- core/chains/inmemory_store_test.go | 153 ------------------ core/chains/tx_store_test.go | 242 +++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 153 deletions(-) delete mode 100644 core/chains/inmemory_store_test.go create mode 100644 core/chains/tx_store_test.go diff --git a/core/chains/inmemory_store_test.go b/core/chains/inmemory_store_test.go deleted file mode 100644 index 9c269761efa..00000000000 --- a/core/chains/inmemory_store_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package chains_test - -import ( - "context" - "fmt" - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink/v2/common/txmgr" - txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" - commontxmmocks "github.com/smartcontractkit/chainlink/v2/common/txmgr/types/mocks" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" - evmtxmgr "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" - "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/pg/datatypes" -) - -func TestInMemoryStore_CreateTransaction(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - idempotencyKey := "11" - lggr := logger.TestLogger(t) - txStore := evmtxmgr.NewTxStore(db, lggr, cfg.Database()) - keyStore := cltest.NewKeyStore(t, db, cfg.Database()) - _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) - - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - chainID := ethClient.ConfiguredChainID() - - subject := uuid.New() - strategy := commontxmmocks.NewTxStrategy(t) - strategy.On("Subject").Return(uuid.NullUUID{UUID: subject, Valid: true}) - strategy.On("PruneQueue", mock.Anything, mock.AnythingOfType("*txmgr.evmTxStore")).Return(int64(0), nil) - ctx := context.Background() - - ims, err := txmgr.NewInMemoryStore[ - *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, - ](chainID, keyStore.Eth(), txStore) - require.NoError(t, err) - - tts := []struct { - scenario string - createTransactionInput createTransactionInput - createTransactionOutputCheck func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) - findTxWithIdempotencyKeyInput findTxWithIdempotencyKeyInput - findTxWithIdempotencyKeyOutput func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) - checkTxQueueCapacityInput checkTxQueueCapacityInput - checkTxQueueCapacityOutput checkTxQueueCapacityOutput - }{ - { - scenario: "success", - createTransactionInput: createTransactionInput{ - txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ - IdempotencyKey: &idempotencyKey, - FromAddress: fromAddress, - ToAddress: common.BytesToAddress([]byte("test")), - EncodedPayload: []byte{1, 2, 3}, - FeeLimit: uint32(1000), - Meta: nil, - Strategy: strategy, - }, - chainID: chainID, - }, - createTransactionOutputCheck: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { - funcName := "CreateTransaction" - require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) - assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) - assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) - assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) - assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) - assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) - assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) - assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) - var expMeta *datatypes.JSON - assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) - assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) - }, - findTxWithIdempotencyKeyInput: findTxWithIdempotencyKeyInput{ - idempotencyKey: "11", - chainID: chainID, - }, - findTxWithIdempotencyKeyOutput: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { - funcName := "FindTxWithIdempotencyKey" - require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) - assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) - assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) - assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) - assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) - assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) - assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) - assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) - var expMeta *datatypes.JSON - assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) - assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) - }, - checkTxQueueCapacityInput: checkTxQueueCapacityInput{ - fromAddress: fromAddress, - maxQueued: uint64(16), - chainID: chainID, - }, - checkTxQueueCapacityOutput: checkTxQueueCapacityOutput{ - err: nil, - }, - }, - } - - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actTx, actErr := ims.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) - tt.createTransactionOutputCheck(t, actTx, actErr) - - actTxPtr, actErr := ims.FindTxWithIdempotencyKey(ctx, tt.findTxWithIdempotencyKeyInput.idempotencyKey, tt.findTxWithIdempotencyKeyInput.chainID) - tt.findTxWithIdempotencyKeyOutput(t, *actTxPtr, actErr) - - actErr = ims.CheckTxQueueCapacity(ctx, tt.checkTxQueueCapacityInput.fromAddress, tt.checkTxQueueCapacityInput.maxQueued, tt.checkTxQueueCapacityInput.chainID) - require.Equal(t, tt.checkTxQueueCapacityOutput.err, actErr, "CheckTxQueueCapacity: expected err to match actual err") - }) - } - -} - -type createTransactionInput struct { - txRequest txmgrtypes.TxRequest[common.Address, common.Hash] - chainID *big.Int -} -type findTxWithIdempotencyKeyInput struct { - idempotencyKey string - chainID *big.Int -} -type checkTxQueueCapacityInput struct { - fromAddress common.Address - maxQueued uint64 - chainID *big.Int -} -type checkTxQueueCapacityOutput struct { - err error -} diff --git a/core/chains/tx_store_test.go b/core/chains/tx_store_test.go new file mode 100644 index 00000000000..abdde942458 --- /dev/null +++ b/core/chains/tx_store_test.go @@ -0,0 +1,242 @@ +package chains_test + +import ( + "context" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" + "github.com/smartcontractkit/chainlink/v2/common/txmgr" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + commontxmmocks "github.com/smartcontractkit/chainlink/v2/common/txmgr/types/mocks" + "github.com/smartcontractkit/chainlink/v2/common/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" + "github.com/smartcontractkit/chainlink/v2/core/services/pg/datatypes" +) + +type TestingTxStore[ + ADDR types.Hashable, + CHAIN_ID types.ID, + TX_HASH types.Hashable, + BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] interface { + CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, gas.EvmFee], err error) +} + +type txStoreFunc func(t *testing.T) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address) + +func evmTxStore(t *testing.T) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address) { + db := pgtest.NewSqlxDB(t) + cfg := configtest.NewGeneralConfig(t, nil) + keyStore := cltest.NewKeyStore(t, db, cfg.Database()) + _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) + + return cltest.NewTxStore(t, db, cfg.Database()), fromAddress +} + +var txStoresFuncs = []txStoreFunc{ + evmTxStore, + /* + ims, err := txmgr.NewInMemoryStore[ + *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, + ](chainID, keyStore.Eth(), txStore) + */ +} + +func TestTxStore_CreateTransaction(t *testing.T) { + for _, f := range txStoresFuncs { + txStore, fromAddress := f(t) + + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + subject := uuid.New() + strategy := commontxmmocks.NewTxStrategy(t) + strategy.On("Subject").Return(uuid.NullUUID{UUID: subject, Valid: true}) + strategy.On("PruneQueue", mock.Anything, mock.AnythingOfType("*txmgr.evmTxStore")).Return(int64(0), nil) + ctx := context.Background() + idempotencyKey := "11" + + tts := []struct { + scenario string + createTransactionInput createTransactionInput + createTransactionOutputCheck func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) + }{ + { + scenario: "success", + createTransactionInput: createTransactionInput{ + txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + ToAddress: common.BytesToAddress([]byte("test")), + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: uint32(1000), + Meta: nil, + Strategy: strategy, + }, + chainID: ethClient.ConfiguredChainID(), + }, + createTransactionOutputCheck: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + funcName := "CreateTransaction" + require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) + assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) + assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) + assert.Equal(t, ethClient.ConfiguredChainID(), tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) + assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) + assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) + assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) + assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) + var expMeta *datatypes.JSON + assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) + assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) + }, + }, + } + + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actTx, actErr := txStore.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) + tt.createTransactionOutputCheck(t, actTx, actErr) + + // TODO(jtw): Check that the transaction was persisted + }) + } + } +} + +/* +func TestTxStore_FindTxWithIdempotencyKey(t *testing.T) { + txStore := evmtxmgr.NewTxStore(nil, nil, nil) + ctx := context.Background() + + tts := []struct { + scenario string + findTxWithIdempotencyKeyInput findTxWithIdempotencyKeyInput + findTxWithIdempotencyKeyOutput func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) + }{ + { + findTxWithIdempotencyKeyInput: findTxWithIdempotencyKeyInput{ + idempotencyKey: "11", + chainID: chainID, + }, + findTxWithIdempotencyKeyOutput: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + funcName := "FindTxWithIdempotencyKey" + require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) + assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) + assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) + assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) + assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) + assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) + assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) + assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) + var expMeta *datatypes.JSON + assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) + assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) + }, + }, + } + + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actTxPtr, actErr := txStore.FindTxWithIdempotencyKey(ctx, tt.findTxWithIdempotencyKeyInput.idempotencyKey, tt.findTxWithIdempotencyKeyInput.chainID) + tt.findTxWithIdempotencyKeyOutput(t, *actTxPtr, actErr) + }) + } +} + +func TestTxStore_CheckTxQueueCapacity(t *testing.T) { + txStore := evmtxmgr.NewTxStore(nil, nil, nil) + ctx := context.Background() + + tts := []struct { + scenario string + checkTxQueueCapacityInput checkTxQueueCapacityInput + expErr error + }{ + { + checkTxQueueCapacityInput: checkTxQueueCapacityInput{ + fromAddress: fromAddress, + maxQueued: uint64(16), + chainID: chainID, + }, + expErr: nil, + }, + } + + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actErr := txStore.CheckTxQueueCapacity(ctx, tt.checkTxQueueCapacityInput.fromAddress, tt.checkTxQueueCapacityInput.maxQueued, tt.checkTxQueueCapacityInput.chainID) + require.Equal(t, tt.expErr, actErr, "CheckTxQueueCapacity: expected err to match actual err") + }) + } +} + +func TestTxStore_FindLatestSequence(t *testing.T) { + txStore := evmtxmgr.NewTxStore(nil, nil, nil) + ctx := context.Background() + + tts := []struct { + scenario string + findLatestSequenceInput findLatestSequenceInput + findLatestSequenceOutput func(*testing.T, evmtypes.Nonce, error) + }{ + { + findLatestSequenceInput: findLatestSequenceInput{ + fromAddress: fromAddress, + chainID: chainID, + }, + findLatestSequenceOutput: func(t *testing.T, seq evmtypes.Nonce, err error) { + funcName := "FindLatestSequence" + require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) + assert.Equal(t, uint64(0), seq, fmt.Sprintf("%s: expected seq to match actual seq", funcName)) + }, + }, + } + + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actSeq, actErr := txStore.FindLatestSequence(ctx, tt.findLatestSequenceInput.fromAddress, tt.findLatestSequenceInput.chainID) + tt.findLatestSequenceOutput(t, actSeq, actErr) + }) + } +} +*/ + +type createTransactionInput struct { + txRequest txmgrtypes.TxRequest[common.Address, common.Hash] + chainID *big.Int +} +type findTxWithIdempotencyKeyInput struct { + idempotencyKey string + chainID *big.Int +} +type checkTxQueueCapacityInput struct { + fromAddress common.Address + maxQueued uint64 + chainID *big.Int +} +type checkTxQueueCapacityOutput struct { + err error +} +type findLatestSequenceInput struct { + fromAddress common.Address + chainID *big.Int +} From a0ac2a3a1fd1ba2ac403c32af3f6032fc07222e8 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 7 Nov 2023 11:12:26 -0500 Subject: [PATCH 09/74] reorganize tests --- core/chains/tx_store_test.go | 146 ++++++++++++++++++++--------------- 1 file changed, 82 insertions(+), 64 deletions(-) diff --git a/core/chains/tx_store_test.go b/core/chains/tx_store_test.go index abdde942458..3cca15519c0 100644 --- a/core/chains/tx_store_test.go +++ b/core/chains/tx_store_test.go @@ -24,6 +24,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/pg/datatypes" ) @@ -37,86 +38,103 @@ type TestingTxStore[ FEE feetypes.Fee, ] interface { CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, gas.EvmFee], err error) + Close() } -type txStoreFunc func(t *testing.T) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address) +type txStoreFunc func(*testing.T, chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) -func evmTxStore(t *testing.T) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address) { +func evmTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) keyStore := cltest.NewKeyStore(t, db, cfg.Database()) _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + chainID := ethClient.ConfiguredChainID() + + return cltest.NewTxStore(t, db, cfg.Database()), fromAddress, chainID +} +func inmemoryTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTxStore(t, db, cfg.Database()) + keyStore := cltest.NewKeyStore(t, db, cfg.Database()) + _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + chainID := ethClient.ConfiguredChainID() - return cltest.NewTxStore(t, db, cfg.Database()), fromAddress + ims, err := txmgr.NewInMemoryStore[ + *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, + ](chainID, keyStore.Eth(), txStore) + require.NoError(t, err) + + return ims, fromAddress, chainID } -var txStoresFuncs = []txStoreFunc{ - evmTxStore, - /* - ims, err := txmgr.NewInMemoryStore[ - *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, - ](chainID, keyStore.Eth(), txStore) - */ +var txStoresFuncs = map[string]txStoreFunc{ + "evm_postgres_tx_store": evmTxStore, + "evm_in_memory_tx_store": inmemoryTxStore, } func TestTxStore_CreateTransaction(t *testing.T) { - for _, f := range txStoresFuncs { - txStore, fromAddress := f(t) - - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - subject := uuid.New() - strategy := commontxmmocks.NewTxStrategy(t) - strategy.On("Subject").Return(uuid.NullUUID{UUID: subject, Valid: true}) - strategy.On("PruneQueue", mock.Anything, mock.AnythingOfType("*txmgr.evmTxStore")).Return(int64(0), nil) - ctx := context.Background() - idempotencyKey := "11" - - tts := []struct { - scenario string - createTransactionInput createTransactionInput - createTransactionOutputCheck func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) - }{ - { - scenario: "success", - createTransactionInput: createTransactionInput{ - txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ - IdempotencyKey: &idempotencyKey, - FromAddress: fromAddress, - ToAddress: common.BytesToAddress([]byte("test")), - EncodedPayload: []byte{1, 2, 3}, - FeeLimit: uint32(1000), - Meta: nil, - Strategy: strategy, + cfg := configtest.NewGeneralConfig(t, nil) + + for n, f := range txStoresFuncs { + t.Run(n, func(t *testing.T) { + txStore, fromAddress, chainID := f(t, cfg) + defer txStore.Close() + + subject := uuid.New() + strategy := commontxmmocks.NewTxStrategy(t) + strategy.On("Subject").Return(uuid.NullUUID{UUID: subject, Valid: true}) + strategy.On("PruneQueue", mock.Anything, mock.AnythingOfType("*txmgr.evmTxStore")).Return(int64(0), nil) + ctx := context.Background() + idempotencyKey := "11" + + tts := []struct { + scenario string + createTransactionInput createTransactionInput + createTransactionOutputCheck func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) + }{ + { + scenario: "success", + createTransactionInput: createTransactionInput{ + txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ + IdempotencyKey: &idempotencyKey, + FromAddress: fromAddress, + ToAddress: common.BytesToAddress([]byte("test")), + EncodedPayload: []byte{1, 2, 3}, + FeeLimit: uint32(1000), + Meta: nil, + Strategy: strategy, + }, + chainID: chainID, + }, + createTransactionOutputCheck: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { + funcName := "CreateTransaction" + require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) + assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) + // Check CreatedAt is within 1 second of now + assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) + assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) + assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) + assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) + assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) + assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) + assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) + var expMeta *datatypes.JSON + assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) + assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) }, - chainID: ethClient.ConfiguredChainID(), - }, - createTransactionOutputCheck: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { - funcName := "CreateTransaction" - require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) - assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) - assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) - assert.Equal(t, ethClient.ConfiguredChainID(), tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) - assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) - assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) - assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) - assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) - var expMeta *datatypes.JSON - assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) - assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) }, - }, - } + } - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actTx, actErr := txStore.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) - tt.createTransactionOutputCheck(t, actTx, actErr) + for _, tt := range tts { + t.Run(tt.scenario, func(t *testing.T) { + actTx, actErr := txStore.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) + tt.createTransactionOutputCheck(t, actTx, actErr) - // TODO(jtw): Check that the transaction was persisted - }) - } + // TODO(jtw): Check that the transaction was persisted + }) + } + }) } } From a28aa808446dcfd24bc2a106b78a5c3e7e52c5ae Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 8 Nov 2023 18:56:21 -0500 Subject: [PATCH 10/74] start initialization function; add priority queue system; change storage mechanics for address state --- common/txmgr/address_state.go | 280 ++++++++++++++++++++++++++ common/txmgr/inmemory_store.go | 258 ++++++++++++------------ core/chains/evm/txmgr/evm_tx_store.go | 17 ++ core/chains/tx_store_test.go | 50 ++--- 4 files changed, 446 insertions(+), 159 deletions(-) create mode 100644 common/txmgr/address_state.go diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go new file mode 100644 index 00000000000..083854b707b --- /dev/null +++ b/common/txmgr/address_state.go @@ -0,0 +1,280 @@ +package txmgr + +import ( + "container/heap" + "fmt" + "sync" + + feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/common/types" + "gopkg.in/guregu/null.v4" +) + +// AddressState is the state of a given from address +type AddressState[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] struct { + fromAddress ADDR + + lock sync.RWMutex + unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + unconfirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] +} + +// NewAddressState returns a new AddressState instance +func NewAddressState[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +](fromAddress ADDR, maxUnstarted int) *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + fromAddress: fromAddress, + unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), + inprogress: nil, + unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + } + + return &as +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close() { + as.lock.Lock() + defer as.lock.Unlock() + + as.unstarted.Close() + as.unstarted = nil + as.inprogress = nil + clear(as.unconfirmed) +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unstartedCount() int { + as.lock.RLock() + defer as.lock.RUnlock() + + return as.unstarted.Len() +} +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unconfirmedCount() int { + as.lock.RLock() + defer as.lock.RUnlock() + + return len(as.unconfirmed) +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + as.lock.RLock() + defer as.lock.RUnlock() + + tx := as.unstarted.PeekNextTx() + if tx == nil { + return nil, fmt.Errorf("peek_next_unstarted_tx: %w (address: %s)", ErrTxnNotFound, as.fromAddress) + } + + return tx, nil +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekInProgressTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + as.lock.RLock() + defer as.lock.RUnlock() + + tx := as.inprogress + if tx == nil { + return nil, fmt.Errorf("peek_in_progress_tx: %w (address: %s)", ErrTxnNotFound, as.fromAddress) + } + + return tx, nil +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveTxToUnstarted(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + as.lock.Lock() + defer as.lock.Unlock() + + if as.unstarted.Len() >= as.unstarted.Cap() { + return fmt.Errorf("move_tx_to_unstarted: address %s unstarted queue capactiry has been reached", as.fromAddress) + } + + as.unstarted.AddTx(tx) + + return nil +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveUnstartedToInProgress(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + as.lock.Lock() + defer as.lock.Unlock() + + if as.inprogress != nil { + return fmt.Errorf("move_unstarted_to_in_progress: address %s already has a transaction in progress", as.fromAddress) + } + + if tx != nil { + // if tx is not nil then remove the tx from the unstarted queue + // TODO(jtw): what should be the unique idenitifier for each transaction? ID is being set by the postgres DB + tx = as.unstarted.RemoveTxByID(tx.ID) + } else { + // if tx is nil then pop the next unstarted transaction + tx = as.unstarted.RemoveNextTx() + } + if tx == nil { + return fmt.Errorf("move_unstarted_to_in_progress: no unstarted transaction to move to in_progress") + } + tx.State = TxInProgress + as.inprogress = tx + + return nil +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveInProgressToUnconfirmed( + txAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) error { + as.lock.Lock() + defer as.lock.Unlock() + + tx := as.inprogress + if tx == nil { + return fmt.Errorf("move_in_progress_to_unconfirmed: no transaction in progress") + } + tx.State = TxUnconfirmed + + var found bool + for i := 0; i < len(tx.TxAttempts); i++ { + if tx.TxAttempts[i].ID == txAttempt.ID { + tx.TxAttempts[i] = txAttempt + found = true + } + } + if !found { + // NOTE(jtw): this would mean that the TxAttempt did not exist for the Tx + // NOTE(jtw): should this log a warning? + // NOTE(jtw): can this happen? + tx.TxAttempts = append(tx.TxAttempts, txAttempt) + } + + as.unconfirmed[tx.ID] = tx + as.inprogress = nil + + return nil +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandon() { + as.lock.Lock() + defer as.lock.Unlock() + + for as.unstarted.Len() > 0 { + tx := as.unstarted.RemoveNextTx() + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + + if as.inprogress != nil { + as.inprogress.State = TxFatalError + as.inprogress.Sequence = nil + as.inprogress.Error = null.NewString("abandoned", true) + as.inprogress = nil + } + for _, tx := range as.unconfirmed { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + clear(as.unconfirmed) +} + +// TxPriorityQueue is a priority queue of transactions prioritized by creation time. The oldest transaction is at the front of the queue. +type TxPriorityQueue[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] struct { + sync.Mutex + txs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] +} + +// NewTxPriorityQueue returns a new TxPriorityQueue instance +func NewTxPriorityQueue[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +](maxUnstarted int) *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + pq := TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + txs: make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 0, maxUnstarted), + } + + return &pq +} + +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Cap() int { + return cap(pq.txs) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Len() int { + return len(pq.txs) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Less(i, j int) bool { + // We want Pop to give us the oldest, not newest, transaction based on creation time + return pq.txs[i].CreatedAt.Before(pq.txs[j].CreatedAt) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Swap(i, j int) { + pq.txs[i], pq.txs[j] = pq.txs[j], pq.txs[i] +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Push(tx any) { + pq.txs = append(pq.txs, tx.(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pop() any { + pq.Lock() + defer pq.Unlock() + + old := pq.txs + n := len(old) + tx := old[n-1] + old[n-1] = nil // avoid memory leak + pq.txs = old[0 : n-1] + return tx +} + +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + pq.Lock() + defer pq.Unlock() + + heap.Push(pq, tx) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + pq.Lock() + defer pq.Unlock() + + return heap.Pop(pq).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveTxByID(id int64) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + pq.Lock() + defer pq.Unlock() + + for i, tx := range pq.txs { + if tx.ID == id { + return heap.Remove(pq, i).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) + } + } + + return nil +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + if len(pq.txs) == 0 { + return nil + } + return pq.txs[0] +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { + pq.Lock() + defer pq.Unlock() + + clear(pq.txs) +} diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 504d5ea35a9..c5224392f2f 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -51,6 +51,26 @@ var ( // 4. Confirmer sets transactions that have failed to (unconfirmed) which will be retried by the resender // 5. Confirmer sets transactions that have been confirmed to (confirmed) and creates a new receipt which is persisted +type PersistentTxStore[ + ADDR types.Hashable, + CHAIN_ID types.ID, + TX_HASH types.Hashable, + BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] interface { + Close() + Abandon(ctx context.Context, id CHAIN_ID, addr ADDR) error + CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + FindLatestSequence(ctx context.Context, fromAddress ADDR, chainId CHAIN_ID) (SEQ, error) + UnstartedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + UpdateTxAttemptInProgressToBroadcast(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState txmgrtypes.TxAttemptState) error + SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + UpdateTxUnstartedToInProgress(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + UpdateTxFatalError(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error +} + type InMemoryStore[ CHAIN_ID types.ID, ADDR, TX_HASH, BLOCK_HASH types.Hashable, @@ -61,20 +81,14 @@ type InMemoryStore[ chainID CHAIN_ID keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] - txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] pendingLock sync.RWMutex // NOTE(jtw): we might need to watch out for txns that finish and are removed from the pending map pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - // unstarted is a map of addresses to a channel of unstarted transactions - unstarted map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - // inprogress is a map of addresses to inprogress transactions - inprogressLock sync.RWMutex - inprogress map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - // unconfirmed is a map of addresses to a map of transaction IDs to unconfirmed transactions - unconfirmedLock sync.RWMutex - unconfirmed map[ADDR]map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + addressStatesLock sync.RWMutex + addressStates map[ADDR]*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] } // NewInMemoryStore returns a new InMemoryStore @@ -87,7 +101,7 @@ func NewInMemoryStore[ ]( chainID CHAIN_ID, keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], - txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], ) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { ms := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ chainID: chainID, @@ -96,19 +110,35 @@ func NewInMemoryStore[ pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - unstarted: map[ADDR]chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - inprogress: map[ADDR]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - unconfirmed: map[ADDR]map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + addressStates: map[ADDR]*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{}, } + maxUnstarted := 50 addresses, err := keyStore.EnabledAddressesForChain(chainID) if err != nil { return nil, fmt.Errorf("new_in_memory_store: %w", err) } for _, fromAddr := range addresses { - // Channel Buffer is set to something high to prevent blocking and allow the pruning to happen - ms.unstarted[fromAddr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) - ms.unconfirmed[fromAddr] = map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + as := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](fromAddr, maxUnstarted) + offset := 0 + limit := 50 + for { + txs, count, err := txStore.UnstartedTransactions(offset, limit, fromAddr, chainID) + if err != nil { + return nil, fmt.Errorf("new_in_memory_store: %w", err) + } + for _, tx := range txs { + if err := as.moveTxToUnstarted(&tx); err != nil { + return nil, fmt.Errorf("new_in_memory_store: %w", err) + } + } + if count <= offset+limit { + break + } + offset += limit + } + + ms.addressStates[fromAddr] = as } return &ms, nil @@ -119,7 +149,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat if ms.chainID.String() != chainID.String() { return tx, fmt.Errorf("create_transaction: %w", ErrInvalidChainID) } - if _, ok := ms.unstarted[txRequest.FromAddress]; !ok { + if _, ok := ms.addressStates[txRequest.FromAddress]; !ok { return tx, fmt.Errorf("create_transaction: %w", ErrAddressNotFound) } @@ -160,11 +190,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check if ms.chainID.String() != chainID.String() { return fmt.Errorf("check_tx_queue_capacity: %w", ErrInvalidChainID) } - if _, ok := ms.unstarted[fromAddress]; !ok { + as, ok := ms.addressStates[fromAddress] + if !ok { return fmt.Errorf("check_tx_queue_capacity: %w", ErrAddressNotFound) } - count := uint64(len(ms.unstarted[fromAddress])) + count := uint64(as.unstartedCount()) if count >= maxQueuedTransactions { return fmt.Errorf("check_tx_queue_capacity: cannot create transaction; too many unstarted transactions in the queue (%v/%v). %s", count, maxQueuedTransactions, label.MaxQueuedTransactionsWarning) } @@ -185,10 +216,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL if ms.chainID.String() != chainID.String() { return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) } - if _, ok := ms.unstarted[fromAddress]; !ok { + if _, ok := ms.addressStates[fromAddress]; !ok { return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) } + // TODO(jtw): replace with inmemory store and use initialization at the start seq, err = ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) if err != nil { return seq, fmt.Errorf("find_latest_sequence: %w", err) @@ -204,12 +236,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count if ms.chainID.String() != chainID.String() { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } - u, ok := ms.unconfirmed[fromAddress] + as, ok := ms.addressStates[fromAddress] if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - return uint32(len(u)), nil + return uint32(as.unconfirmedCount()), nil } // CountUnstartedTransactions returns the number of unstarted transactions for a given address. @@ -219,11 +251,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count if ms.chainID.String() != chainID.String() { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } - if _, ok := ms.unstarted[fromAddress]; !ok { + as, ok := ms.addressStates[fromAddress] + if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - return uint32(len(ms.unstarted[fromAddress])), nil + return uint32(as.unstartedCount()), nil } // UpdateTxUnstartedToInProgress updates a transaction from unstarted to in_progress. @@ -241,6 +274,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat if attempt.State != txmgrtypes.TxAttemptInProgress { return fmt.Errorf("update_tx_unstarted_to_in_progress: attempt state must be in_progress") } + as, ok := ms.addressStates[tx.FromAddress] + if !ok { + return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", ErrAddressNotFound) + } // Persist to persistent storage if err := ms.txStore.UpdateTxUnstartedToInProgress(ctx, tx, attempt); err != nil { @@ -248,22 +285,28 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } tx.TxAttempts = append(tx.TxAttempts, *attempt) - // Update in memory store - ms.inprogressLock.Lock() - ms.inprogress[tx.FromAddress] = tx - ms.inprogressLock.Unlock() + // Update in address state in memory + if err := as.moveUnstartedToInProgress(tx); err != nil { + return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) + } return nil } // GetTxInProgress returns the in_progress transaction for a given address. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxInProgress(ctx context.Context, fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - ms.inprogressLock.RLock() - defer ms.inprogressLock.RUnlock() - tx, ok := ms.inprogress[fromAddress] + as, ok := ms.addressStates[fromAddress] if !ok { - return nil, nil + return nil, fmt.Errorf("get_tx_in_progress: %w", ErrAddressNotFound) + } + + tx, err := as.peekInProgressTx() + if tx == nil { + return nil, fmt.Errorf("get_tx_in_progress: %w", err) } + + // NOTE(jtw): should this exist in the in-memory store? or just the persistent store? + // NOTE(jtw): where should this live? if len(tx.TxAttempts) != 1 || tx.TxAttempts[0].State != txmgrtypes.TxAttemptInProgress { return nil, fmt.Errorf("get_tx_in_progress: expected in_progress transaction %v to have exactly one unsent attempt. "+ "Your database is in an inconsistent state and this node will not function correctly until the problem is resolved", tx.ID) @@ -280,7 +323,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat newAttemptState txmgrtypes.TxAttemptState, ) error { if tx.BroadcastAt == nil { - return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: unconfirmed transaction must have broadcast)at time") + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: unconfirmed transaction must have broadcast_at time") } if tx.InitialBroadcastAt == nil { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: unconfirmed transaction must have initial_broadcast_at time") @@ -300,30 +343,16 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } // Ensure that the tx state is updated to unconfirmed since this is a chain agnostic operation - tx.State = TxUnconfirmed attempt.State = newAttemptState - var found bool - for i := 0; i < len(tx.TxAttempts); i++ { - if tx.TxAttempts[i].ID == attempt.ID { - tx.TxAttempts[i] = attempt - found = true - } + + as, ok := ms.addressStates[tx.FromAddress] + if !ok { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", ErrAddressNotFound) } - if !found { - tx.TxAttempts = append(tx.TxAttempts, attempt) - // NOTE(jtw): should this log a warning? + if err := as.moveInProgressToUnconfirmed(attempt); err != nil { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } - // remove the transaction from the inprogress map - ms.inprogressLock.Lock() - ms.inprogress[tx.FromAddress] = nil - ms.inprogressLock.Unlock() - - // add the transaction to the unconfirmed map - ms.unconfirmedLock.Lock() - ms.unconfirmed[tx.FromAddress][tx.ID] = tx - ms.unconfirmedLock.Unlock() - return nil } @@ -333,23 +362,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN if ms.chainID.String() != chainID.String() { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } - if _, ok := ms.unstarted[fromAddress]; !ok { + as, ok := ms.addressStates[fromAddress] + if !ok { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) } + // ensure that the address is not already busy with a transaction in progress - ms.inprogressLock.RLock() - if ms.inprogress[fromAddress] != nil { - ms.inprogressLock.RUnlock() + if as.inprogress != nil { return fmt.Errorf("find_next_unstarted_transaction_from_address: address %s is already busy with a transaction in progress", fromAddress) } - ms.inprogressLock.RUnlock() - select { - case tx = <-ms.unstarted[fromAddress]: - return nil - default: - return ErrTxnNotFound + var err error + tx, err = as.peekNextUnstartedTx() + if tx == nil { + return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", err) } + + return nil } // SaveReplacementInProgressAttempt saves a replacement attempt for a transaction that is in_progress. @@ -371,12 +400,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR } // Update in memory store - ms.inprogressLock.Lock() - tx, ok := ms.inprogress[oldAttempt.Tx.FromAddress] - if tx == nil || !ok { - ms.inprogressLock.Unlock() + as, ok := ms.addressStates[oldAttempt.Tx.FromAddress] + if !ok { return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) } + tx, err := as.peekInProgressTx() + if tx == nil { + return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) + } var found bool for i := 0; i < len(tx.TxAttempts); i++ { if tx.TxAttempts[i].ID == oldAttempt.ID { @@ -388,9 +419,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR tx.TxAttempts = append(tx.TxAttempts, *replacementAttempt) // NOTE(jtw): should this log a warning? } - ms.inprogressLock.Unlock() - return fmt.Errorf("save_replacement_in_progress_attempt: not implemented") + return nil } // UpdateTxFatalError updates a transaction to fatal_error. @@ -419,23 +449,17 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close // Close the event recorder ms.txStore.Close() - // Close all channels - for _, ch := range ms.unstarted { - close(ch) - } - - // Clear all pending requests ms.pendingLock.Lock() clear(ms.pendingIdempotencyKeys) ms.pendingLock.Unlock() - // Clear all unstarted transactions - ms.inprogressLock.Lock() - clear(ms.inprogress) - ms.inprogressLock.Unlock() - // Clear all unconfirmed transactions - ms.unconfirmedLock.Lock() - clear(ms.unconfirmed) - ms.unconfirmedLock.Unlock() + + // Clear all address states + ms.addressStatesLock.Lock() + for _, as := range ms.addressStates { + as.close() + } + clear(ms.addressStates) + ms.addressStatesLock.Unlock() } // Abandon removes all transactions for a given address @@ -450,46 +474,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband } // check that the address exists in the unstarted transactions - if _, ok := ms.unstarted[addr]; !ok { - return fmt.Errorf("abandon: %w", ErrAddressNotFound) - } - // Mark all unstarted transactions as abandoned - close(ms.unstarted[addr]) - for tx := range ms.unstarted[addr] { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) - } - // reset the unstarted channel - ms.unstarted[addr] = make(chan *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 100) - - ms.inprogressLock.Lock() - if _, ok := ms.inprogress[addr]; !ok { - ms.inprogressLock.Unlock() - return fmt.Errorf("abandon: %w", ErrAddressNotFound) - } - // Mark all inprogress transactions as abandoned - if tx, ok := ms.inprogress[addr]; ok { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) - } - ms.inprogress[addr] = nil - ms.inprogressLock.Unlock() - - ms.unconfirmedLock.Lock() - if _, ok := ms.unconfirmed[addr]; !ok { - ms.unconfirmedLock.Unlock() + as, ok := ms.addressStates[addr] + if !ok { return fmt.Errorf("abandon: %w", ErrAddressNotFound) } - // Mark all unconfirmed transactions as abandoned - for _, tx := range ms.unconfirmed[addr] { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) - } - ms.unconfirmed[addr] = nil - ms.unconfirmedLock.Unlock() + as.abandon() ms.pendingLock.Lock() // Mark all pending transactions as abandoned @@ -506,22 +495,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendTxToUnstartedQueue(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + as, ok := ms.addressStates[tx.FromAddress] + if !ok { + return fmt.Errorf("send_tx_to_unstarted_queue: %w", ErrAddressNotFound) + } + // TODO(jtw); HANDLE PRUNING STEP - select { // Add the request to the Unstarted channel to be processed by the Broadcaster - case ms.unstarted[tx.FromAddress] <- &tx: - // Persist to persistent storage - - ms.pendingLock.Lock() - if tx.IdempotencyKey != nil { - ms.pendingIdempotencyKeys[*tx.IdempotencyKey] = &tx - } - ms.pendingLock.Unlock() + if err := as.moveTxToUnstarted(&tx); err != nil { + return fmt.Errorf("send_tx_to_unstarted_queue: %w", err) + } - return nil - default: - // Return an error if the Manager Queue Capacity has been reached - return fmt.Errorf("transaction manager queue capacity has been reached") + ms.pendingLock.Lock() + if tx.IdempotencyKey != nil { + ms.pendingIdempotencyKeys[*tx.IdempotencyKey] = &tx } + ms.pendingLock.Unlock() + + return nil } diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 971103bdfd5..00d19e79e15 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -58,6 +58,7 @@ type TxStoreWebApi interface { TransactionsWithAttempts(offset, limit int) ([]Tx, int, error) FindTxAttempt(hash common.Hash) (*TxAttempt, error) FindTxWithAttempts(etxID int64) (etx Tx, err error) + UnstartedTransactions(limit, offset int, fromAddress common.Address, chainID *big.Int) ([]Tx, int, error) } type TestEvmTxStore interface { @@ -450,6 +451,22 @@ func (o *evmTxStore) TransactionsWithAttempts(offset, limit int) (txs []Tx, coun return } +// UnstartedTransactions returns all eth transactions that have no attempts. +func (o *evmTxStore) UnstartedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { + sql := `SELECT count(*) FROM evm.txes WHERE state = 'unstarted' AND from_address = $1 AND evm_chain_id = $2` + if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { + return + } + + sql = `SELECT * FROM evm.txes WHERE state = 'unstarted' AND from_address = $1 AND evm_chain_id = $2 ORDER BY value ASC, created_at ASC, id ASC LIMIT $3 OFFSET $4` + var dbTxs []DbEthTx + if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { + return + } + txs = dbEthTxsToEvmEthTxs(dbTxs) + return +} + // TxAttempts returns the last tx attempts sorted by created_at descending. func (o *evmTxStore) TxAttempts(offset, limit int) (txs []TxAttempt, count int, err error) { sql := `SELECT count(*) FROM evm.tx_attempts` diff --git a/core/chains/tx_store_test.go b/core/chains/tx_store_test.go index 3cca15519c0..bd2fdc71ef6 100644 --- a/core/chains/tx_store_test.go +++ b/core/chains/tx_store_test.go @@ -43,31 +43,6 @@ type TestingTxStore[ type txStoreFunc func(*testing.T, chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) -func evmTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { - db := pgtest.NewSqlxDB(t) - keyStore := cltest.NewKeyStore(t, db, cfg.Database()) - _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - chainID := ethClient.ConfiguredChainID() - - return cltest.NewTxStore(t, db, cfg.Database()), fromAddress, chainID -} -func inmemoryTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTxStore(t, db, cfg.Database()) - keyStore := cltest.NewKeyStore(t, db, cfg.Database()) - _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - chainID := ethClient.ConfiguredChainID() - - ims, err := txmgr.NewInMemoryStore[ - *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, - ](chainID, keyStore.Eth(), txStore) - require.NoError(t, err) - - return ims, fromAddress, chainID -} - var txStoresFuncs = map[string]txStoreFunc{ "evm_postgres_tx_store": evmTxStore, "evm_in_memory_tx_store": inmemoryTxStore, @@ -258,3 +233,28 @@ type findLatestSequenceInput struct { fromAddress common.Address chainID *big.Int } + +func evmTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { + db := pgtest.NewSqlxDB(t) + keyStore := cltest.NewKeyStore(t, db, cfg.Database()) + _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + chainID := ethClient.ConfiguredChainID() + + return cltest.NewTxStore(t, db, cfg.Database()), fromAddress, chainID +} +func inmemoryTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { + db := pgtest.NewSqlxDB(t) + txStore := cltest.NewTxStore(t, db, cfg.Database()) + keyStore := cltest.NewKeyStore(t, db, cfg.Database()) + _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + chainID := ethClient.ConfiguredChainID() + + ims, err := txmgr.NewInMemoryStore[ + *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, + ](chainID, keyStore.Eth(), txStore) + require.NoError(t, err) + + return ims, fromAddress, chainID +} From e2f3a8787c20df94104f064075803d052bdcc255 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 13 Nov 2023 21:38:23 -0500 Subject: [PATCH 11/74] add initialization to address state --- common/txmgr/address_state.go | 62 +++++++++++++++++++++++++-- common/txmgr/inmemory_store.go | 26 +++-------- core/chains/evm/txmgr/evm_tx_store.go | 17 ++++++++ 3 files changed, 82 insertions(+), 23 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 083854b707b..dcd233438dd 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -2,6 +2,7 @@ package txmgr import ( "container/heap" + "context" "fmt" "sync" @@ -19,11 +20,15 @@ type AddressState[ SEQ types.Sequence, FEE feetypes.Fee, ] struct { + chainID CHAIN_ID fromAddress ADDR - lock sync.RWMutex - unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + lock sync.RWMutex + unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // TODO(jtw): using the TX ID as the key for the map might not make sense since the ID is set by the + // postgres DB which creates a dependency on the postgres DB. We should consider creating a UUID or ULID + // TX ID -> TX unconfirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } @@ -34,8 +39,9 @@ func NewAddressState[ R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee, -](fromAddress ADDR, maxUnstarted int) *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +](chainID CHAIN_ID, fromAddress ADDR, maxUnstarted int) *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + chainID: chainID, fromAddress: fromAddress, unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), inprogress: nil, @@ -45,6 +51,54 @@ func NewAddressState[ return &as } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initialize(txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) error { + // Load all unstarted transactions from persistent storage + offset := 0 + limit := 50 + for { + txs, count, err := txStore.UnstartedTransactions(offset, limit, as.fromAddress, as.chainID) + if err != nil { + return fmt.Errorf("address_state: initialization: %w", err) + } + for _, tx := range txs { + if err := as.moveTxToUnstarted(&tx); err != nil { + return fmt.Errorf("address_state: initialization: %w", err) + } + } + if count <= offset+limit { + break + } + offset += limit + } + + // Load all in progress transactions from persistent storage + ctx := context.Background() + tx, err := txStore.GetTxInProgress(ctx, as.fromAddress) + if err != nil { + return fmt.Errorf("address_state: initialization: %w", err) + } + as.inprogress = tx + + // Load all unconfirmed transactions from persistent storage + offset = 0 + limit = 50 + for { + txs, count, err := txStore.UnconfirmedTransactions(offset, limit, as.fromAddress, as.chainID) + if err != nil { + return fmt.Errorf("address_state: initialization: %w", err) + } + for _, tx := range txs { + as.unconfirmed[tx.ID] = &tx + } + if count <= offset+limit { + break + } + offset += limit + } + + return nil +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close() { as.lock.Lock() defer as.lock.Unlock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index c5224392f2f..518d7d5e9c5 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -35,6 +35,7 @@ var ( // 4. Transaction Manager prunes the Unstarted Queue based on the transaction prune strategy // NOTE(jtw): Only one transaction per address can be in_progress at a time +// NOTE(jtw): Only one transaction attempt per transaction can be in_progress at a time // NOTE(jtw): Only one broadcasted attempt exists per transaction the rest are errored or abandoned // 1. Broadcaster assigns a sequence number to the transaction // 2. Broadcaster creates and persists a new transaction attempt (in_progress) from the transaction (in_progress) @@ -65,10 +66,12 @@ type PersistentTxStore[ CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainId CHAIN_ID) (SEQ, error) UnstartedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + UnconfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) UpdateTxAttemptInProgressToBroadcast(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState txmgrtypes.TxAttemptState) error SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error UpdateTxUnstartedToInProgress(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error UpdateTxFatalError(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error + GetTxInProgress(ctx context.Context, fromAddress ADDR) (etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) } type InMemoryStore[ @@ -83,8 +86,7 @@ type InMemoryStore[ keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - pendingLock sync.RWMutex - // NOTE(jtw): we might need to watch out for txns that finish and are removed from the pending map + pendingLock sync.RWMutex pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] addressStatesLock sync.RWMutex @@ -119,23 +121,9 @@ func NewInMemoryStore[ return nil, fmt.Errorf("new_in_memory_store: %w", err) } for _, fromAddr := range addresses { - as := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](fromAddr, maxUnstarted) - offset := 0 - limit := 50 - for { - txs, count, err := txStore.UnstartedTransactions(offset, limit, fromAddr, chainID) - if err != nil { - return nil, fmt.Errorf("new_in_memory_store: %w", err) - } - for _, tx := range txs { - if err := as.moveTxToUnstarted(&tx); err != nil { - return nil, fmt.Errorf("new_in_memory_store: %w", err) - } - } - if count <= offset+limit { - break - } - offset += limit + as := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](chainID, fromAddr, maxUnstarted) + if err := as.Initialize(txStore); err != nil { + return nil, fmt.Errorf("new_in_memory_store: %w", err) } ms.addressStates[fromAddr] = as diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 00d19e79e15..248db7ccc21 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -467,6 +467,23 @@ func (o *evmTxStore) UnstartedTransactions(offset, limit int, fromAddress common return } +// UnconfirmedTransactions returns all eth transactions that have at least one attempt and in the unconfirmed state. +func (o *evmTxStore) UnconfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { + sql := `SELECT count(*) FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'unconfirmed' AND from_address = $1 AND evm_chain_id = $2` + if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { + return + } + + sql = `SELECT * FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'unconfirmed' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` + var dbTxs []DbEthTx + if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { + return + } + txs = dbEthTxsToEvmEthTxs(dbTxs) + err = o.preloadTxAttempts(txs) + return +} + // TxAttempts returns the last tx attempts sorted by created_at descending. func (o *evmTxStore) TxAttempts(offset, limit int) (txs []TxAttempt, count int, err error) { sql := `SELECT count(*) FROM evm.tx_attempts` From 91bea750c2d40bc3231a70f8ef8a581698aa2921 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 14 Nov 2023 00:43:08 -0500 Subject: [PATCH 12/74] change loops to not use range --- common/txmgr/address_state.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index dcd233438dd..3ceaf872a08 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -60,7 +60,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia if err != nil { return fmt.Errorf("address_state: initialization: %w", err) } - for _, tx := range txs { + for i := 0; i < len(txs); i++ { + tx := txs[i] if err := as.moveTxToUnstarted(&tx); err != nil { return fmt.Errorf("address_state: initialization: %w", err) } @@ -87,7 +88,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia if err != nil { return fmt.Errorf("address_state: initialization: %w", err) } - for _, tx := range txs { + for i := 0; i < len(txs); i++ { + tx := txs[i] as.unconfirmed[tx.ID] = &tx } if count <= offset+limit { From 5314284d1c8bf07c46b4c957f680996e8bdcfcd8 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 15 Nov 2023 15:38:10 -0500 Subject: [PATCH 13/74] clean up idempotency key location --- common/txmgr/address_state.go | 51 ++++++++++++++++++++++++++-------- common/txmgr/inmemory_store.go | 45 +++++++----------------------- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 3ceaf872a08..1d97385d8b4 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -23,9 +23,10 @@ type AddressState[ chainID CHAIN_ID fromAddress ADDR - lock sync.RWMutex - unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + lock sync.RWMutex + idempotencyKeyToTx map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] // TODO(jtw): using the TX ID as the key for the map might not make sense since the ID is set by the // postgres DB which creates a dependency on the postgres DB. We should consider creating a UUID or ULID // TX ID -> TX @@ -41,17 +42,21 @@ func NewAddressState[ FEE feetypes.Fee, ](chainID CHAIN_ID, fromAddress ADDR, maxUnstarted int) *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - chainID: chainID, - fromAddress: fromAddress, - unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), - inprogress: nil, - unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + chainID: chainID, + fromAddress: fromAddress, + idempotencyKeyToTx: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), + inprogress: nil, + unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } return &as } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initialize(txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) error { + as.lock.Lock() + defer as.lock.Unlock() + // Load all unstarted transactions from persistent storage offset := 0 limit := 50 @@ -62,8 +67,9 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia } for i := 0; i < len(txs); i++ { tx := txs[i] - if err := as.moveTxToUnstarted(&tx); err != nil { - return fmt.Errorf("address_state: initialization: %w", err) + as.unstarted.AddTx(&tx) + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx } } if count <= offset+limit { @@ -79,6 +85,9 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia return fmt.Errorf("address_state: initialization: %w", err) } as.inprogress = tx + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx + } // Load all unconfirmed transactions from persistent storage offset = 0 @@ -91,6 +100,9 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia for i := 0; i < len(txs); i++ { tx := txs[i] as.unconfirmed[tx.ID] = &tx + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx + } } if count <= offset+limit { break @@ -109,6 +121,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close( as.unstarted = nil as.inprogress = nil clear(as.unconfirmed) + clear(as.idempotencyKeyToTx) } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unstartedCount() int { @@ -124,6 +137,13 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unconf return len(as.unconfirmed) } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findTxWithIdempotencyKey(key string) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + as.lock.RLock() + defer as.lock.RUnlock() + + return as.idempotencyKeyToTx[key] +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.lock.RLock() defer as.lock.RUnlock() @@ -148,7 +168,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekIn return tx, nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveTxToUnstarted(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) addTxToUnstarted(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { as.lock.Lock() defer as.lock.Unlock() @@ -157,6 +177,9 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveTx } as.unstarted.AddTx(tx) + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx + } return nil } @@ -240,6 +263,12 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abando tx.Sequence = nil tx.Error = null.NewString("abandoned", true) } + for _, tx := range as.idempotencyKeyToTx { + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) + } + clear(as.unconfirmed) } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 518d7d5e9c5..d5b77b4b91a 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -9,7 +9,6 @@ import ( txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/label" - "gopkg.in/guregu/null.v4" ) var ( @@ -86,9 +85,6 @@ type InMemoryStore[ keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - pendingLock sync.RWMutex - pendingIdempotencyKeys map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - addressStatesLock sync.RWMutex addressStates map[ADDR]*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] } @@ -110,8 +106,6 @@ func NewInMemoryStore[ keyStore: keyStore, txStore: txStore, - pendingIdempotencyKeys: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - addressStates: map[ADDR]*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{}, } @@ -159,15 +153,17 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_tx_with_idempotency_key: %w", ErrInvalidChainID) } - ms.pendingLock.Lock() - defer ms.pendingLock.Unlock() - - tx, ok := ms.pendingIdempotencyKeys[idempotencyKey] - if !ok { - return nil, fmt.Errorf("find_tx_with_idempotency_key: %w", ErrTxnNotFound) + // Check if the transaction is in the pending queue of all address states + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() + for _, as := range ms.addressStates { + if tx := as.findTxWithIdempotencyKey(idempotencyKey); tx != nil { + return tx, nil + } } - return tx, nil + return nil, fmt.Errorf("find_tx_with_idempotency_key: %w", ErrTxnNotFound) + } // CheckTxQueueCapacity checks if the queue capacity has been reached for a given address @@ -437,10 +433,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close // Close the event recorder ms.txStore.Close() - ms.pendingLock.Lock() - clear(ms.pendingIdempotencyKeys) - ms.pendingLock.Unlock() - // Clear all address states ms.addressStatesLock.Lock() for _, as := range ms.addressStates { @@ -468,17 +460,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband } as.abandon() - ms.pendingLock.Lock() - // Mark all pending transactions as abandoned - for _, tx := range ms.pendingIdempotencyKeys { - if tx.FromAddress == addr { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) - } - } - ms.pendingLock.Unlock() - return nil } @@ -491,15 +472,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendT // TODO(jtw); HANDLE PRUNING STEP // Add the request to the Unstarted channel to be processed by the Broadcaster - if err := as.moveTxToUnstarted(&tx); err != nil { + if err := as.addTxToUnstarted(&tx); err != nil { return fmt.Errorf("send_tx_to_unstarted_queue: %w", err) } - ms.pendingLock.Lock() - if tx.IdempotencyKey != nil { - ms.pendingIdempotencyKeys[*tx.IdempotencyKey] = &tx - } - ms.pendingLock.Unlock() - return nil } From 5a908be976e0f635d7ff0096d4507560e5abf092 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 15 Nov 2023 15:54:45 -0500 Subject: [PATCH 14/74] add find latest sequence --- common/txmgr/address_state.go | 22 ++++++++++++++++++++++ common/txmgr/inmemory_store.go | 12 +++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 1d97385d8b4..00d910db8a9 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -144,6 +144,28 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findTx return as.idempotencyKeyToTx[key] } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLatestSequence() SEQ { + as.lock.RLock() + defer as.lock.RUnlock() + + var maxSeq SEQ + if as.inprogress != nil && as.inprogress.Sequence != nil { + if (*as.inprogress.Sequence).Int64() > maxSeq.Int64() { + maxSeq = *as.inprogress.Sequence + } + } + for _, tx := range as.unconfirmed { + if tx.Sequence == nil { + continue + } + if (*tx.Sequence).Int64() > maxSeq.Int64() { + maxSeq = *tx.Sequence + } + } + + return maxSeq +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.lock.RLock() defer as.lock.RUnlock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index d5b77b4b91a..2c69d10ae59 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -20,6 +20,8 @@ var ( ErrExistingIdempotencyKey = fmt.Errorf("transaction with idempotency key already exists") // ErrAddressNotFound is returned when an address is not found ErrAddressNotFound = fmt.Errorf("address not found") + // ErrSequenceNotFound is returned when a sequence is not found + ErrSequenceNotFound = fmt.Errorf("sequence not found") ) // Store and update all transaction state as files @@ -200,14 +202,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL if ms.chainID.String() != chainID.String() { return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) } - if _, ok := ms.addressStates[fromAddress]; !ok { + as, ok := ms.addressStates[fromAddress] + if !ok { return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) } - // TODO(jtw): replace with inmemory store and use initialization at the start - seq, err = ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) - if err != nil { - return seq, fmt.Errorf("find_latest_sequence: %w", err) + seq = as.findLatestSequence() + if seq.Int64() == 0 { + return seq, fmt.Errorf("find_latest_sequence: %w", ErrSequenceNotFound) } return seq, nil From 987228313935644cef53b34b6693c4fac439f417 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 15 Nov 2023 18:40:43 -0500 Subject: [PATCH 15/74] remove FindLatestSequence from persistent interface --- common/txmgr/inmemory_store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 2c69d10ae59..bb1773e3a83 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -65,7 +65,6 @@ type PersistentTxStore[ Close() Abandon(ctx context.Context, id CHAIN_ID, addr ADDR) error CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) - FindLatestSequence(ctx context.Context, fromAddress ADDR, chainId CHAIN_ID) (SEQ, error) UnstartedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) UnconfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) UpdateTxAttemptInProgressToBroadcast(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState txmgrtypes.TxAttemptState) error From df5e9468e19deeceb9adfe825d563e938ab34dda Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 15 Nov 2023 18:56:49 -0500 Subject: [PATCH 16/74] add UnconfirmedTransactions to TxSoreWebApi --- core/chains/evm/txmgr/evm_tx_store.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 248db7ccc21..f7dd768741a 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -59,6 +59,7 @@ type TxStoreWebApi interface { FindTxAttempt(hash common.Hash) (*TxAttempt, error) FindTxWithAttempts(etxID int64) (etx Tx, err error) UnstartedTransactions(limit, offset int, fromAddress common.Address, chainID *big.Int) ([]Tx, int, error) + UnconfirmedTransactions(limit, offset int, fromAddress common.Address, chainID *big.Int) ([]Tx, int, error) } type TestEvmTxStore interface { From 5d788681ca8f7bf8eadd138d0899086e28b57cf6 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 11 Dec 2023 14:56:57 -0500 Subject: [PATCH 17/74] address comments --- common/txmgr/address_state.go | 190 +++++++++--------------------- common/txmgr/inmemory_store.go | 33 +----- common/txmgr/tx_priority_queue.go | 104 ++++++++++++++++ 3 files changed, 161 insertions(+), 166 deletions(-) create mode 100644 common/txmgr/tx_priority_queue.go diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 00d910db8a9..35122b4297a 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -1,7 +1,6 @@ package txmgr import ( - "container/heap" "context" "fmt" "sync" @@ -22,14 +21,13 @@ type AddressState[ ] struct { chainID CHAIN_ID fromAddress ADDR + txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - lock sync.RWMutex + sync.RWMutex idempotencyKeyToTx map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - // TODO(jtw): using the TX ID as the key for the map might not make sense since the ID is set by the - // postgres DB which creates a dependency on the postgres DB. We should consider creating a UUID or ULID - // TX ID -> TX + // NOTE: currently the unconfirmed map's key is the transaction ID that is assigned via the postgres DB unconfirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } @@ -40,22 +38,25 @@ func NewAddressState[ R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee, -](chainID CHAIN_ID, fromAddress ADDR, maxUnstarted int) *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { +]( + chainID CHAIN_ID, + fromAddress ADDR, + maxUnstarted int, + txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], +) (*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - chainID: chainID, - fromAddress: fromAddress, + chainID: chainID, + fromAddress: fromAddress, + txStore: txStore, + idempotencyKeyToTx: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), inprogress: nil, unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } - return &as -} - -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initialize(txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) error { - as.lock.Lock() - defer as.lock.Unlock() + as.Lock() + defer as.Unlock() // Load all unstarted transactions from persistent storage offset := 0 @@ -63,7 +64,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia for { txs, count, err := txStore.UnstartedTransactions(offset, limit, as.fromAddress, as.chainID) if err != nil { - return fmt.Errorf("address_state: initialization: %w", err) + return nil, fmt.Errorf("address_state: initialization: %w", err) } for i := 0; i < len(txs); i++ { tx := txs[i] @@ -82,7 +83,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia ctx := context.Background() tx, err := txStore.GetTxInProgress(ctx, as.fromAddress) if err != nil { - return fmt.Errorf("address_state: initialization: %w", err) + return nil, fmt.Errorf("address_state: initialization: %w", err) } as.inprogress = tx if tx.IdempotencyKey != nil { @@ -95,7 +96,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia for { txs, count, err := txStore.UnconfirmedTransactions(offset, limit, as.fromAddress, as.chainID) if err != nil { - return fmt.Errorf("address_state: initialization: %w", err) + return nil, fmt.Errorf("address_state: initialization: %w", err) } for i := 0; i < len(txs); i++ { tx := txs[i] @@ -110,12 +111,13 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Initia offset += limit } - return nil + return &as, nil + } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close() { - as.lock.Lock() - defer as.lock.Unlock() + as.Lock() + defer as.Unlock() as.unstarted.Close() as.unstarted = nil @@ -125,28 +127,28 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close( } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unstartedCount() int { - as.lock.RLock() - defer as.lock.RUnlock() + as.RLock() + defer as.RUnlock() return as.unstarted.Len() } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unconfirmedCount() int { - as.lock.RLock() - defer as.lock.RUnlock() + as.RLock() + defer as.RUnlock() return len(as.unconfirmed) } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findTxWithIdempotencyKey(key string) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - as.lock.RLock() - defer as.lock.RUnlock() + as.RLock() + defer as.RUnlock() return as.idempotencyKeyToTx[key] } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLatestSequence() SEQ { - as.lock.RLock() - defer as.lock.RUnlock() + as.RLock() + defer as.RUnlock() var maxSeq SEQ if as.inprogress != nil && as.inprogress.Sequence != nil { @@ -167,8 +169,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLa } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - as.lock.RLock() - defer as.lock.RUnlock() + as.RLock() + defer as.RUnlock() tx := as.unstarted.PeekNextTx() if tx == nil { @@ -179,8 +181,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNe } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekInProgressTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - as.lock.RLock() - defer as.lock.RUnlock() + as.RLock() + defer as.RUnlock() tx := as.inprogress if tx == nil { @@ -191,8 +193,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekIn } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) addTxToUnstarted(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - as.lock.Lock() - defer as.lock.Unlock() + as.Lock() + defer as.Unlock() if as.unstarted.Len() >= as.unstarted.Cap() { return fmt.Errorf("move_tx_to_unstarted: address %s unstarted queue capactiry has been reached", as.fromAddress) @@ -207,8 +209,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) addTxT } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveUnstartedToInProgress(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - as.lock.Lock() - defer as.lock.Unlock() + as.Lock() + defer as.Unlock() if as.inprogress != nil { return fmt.Errorf("move_unstarted_to_in_progress: address %s already has a transaction in progress", as.fromAddress) @@ -234,8 +236,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveUn func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveInProgressToUnconfirmed( txAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) error { - as.lock.Lock() - defer as.lock.Unlock() + as.Lock() + defer as.Unlock() tx := as.inprogress if tx == nil { @@ -248,6 +250,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveIn if tx.TxAttempts[i].ID == txAttempt.ID { tx.TxAttempts[i] = txAttempt found = true + break } } if !found { @@ -264,124 +267,41 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveIn } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandon() { - as.lock.Lock() - defer as.lock.Unlock() + as.Lock() + defer as.Unlock() for as.unstarted.Len() > 0 { tx := as.unstarted.RemoveNextTx() - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) + abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) } if as.inprogress != nil { - as.inprogress.State = TxFatalError - as.inprogress.Sequence = nil - as.inprogress.Error = null.NewString("abandoned", true) + tx := as.inprogress + abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) as.inprogress = nil } for _, tx := range as.unconfirmed { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) + abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) } for _, tx := range as.idempotencyKeyToTx { - tx.State = TxFatalError - tx.Sequence = nil - tx.Error = null.NewString("abandoned", true) + abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) } clear(as.unconfirmed) } -// TxPriorityQueue is a priority queue of transactions prioritized by creation time. The oldest transaction is at the front of the queue. -type TxPriorityQueue[ +func abandon[ CHAIN_ID types.ID, ADDR, TX_HASH, BLOCK_HASH types.Hashable, R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], SEQ types.Sequence, FEE feetypes.Fee, -] struct { - sync.Mutex - txs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] -} - -// NewTxPriorityQueue returns a new TxPriorityQueue instance -func NewTxPriorityQueue[ - CHAIN_ID types.ID, - ADDR, TX_HASH, BLOCK_HASH types.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], - SEQ types.Sequence, - FEE feetypes.Fee, -](maxUnstarted int) *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { - pq := TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ - txs: make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 0, maxUnstarted), - } - - return &pq -} - -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Cap() int { - return cap(pq.txs) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Len() int { - return len(pq.txs) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Less(i, j int) bool { - // We want Pop to give us the oldest, not newest, transaction based on creation time - return pq.txs[i].CreatedAt.Before(pq.txs[j].CreatedAt) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Swap(i, j int) { - pq.txs[i], pq.txs[j] = pq.txs[j], pq.txs[i] -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Push(tx any) { - pq.txs = append(pq.txs, tx.(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pop() any { - pq.Lock() - defer pq.Unlock() - - old := pq.txs - n := len(old) - tx := old[n-1] - old[n-1] = nil // avoid memory leak - pq.txs = old[0 : n-1] - return tx -} - -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - pq.Lock() - defer pq.Unlock() - - heap.Push(pq, tx) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - pq.Lock() - defer pq.Unlock() - - return heap.Pop(pq).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveTxByID(id int64) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - pq.Lock() - defer pq.Unlock() - - for i, tx := range pq.txs { - if tx.ID == id { - return heap.Remove(pq, i).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) - } - } - - return nil -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - if len(pq.txs) == 0 { - return nil +](tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx == nil { + return } - return pq.txs[0] -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { - pq.Lock() - defer pq.Unlock() - clear(pq.txs) + tx.State = TxFatalError + tx.Sequence = nil + tx.Error = null.NewString("abandoned", true) } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index bb1773e3a83..a8281d1249d 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -24,35 +24,6 @@ var ( ErrSequenceNotFound = fmt.Errorf("sequence not found") ) -// Store and update all transaction state as files -// Read from the files to restore state at startup -// Delete files when transactions are completed or reaped - -// Life of a Transaction -// 1. Transaction Request is created -// 2. Transaction Request is submitted to the Transaction Manager -// 3. Transaction Manager creates and persists a new transaction (unstarted) from the transaction request (not persisted) -// 4. Transaction Manager sends the transaction (unstarted) to the Broadcaster Unstarted Queue -// 4. Transaction Manager prunes the Unstarted Queue based on the transaction prune strategy - -// NOTE(jtw): Only one transaction per address can be in_progress at a time -// NOTE(jtw): Only one transaction attempt per transaction can be in_progress at a time -// NOTE(jtw): Only one broadcasted attempt exists per transaction the rest are errored or abandoned -// 1. Broadcaster assigns a sequence number to the transaction -// 2. Broadcaster creates and persists a new transaction attempt (in_progress) from the transaction (in_progress) -// 3. Broadcaster asks the Checker to check if the transaction should not be sent -// 4. Broadcaster asks the Attempt builder to figure out gas fee for the transaction -// 5. Broadcaster attempts to send the Transaction to TransactionClient to be published on-chain -// 6. Broadcaster updates the transaction attempt (broadcast) and transaction (unconfirmed) -// 7. Broadcaster increments global sequence number for address for next transaction attempt - -// NOTE(jtw): Only one receipt should exist per confirmed transaction -// 1. Confirmer listens and reads new Head events from the Chain -// 2. Confirmer sets the last known block number for the transaction attempts that have been broadcast -// 3. Confirmer checks for missing receipts for transactions that have been broadcast -// 4. Confirmer sets transactions that have failed to (unconfirmed) which will be retried by the resender -// 5. Confirmer sets transactions that have been confirmed to (confirmed) and creates a new receipt which is persisted - type PersistentTxStore[ ADDR types.Hashable, CHAIN_ID types.ID, @@ -116,8 +87,8 @@ func NewInMemoryStore[ return nil, fmt.Errorf("new_in_memory_store: %w", err) } for _, fromAddr := range addresses { - as := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](chainID, fromAddr, maxUnstarted) - if err := as.Initialize(txStore); err != nil { + as, err := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](chainID, fromAddr, maxUnstarted, txStore) + if err != nil { return nil, fmt.Errorf("new_in_memory_store: %w", err) } diff --git a/common/txmgr/tx_priority_queue.go b/common/txmgr/tx_priority_queue.go new file mode 100644 index 00000000000..71db08a2dab --- /dev/null +++ b/common/txmgr/tx_priority_queue.go @@ -0,0 +1,104 @@ +package txmgr + +import ( + "container/heap" + "sync" + + feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" + txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" + "github.com/smartcontractkit/chainlink/v2/common/types" +) + +// TxPriorityQueue is a priority queue of transactions prioritized by creation time. The oldest transaction is at the front of the queue. +type TxPriorityQueue[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] struct { + sync.RWMutex + txs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + idToIndex map[int64]int +} + +// NewTxPriorityQueue returns a new TxPriorityQueue instance +func NewTxPriorityQueue[ + CHAIN_ID types.ID, + ADDR, TX_HASH, BLOCK_HASH types.Hashable, + R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +](maxUnstarted int) *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] { + pq := TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + txs: make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], 0, maxUnstarted), + idToIndex: make(map[int64]int), + } + + return &pq +} + +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Cap() int { + return cap(pq.txs) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Len() int { + return len(pq.txs) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Less(i, j int) bool { + // We want Pop to give us the oldest, not newest, transaction based on creation time + return pq.txs[i].CreatedAt.Before(pq.txs[j].CreatedAt) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Swap(i, j int) { + pq.txs[i], pq.txs[j] = pq.txs[j], pq.txs[i] + pq.idToIndex[pq.txs[i].ID] = j + pq.idToIndex[pq.txs[j].ID] = i +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Push(tx any) { + pq.txs = append(pq.txs, tx.(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pop() any { + old := pq.txs + n := len(old) + tx := old[n-1] + old[n-1] = nil // avoid memory leak + pq.txs = old[0 : n-1] + return tx +} + +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + pq.Lock() + defer pq.Unlock() + + heap.Push(pq, tx) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + pq.Lock() + defer pq.Unlock() + + return heap.Pop(pq).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveTxByID(id int64) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + pq.Lock() + defer pq.Unlock() + + if i, ok := pq.idToIndex[id]; ok { + return heap.Remove(pq, i).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) + } + + return nil +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + pq.Lock() + defer pq.Unlock() + + if len(pq.txs) == 0 { + return nil + } + return pq.txs[0] +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { + pq.Lock() + defer pq.Unlock() + + clear(pq.txs) +} From a0a1a7931b4f5e71912bac7277dc397e0957a465 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 11 Dec 2023 14:59:36 -0500 Subject: [PATCH 18/74] fix tests --- core/chains/tx_store_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/chains/tx_store_test.go b/core/chains/tx_store_test.go index bd2fdc71ef6..1430715fffb 100644 --- a/core/chains/tx_store_test.go +++ b/core/chains/tx_store_test.go @@ -241,11 +241,11 @@ func evmTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[commo ethClient := evmtest.NewEthClientMockWithDefaultChain(t) chainID := ethClient.ConfiguredChainID() - return cltest.NewTxStore(t, db, cfg.Database()), fromAddress, chainID + return cltest.NewTestTxStore(t, db, cfg.Database()), fromAddress, chainID } func inmemoryTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTxStore(t, db, cfg.Database()) + txStore := cltest.NewTestTxStore(t, db, cfg.Database()) keyStore := cltest.NewKeyStore(t, db, cfg.Database()) _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) ethClient := evmtest.NewEthClientMockWithDefaultChain(t) From 89d1056bc877794e02848310b7bfa779b9d57274 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 11 Dec 2023 15:15:59 -0500 Subject: [PATCH 19/74] fix bug in tests --- common/txmgr/address_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 35122b4297a..e1e13e9b0ca 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -86,7 +86,7 @@ func NewAddressState[ return nil, fmt.Errorf("address_state: initialization: %w", err) } as.inprogress = tx - if tx.IdempotencyKey != nil { + if tx != nil && tx.IdempotencyKey != nil { as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx } From 3663bd013832696424e15f65d57653ee325225ba Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 11 Dec 2023 22:12:39 -0500 Subject: [PATCH 20/74] run go generate --- core/chains/evm/txmgr/mocks/evm_tx_store.go | 74 +++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index a9a7023ac1f..fa1988c48c6 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -1340,6 +1340,80 @@ func (_m *EvmTxStore) TxAttempts(offset int, limit int) ([]types.TxAttempt[*big. return r0, r1, r2 } +// UnconfirmedTransactions provides a mock function with given fields: limit, offset, fromAddress, chainID +func (_m *EvmTxStore) UnconfirmedTransactions(limit int, offset int, fromAddress common.Address, chainID *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error) { + ret := _m.Called(limit, offset, fromAddress, chainID) + + if len(ret) == 0 { + panic("no return value specified for UnconfirmedTransactions") + } + + var r0 []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] + var r1 int + var r2 error + if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error)); ok { + return rf(limit, offset, fromAddress, chainID) + } + if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { + r0 = rf(limit, offset, fromAddress, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) + } + } + + if rf, ok := ret.Get(1).(func(int, int, common.Address, *big.Int) int); ok { + r1 = rf(limit, offset, fromAddress, chainID) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(int, int, common.Address, *big.Int) error); ok { + r2 = rf(limit, offset, fromAddress, chainID) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// UnstartedTransactions provides a mock function with given fields: limit, offset, fromAddress, chainID +func (_m *EvmTxStore) UnstartedTransactions(limit int, offset int, fromAddress common.Address, chainID *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error) { + ret := _m.Called(limit, offset, fromAddress, chainID) + + if len(ret) == 0 { + panic("no return value specified for UnstartedTransactions") + } + + var r0 []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] + var r1 int + var r2 error + if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error)); ok { + return rf(limit, offset, fromAddress, chainID) + } + if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { + r0 = rf(limit, offset, fromAddress, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) + } + } + + if rf, ok := ret.Get(1).(func(int, int, common.Address, *big.Int) int); ok { + r1 = rf(limit, offset, fromAddress, chainID) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(int, int, common.Address, *big.Int) error); ok { + r2 = rf(limit, offset, fromAddress, chainID) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + // UpdateBroadcastAts provides a mock function with given fields: ctx, now, etxIDs func (_m *EvmTxStore) UpdateBroadcastAts(ctx context.Context, now time.Time, etxIDs []int64) error { ret := _m.Called(ctx, now, etxIDs) From 54b7ecc49f10c8804c8238e26d9b4e31eb87a248 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 12 Dec 2023 08:42:00 -0500 Subject: [PATCH 21/74] address comment --- common/txmgr/inmemory_store.go | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index a8281d1249d..39c512c1d7a 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -103,7 +103,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat if ms.chainID.String() != chainID.String() { return tx, fmt.Errorf("create_transaction: %w", ErrInvalidChainID) } - if _, ok := ms.addressStates[txRequest.FromAddress]; !ok { + as, ok := ms.addressStates[tx.FromAddress] + if !ok { return tx, fmt.Errorf("create_transaction: %w", ErrAddressNotFound) } @@ -112,7 +113,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat if err != nil { return tx, fmt.Errorf("create_transaction: %w", err) } - if err := ms.sendTxToUnstartedQueue(tx); err != nil { + + // TODO(jtw); HANDLE PRUNING STEP + + // Add the request to the Unstarted channel to be processed by the Broadcaster + if err := as.addTxToUnstarted(&tx); err != nil { return tx, fmt.Errorf("create_transaction: %w", err) } @@ -434,19 +439,3 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband return nil } - -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) sendTxToUnstartedQueue(tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - as, ok := ms.addressStates[tx.FromAddress] - if !ok { - return fmt.Errorf("send_tx_to_unstarted_queue: %w", ErrAddressNotFound) - } - - // TODO(jtw); HANDLE PRUNING STEP - - // Add the request to the Unstarted channel to be processed by the Broadcaster - if err := as.addTxToUnstarted(&tx); err != nil { - return fmt.Errorf("send_tx_to_unstarted_queue: %w", err) - } - - return nil -} From f8e01750fc1dbc30bb100e9df2097b3a6af6b818 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 12 Dec 2023 09:16:03 -0500 Subject: [PATCH 22/74] update inmemory store to implement txstore interface --- common/txmgr/inmemory_store.go | 126 ++++++++++++++++++++++++++++++--- 1 file changed, 118 insertions(+), 8 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 39c512c1d7a..faeb0c19729 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -3,12 +3,16 @@ package txmgr import ( "context" "fmt" + "math/big" "sync" + "time" + "github.com/google/uuid" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/label" + "gopkg.in/guregu/null.v4" ) var ( @@ -33,16 +37,10 @@ type PersistentTxStore[ SEQ types.Sequence, FEE feetypes.Fee, ] interface { - Close() - Abandon(ctx context.Context, id CHAIN_ID, addr ADDR) error - CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) + txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + UnstartedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) UnconfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - UpdateTxAttemptInProgressToBroadcast(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], NewAttemptState txmgrtypes.TxAttemptState) error - SaveReplacementInProgressAttempt(ctx context.Context, oldAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], replacementAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - UpdateTxUnstartedToInProgress(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - UpdateTxFatalError(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error - GetTxInProgress(ctx context.Context, fromAddress ADDR) (etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) } type InMemoryStore[ @@ -439,3 +437,115 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband return nil } + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) (receiptsPlus []txmgrtypes.ReceiptPlus[R], err error) { + return ms.txStore.FindTxesPendingCallback(ctx, blockNum, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID) error { + return ms.txStore.UpdateTxCallbackCompleted(ctx, pipelineTaskRunRid, chainId) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx context.Context, receipts []R, chainID CHAIN_ID) (err error) { + return ms.txStore.SaveFetchedReceipts(ctx, receipts, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxesByMetaFieldAndStates(ctx, metaField, metaValue, states, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxesWithMetaFieldByStates(ctx, metaField, states, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxesWithMetaFieldByReceiptBlockNum(ctx, metaField, blockNum, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []big.Int, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx, ids, states, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (n int64, err error) { + return ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapTxHistory(ctx context.Context, minBlockNumberToKeep int64, timeThreshold time.Time, chainID CHAIN_ID) error { + return ms.txStore.ReapTxHistory(ctx, minBlockNumberToKeep, timeThreshold, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState, chainID CHAIN_ID) (count uint32, err error) { + return ms.txStore.CountTransactionsByState(ctx, state, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) DeleteInProgressAttempt(ctx context.Context, attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + return ms.txStore.DeleteInProgressAttempt(ctx, attempt) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxsRequiringGasBump(ctx, address, blockNum, gasBumpThreshold, depth, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringResubmissionDueToInsufficientFunds(ctx context.Context, address ADDR, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxsRequiringResubmissionDueToInsufficientFunds(ctx, address, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxAttemptsConfirmedMissingReceipt(ctx, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxAttemptsRequiringReceiptFetch(ctx, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxAttemptsRequiringResend(ctx, olderThan, maxInFlightTransactions, chainID, address) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithSequence(ctx context.Context, fromAddress ADDR, seq SEQ) (etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTxWithSequence(ctx, fromAddress, seq) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTransactionsConfirmedInBlockRange(ctx context.Context, highBlockNumber, lowBlockNumber int64, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.FindTransactionsConfirmedInBlockRange(ctx, highBlockNumber, lowBlockNumber, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context, chainID CHAIN_ID) (null.Time, error) { + return ms.txStore.FindEarliestUnconfirmedBroadcastTime(ctx, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context, chainID CHAIN_ID) (null.Int, error) { + return ms.txStore.FindEarliestUnconfirmedTxAttemptBlock(ctx, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetInProgressTxAttempts(ctx context.Context, address ADDR, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.GetInProgressTxAttempts(ctx, address, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNonFatalTransactions(ctx context.Context, chainID CHAIN_ID) (txs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.GetNonFatalTransactions(ctx, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxByID(ctx context.Context, id int64) (tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + return ms.txStore.GetTxByID(ctx, id) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasInProgressTransaction(ctx context.Context, account ADDR, chainID CHAIN_ID) (exists bool, err error) { + return ms.txStore.HasInProgressTransaction(ctx, account, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadTxAttempts(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + return ms.txStore.LoadTxAttempts(ctx, etx) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (err error) { + return ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { + return ms.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, finalityDepth, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PreloadTxes(ctx context.Context, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + return ms.txStore.PreloadTxes(ctx, attempts) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + return ms.txStore.SaveConfirmedMissingReceiptAttempt(ctx, timeout, attempt, broadcastAt) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInProgressAttempt(ctx context.Context, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + return ms.txStore.SaveInProgressAttempt(ctx, attempt) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInsufficientFundsAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + return ms.txStore.SaveInsufficientFundsAttempt(ctx, timeout, attempt, broadcastAt) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveSentAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + return ms.txStore.SaveSentAttempt(ctx, timeout, attempt, broadcastAt) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBroadcastBeforeBlockNum(ctx context.Context, blockNum int64, chainID CHAIN_ID) error { + return ms.txStore.SetBroadcastBeforeBlockNum(ctx, blockNum, chainID) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateBroadcastAts(ctx context.Context, now time.Time, etxIDs []int64) error { + return ms.txStore.UpdateBroadcastAts(ctx, now, etxIDs) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx context.Context, ids []int64) error { + return ms.txStore.UpdateTxsUnconfirmed(ctx, ids) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + return ms.txStore.UpdateTxForRebroadcast(ctx, etx, etxAttempt) +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxFinalized(ctx context.Context, blockHeight int64, txID int64, chainID CHAIN_ID) (finalized bool, err error) { + return ms.txStore.IsTxFinalized(ctx, blockHeight, txID, chainID) +} From df1a11aa693177685756b04d5772ab9f065a24a4 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 12 Dec 2023 09:59:00 -0500 Subject: [PATCH 23/74] cleanup tests --- core/chains/tx_store_test.go | 118 ----------------------------------- 1 file changed, 118 deletions(-) diff --git a/core/chains/tx_store_test.go b/core/chains/tx_store_test.go index 1430715fffb..c1fc3383d0a 100644 --- a/core/chains/tx_store_test.go +++ b/core/chains/tx_store_test.go @@ -105,134 +105,16 @@ func TestTxStore_CreateTransaction(t *testing.T) { t.Run(tt.scenario, func(t *testing.T) { actTx, actErr := txStore.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) tt.createTransactionOutputCheck(t, actTx, actErr) - - // TODO(jtw): Check that the transaction was persisted }) } }) } } -/* -func TestTxStore_FindTxWithIdempotencyKey(t *testing.T) { - txStore := evmtxmgr.NewTxStore(nil, nil, nil) - ctx := context.Background() - - tts := []struct { - scenario string - findTxWithIdempotencyKeyInput findTxWithIdempotencyKeyInput - findTxWithIdempotencyKeyOutput func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) - }{ - { - findTxWithIdempotencyKeyInput: findTxWithIdempotencyKeyInput{ - idempotencyKey: "11", - chainID: chainID, - }, - findTxWithIdempotencyKeyOutput: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { - funcName := "FindTxWithIdempotencyKey" - require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) - assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) - assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) - assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) - assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) - assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) - assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) - assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) - var expMeta *datatypes.JSON - assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) - assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) - }, - }, - } - - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actTxPtr, actErr := txStore.FindTxWithIdempotencyKey(ctx, tt.findTxWithIdempotencyKeyInput.idempotencyKey, tt.findTxWithIdempotencyKeyInput.chainID) - tt.findTxWithIdempotencyKeyOutput(t, *actTxPtr, actErr) - }) - } -} - -func TestTxStore_CheckTxQueueCapacity(t *testing.T) { - txStore := evmtxmgr.NewTxStore(nil, nil, nil) - ctx := context.Background() - - tts := []struct { - scenario string - checkTxQueueCapacityInput checkTxQueueCapacityInput - expErr error - }{ - { - checkTxQueueCapacityInput: checkTxQueueCapacityInput{ - fromAddress: fromAddress, - maxQueued: uint64(16), - chainID: chainID, - }, - expErr: nil, - }, - } - - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actErr := txStore.CheckTxQueueCapacity(ctx, tt.checkTxQueueCapacityInput.fromAddress, tt.checkTxQueueCapacityInput.maxQueued, tt.checkTxQueueCapacityInput.chainID) - require.Equal(t, tt.expErr, actErr, "CheckTxQueueCapacity: expected err to match actual err") - }) - } -} - -func TestTxStore_FindLatestSequence(t *testing.T) { - txStore := evmtxmgr.NewTxStore(nil, nil, nil) - ctx := context.Background() - - tts := []struct { - scenario string - findLatestSequenceInput findLatestSequenceInput - findLatestSequenceOutput func(*testing.T, evmtypes.Nonce, error) - }{ - { - findLatestSequenceInput: findLatestSequenceInput{ - fromAddress: fromAddress, - chainID: chainID, - }, - findLatestSequenceOutput: func(t *testing.T, seq evmtypes.Nonce, err error) { - funcName := "FindLatestSequence" - require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) - assert.Equal(t, uint64(0), seq, fmt.Sprintf("%s: expected seq to match actual seq", funcName)) - }, - }, - } - - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actSeq, actErr := txStore.FindLatestSequence(ctx, tt.findLatestSequenceInput.fromAddress, tt.findLatestSequenceInput.chainID) - tt.findLatestSequenceOutput(t, actSeq, actErr) - }) - } -} -*/ - type createTransactionInput struct { txRequest txmgrtypes.TxRequest[common.Address, common.Hash] chainID *big.Int } -type findTxWithIdempotencyKeyInput struct { - idempotencyKey string - chainID *big.Int -} -type checkTxQueueCapacityInput struct { - fromAddress common.Address - maxQueued uint64 - chainID *big.Int -} -type checkTxQueueCapacityOutput struct { - err error -} -type findLatestSequenceInput struct { - fromAddress common.Address - chainID *big.Int -} func evmTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { db := pgtest.NewSqlxDB(t) From f24eea7dd37978dfb28095be6685c6c770a95ca0 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 13 Dec 2023 21:27:23 -0500 Subject: [PATCH 24/74] add deepcopy --- common/txmgr/inmemory_store.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index faeb0c19729..77fc7d6579e 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -133,7 +133,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { if tx := as.findTxWithIdempotencyKey(idempotencyKey); tx != nil { - return tx, nil + return ms.deepCopyTx(tx), nil } } @@ -271,7 +271,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx "Your database is in an inconsistent state and this node will not function correctly until the problem is resolved", tx.ID) } - return tx, nil + return ms.deepCopyTx(tx), nil } // UpdateTxAttemptInProgressToBroadcast updates a transaction attempt from in_progress to broadcast. @@ -549,3 +549,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxFinalized(ctx context.Context, blockHeight int64, txID int64, chainID CHAIN_ID) (finalized bool, err error) { return ms.txStore.IsTxFinalized(ctx, blockHeight, txID, chainID) } + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + etx := *tx + etx.TxAttempts = make([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(tx.TxAttempts)) + copy(etx.TxAttempts, tx.TxAttempts) + + return &etx +} From ff892a16b91710ef007f860a0925ba0d64ea2c14 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 13 Dec 2023 23:35:38 -0500 Subject: [PATCH 25/74] initial confirmer work --- common/txmgr/address_state.go | 142 ++++++++++++++++++++++++-- common/txmgr/inmemory_store.go | 135 +++++++++++++++++++++--- core/chains/evm/txmgr/evm_tx_store.go | 45 +++++++- 3 files changed, 297 insertions(+), 25 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index e1e13e9b0ca..78661684d63 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -28,7 +28,10 @@ type AddressState[ unstarted *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] inprogress *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] // NOTE: currently the unconfirmed map's key is the transaction ID that is assigned via the postgres DB - unconfirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + unconfirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + confirmedMissingReceipt map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + confirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + allTransactions map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } // NewAddressState returns a new AddressState instance @@ -49,10 +52,13 @@ func NewAddressState[ fromAddress: fromAddress, txStore: txStore, - idempotencyKeyToTx: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), - inprogress: nil, - unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + idempotencyKeyToTx: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), + inprogress: nil, + unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + confirmedMissingReceipt: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + confirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + allTransactions: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } as.Lock() @@ -69,6 +75,7 @@ func NewAddressState[ for i := 0; i < len(txs); i++ { tx := txs[i] as.unstarted.AddTx(&tx) + as.allTransactions[tx.ID] = &tx if tx.IdempotencyKey != nil { as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx } @@ -86,8 +93,11 @@ func NewAddressState[ return nil, fmt.Errorf("address_state: initialization: %w", err) } as.inprogress = tx - if tx != nil && tx.IdempotencyKey != nil { - as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx + if tx != nil { + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx + } + as.allTransactions[tx.ID] = tx } // Load all unconfirmed transactions from persistent storage @@ -101,6 +111,7 @@ func NewAddressState[ for i := 0; i < len(txs); i++ { tx := txs[i] as.unconfirmed[tx.ID] = &tx + as.allTransactions[tx.ID] = &tx if tx.IdempotencyKey != nil { as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx } @@ -111,8 +122,51 @@ func NewAddressState[ offset += limit } - return &as, nil + // Load all confirmed transactions from persistent storage + offset = 0 + limit = 50 + for { + txs, count, err := txStore.ConfirmedTransactions(offset, limit, as.fromAddress, as.chainID) + if err != nil { + return nil, fmt.Errorf("address_state: initialization: %w", err) + } + for i := 0; i < len(txs); i++ { + tx := txs[i] + as.confirmed[tx.ID] = &tx + as.allTransactions[tx.ID] = &tx + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx + } + } + if count <= offset+limit { + break + } + offset += limit + } + // Load all unconfirmed transactions from persistent storage + offset = 0 + limit = 50 + for { + txs, count, err := txStore.ConfirmedMissingReceiptTransactions(offset, limit, as.fromAddress, as.chainID) + if err != nil { + return nil, fmt.Errorf("address_state: initialization: %w", err) + } + for i := 0; i < len(txs); i++ { + tx := txs[i] + as.confirmedMissingReceipt[tx.ID] = &tx + as.allTransactions[tx.ID] = &tx + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx + } + } + if count <= offset+limit { + break + } + offset += limit + } + + return &as, nil } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close() { @@ -146,6 +200,18 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findTx return as.idempotencyKeyToTx[key] } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) confirmedMissingReceiptTxs() []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + as.RLock() + defer as.RUnlock() + + var txAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, tx := range as.confirmedMissingReceipt { + txAttempts = append(txAttempts, tx.TxAttempts...) + } + + return txAttempts +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLatestSequence() SEQ { as.RLock() defer as.RUnlock() @@ -168,6 +234,65 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLa return maxSeq } +// TODO(jtw): THIS MIGHT BE ABLE TO BE MERGED WITH OTHER FILTER AND APPLY FUNCTIONS +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToAll( + fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), + txIDs ...int64, +) { + as.Lock() + defer as.Unlock() + + // if txIDs is not empty then only apply the filter to those transactions + if len(txIDs) > 0 { + for _, txID := range txIDs { + tx := as.allTransactions[txID] + if tx != nil { + fn(tx) + } + } + return + } + + // if txIDs is empty then apply the filter to all transactions + for _, tx := range as.allTransactions { + fn(tx) + } +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToUnconfirmed( + fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), + txIDs ...int64, +) { + as.Lock() + defer as.Unlock() + + // if txIDs is not empty then only apply the filter to those transactions + if len(txIDs) > 0 { + for _, txID := range txIDs { + tx := as.unconfirmed[txID] + if tx != nil { + fn(tx) + } + } + return + } + + // if txIDs is empty then apply the filter to all transactions + for _, tx := range as.unconfirmed { + fn(tx) + } +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxAttempts( + txStates []txmgrtypes.TxState, + filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txIDs ...int64, +) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + as.RLock() + defer as.RUnlock() + // TODO: this is a naive implementation +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.RLock() defer as.RUnlock() @@ -201,6 +326,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) addTxT } as.unstarted.AddTx(tx) + as.allTransactions[tx.ID] = tx if tx.IdempotencyKey != nil { as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 77fc7d6579e..a897780d745 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "sort" "sync" "time" @@ -41,6 +42,8 @@ type PersistentTxStore[ UnstartedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) UnconfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + ConfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + ConfirmedMissingReceiptTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) } type InMemoryStore[ @@ -438,6 +441,123 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband return nil } +// SetBroadcastBeforeBlockNum sets the broadcast_before_block_num for a given chain ID +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBroadcastBeforeBlockNum(ctx context.Context, blockNum int64, chainID CHAIN_ID) error { + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("set_broadcast_before_block_num: %w", ErrInvalidChainID) + } + + // Persist to persistent storage + if err := ms.txStore.SetBroadcastBeforeBlockNum(ctx, blockNum, chainID); err != nil { + return fmt.Errorf("set_broadcast_before_block_num: %w", err) + } + + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return + } + // TODO(jtw): how many tx_attempts are actually stored in the db for each tx? It looks like its only 1 + attempt := tx.TxAttempts[0] + if attempt.State == txmgrtypes.TxAttemptBroadcast && attempt.BroadcastBeforeBlockNum == nil && + tx.ChainID.String() == chainID.String() { + tx.TxAttempts[0].BroadcastBeforeBlockNum = &blockNum + } + } + for _, as := range ms.addressStates { + as.applyToUnconfirmed(fn) + } + + return nil +} + +// FindTxAttemptsConfirmedMissingReceipt returns all transactions that are confirmed but missing a receipt +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) + } + + attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + attempts = append(attempts, as.txAttemptsWIthConfirmedMissingReceiptTx()...) + } + // sort by tx_id ASC, gas_price DESC, gas_tip_cap DESC + sort.SliceStable(attempts, func(i, j int) bool { + /* + // TODO: THIS IS CURRENTLY TIED TO ETHEREUM WE MIGHT WANT TO MAKE METHODS ON FEE FOR PRICE AND TIP CAP + if attempts[i].TxID == attempts[j].TxID { + if attempts[i].TxFee.GasPrice == attempts[j].TxFee.GasPrice { + return attempts[i].TxFee.GasTipCap > attempts[j].TxFee.GasTipCap + } + return attempts[i].TxFee.GasPrice > attempts[j].TxFee.GasPrice + } + */ + return attempts[i].TxID < attempts[j].TxID + }) + + return attempts, nil +} + +// UpdateBroadcastAts updates the broadcast_at time for a given set of attempts +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateBroadcastAts(ctx context.Context, now time.Time, txIDs []int64) error { + // Persist to persistent storage + if err := ms.txStore.UpdateBroadcastAts(ctx, now, txIDs); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.BroadcastAt != nil { + tx.BroadcastAt = &now + } + } + + for _, as := range ms.addressStates { + as.applyToUnconfirmed(fn, txIDs...) + } + + return nil +} + +// UpdateTxsUnconfirmed updates the unconfirmed transactions for a given set of ids +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx context.Context, txIDs []int64) error { + // Persist to persistent storage + if err := ms.txStore.UpdateTxsUnconfirmed(ctx, txIDs); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + tx.State = TxUnconfirmed + } + + for _, as := range ms.addressStates { + as.applyToAll(fn, txIDs...) + } + + return nil +} + +// FindTxAttemptsRequiringReceiptFetch returns all transactions that are missing a receipt +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { + if ms.chainID.String() != chainID.String() { + return attempts, fmt.Errorf("find_tx_attempts_requiring_receipt_fetch: %w", ErrInvalidChainID) + } + + states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt} + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { + attempt := tx.TxAttempts[0] + return attempt.State == txmgrtypes.TxAttemptInsufficientFunds + } + + return false + } + attempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + attempts = append(as.fetchTxAttempts(states, filterFn), attempts...) + } +} + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) (receiptsPlus []txmgrtypes.ReceiptPlus[R], err error) { return ms.txStore.FindTxesPendingCallback(ctx, blockNum, chainID) } @@ -477,12 +597,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringResubmissionDueToInsufficientFunds(ctx context.Context, address ADDR, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { return ms.txStore.FindTxsRequiringResubmissionDueToInsufficientFunds(ctx, address, chainID) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxAttemptsConfirmedMissingReceipt(ctx, chainID) -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxAttemptsRequiringReceiptFetch(ctx, chainID) -} func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { return ms.txStore.FindTxAttemptsRequiringResend(ctx, olderThan, maxInFlightTransactions, chainID, address) } @@ -534,15 +648,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveSentAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { return ms.txStore.SaveSentAttempt(ctx, timeout, attempt, broadcastAt) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBroadcastBeforeBlockNum(ctx context.Context, blockNum int64, chainID CHAIN_ID) error { - return ms.txStore.SetBroadcastBeforeBlockNum(ctx, blockNum, chainID) -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateBroadcastAts(ctx context.Context, now time.Time, etxIDs []int64) error { - return ms.txStore.UpdateBroadcastAts(ctx, now, etxIDs) -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxsUnconfirmed(ctx context.Context, ids []int64) error { - return ms.txStore.UpdateTxsUnconfirmed(ctx, ids) -} func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { return ms.txStore.UpdateTxForRebroadcast(ctx, etx, etxAttempt) } diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index bfaa8f38610..08d63cc1aaf 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -48,6 +48,7 @@ type EvmTxStore interface { // redeclare TxStore for mockery txmgrtypes.TxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee] TxStoreWebApi + TxStoreInMemory } // TxStoreWebApi encapsulates the methods that are not used by the txmgr and only used by the various web controllers and readers @@ -59,8 +60,14 @@ type TxStoreWebApi interface { TransactionsWithAttempts(offset, limit int) ([]Tx, int, error) FindTxAttempt(hash common.Hash) (*TxAttempt, error) FindTxWithAttempts(etxID int64) (etx Tx, err error) - UnstartedTransactions(limit, offset int, fromAddress common.Address, chainID *big.Int) ([]Tx, int, error) - UnconfirmedTransactions(limit, offset int, fromAddress common.Address, chainID *big.Int) ([]Tx, int, error) +} + +// TxStoreInMemory encapsulates the methods that are not used by the txmgr and only used by the in memory tx store. +type TxStoreInMemory interface { + UnstartedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) + UnconfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) + ConfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) + ConfirmedMissingReceiptTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) } type TestEvmTxStore interface { @@ -495,6 +502,40 @@ func (o *evmTxStore) UnconfirmedTransactions(offset, limit int, fromAddress comm return } +// ConfirmedTransactions returns all eth transactions that have at least one attempt and in the confirmed state. +func (o *evmTxStore) ConfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { + sql := `SELECT count(*) FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed' AND from_address = $1 AND evm_chain_id = $2` + if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { + return + } + + sql = `SELECT * FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` + var dbTxs []DbEthTx + if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { + return + } + txs = dbEthTxsToEvmEthTxs(dbTxs) + err = o.preloadTxAttempts(txs) + return +} + +// ConfirmedMissingReceiptTransactions returns all eth transactions that have at least one attempt and in the confirmed_missing_receipt state. +func (o *evmTxStore) ConfirmedMissingReceiptTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { + sql := `SELECT count(*) FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed_missing_receipt' AND from_address = $1 AND evm_chain_id = $2` + if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { + return + } + + sql = `SELECT * FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed_missing_receipt' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` + var dbTxs []DbEthTx + if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { + return + } + txs = dbEthTxsToEvmEthTxs(dbTxs) + err = o.preloadTxAttempts(txs) + return +} + // TxAttempts returns the last tx attempts sorted by created_at descending. func (o *evmTxStore) TxAttempts(offset, limit int) (txs []TxAttempt, count int, err error) { sql := `SELECT count(*) FROM evm.tx_attempts` From cabb7bfd0d56dbaf4184cdb9005a373341c4c131 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 15 Dec 2023 18:09:06 -0500 Subject: [PATCH 26/74] implement a few more methods --- common/txmgr/address_state.go | 92 ++++++++++++++++++++++----- common/txmgr/inmemory_store.go | 52 +++++++++------ common/txmgr/types/tx_store.go | 17 +++++ core/chains/evm/txmgr/evm_tx_store.go | 2 +- 4 files changed, 127 insertions(+), 36 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 78661684d63..1b0c6641c6b 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -234,32 +234,38 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLa return maxSeq } -// TODO(jtw): THIS MIGHT BE ABLE TO BE MERGED WITH OTHER FILTER AND APPLY FUNCTIONS -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToAll( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToTxs( + txStates []txmgrtypes.TxState, fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), txIDs ...int64, ) { as.Lock() defer as.Unlock() - // if txIDs is not empty then only apply the filter to those transactions - if len(txIDs) > 0 { - for _, txID := range txIDs { - tx := as.allTransactions[txID] - if tx != nil { - fn(tx) - } - } + // if txStates is empty then apply the filter to only the as.allTransactions map + if txStates == nil || len(txStates) == 0 { + as.applyToStorage(as.allTransactions, fn, txIDs...) return } - // if txIDs is empty then apply the filter to all transactions - for _, tx := range as.allTransactions { - fn(tx) + for _, txState := range txStates { + switch txState { + case TxInProgress: + if as.inprogress != nil { + fn(as.inprogress) + } + case TxUnconfirmed: + as.applyToStorage(as.unconfirmed, fn, txIDs...) + case TxConfirmedMissingReceipt: + as.applyToStorage(as.confirmedMissingReceipt, fn, txIDs...) + case TxConfirmed: + as.applyToStorage(as.confirmed, fn, txIDs...) + } } } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToUnconfirmed( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToStorage( + txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), txIDs ...int64, ) { @@ -269,7 +275,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyT // if txIDs is not empty then only apply the filter to those transactions if len(txIDs) > 0 { for _, txID := range txIDs { - tx := as.unconfirmed[txID] + tx := txIDsToTx[txID] if tx != nil { fn(tx) } @@ -278,7 +284,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyT } // if txIDs is empty then apply the filter to all transactions - for _, tx := range as.unconfirmed { + for _, tx := range txIDsToTx { fn(tx) } } @@ -290,7 +296,59 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT ) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { as.RLock() defer as.RUnlock() - // TODO: this is a naive implementation + + // if txStates is empty then apply the filter to only the as.allTransactions map + if txStates == nil || len(txStates) == 0 { + return as.fetchTxAttemptsFromStorage(as.allTransactions, filter, txIDs...) + } + + var txAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, txState := range txStates { + switch txState { + case TxInProgress: + if as.inprogress != nil && filter(as.inprogress) { + txAttempts = append(txAttempts, as.inprogress.TxAttempts...) + } + case TxUnconfirmed: + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.unconfirmed, filter, txIDs...)...) + case TxConfirmedMissingReceipt: + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmedMissingReceipt, filter, txIDs...)...) + case TxConfirmed: + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmed, filter, txIDs...)...) + } + } + + return txAttempts +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxAttemptsFromStorage( + txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txIDs ...int64, +) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + as.RLock() + defer as.RUnlock() + + var txAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // if txIDs is not empty then only apply the filter to those transactions + if len(txIDs) > 0 { + for _, txID := range txIDs { + tx := txIDsToTx[txID] + if tx != nil && filter(tx) { + txAttempts = append(txAttempts, tx.TxAttempts...) + } + } + return txAttempts + } + + // if txIDs is empty then apply the filter to all transactions + for _, tx := range txIDsToTx { + if filter(tx) { + txAttempts = append(txAttempts, tx.TxAttempts...) + } + } + + return txAttempts } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index a897780d745..4b85369c8e9 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -4,12 +4,12 @@ import ( "context" "fmt" "math/big" - "sort" "sync" "time" "github.com/google/uuid" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" + "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/label" @@ -464,7 +464,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr } } for _, as := range ms.addressStates { - as.applyToUnconfirmed(fn) + as.applyToTxs([]txmgrtypes.TxState{txmgr.TxUnconfirmed}, fn) } return nil @@ -476,23 +476,35 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { + if tx.ChainID.String() != chainID.String() { + return false + } + return true + } + + return false + } + states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} for _, as := range ms.addressStates { - attempts = append(attempts, as.txAttemptsWIthConfirmedMissingReceiptTx()...) + attempts = append(attempts, as.fetchTxAttempts(states, filter)...) } // sort by tx_id ASC, gas_price DESC, gas_tip_cap DESC - sort.SliceStable(attempts, func(i, j int) bool { - /* - // TODO: THIS IS CURRENTLY TIED TO ETHEREUM WE MIGHT WANT TO MAKE METHODS ON FEE FOR PRICE AND TIP CAP - if attempts[i].TxID == attempts[j].TxID { - if attempts[i].TxFee.GasPrice == attempts[j].TxFee.GasPrice { - return attempts[i].TxFee.GasTipCap > attempts[j].TxFee.GasTipCap + // TODO + /* + sort.SliceStable(attempts, func(i, j int) bool { + // TODO: THIS IS CURRENTLY TIED TO ETHEREUM WE MIGHT WANT TO MAKE METHODS ON FEE FOR PRICE AND TIP CAP + if attempts[i].TxID == attempts[j].TxID { + if attempts[i].TxFee.GasPrice == attempts[j].TxFee.GasPrice { + return attempts[i].TxFee.GasTipCap > attempts[j].TxFee.GasTipCap + } + return attempts[i].TxFee.GasPrice > attempts[j].TxFee.GasPrice } - return attempts[i].TxFee.GasPrice > attempts[j].TxFee.GasPrice - } - */ - return attempts[i].TxID < attempts[j].TxID - }) + return attempts[i].TxID < attempts[j].TxID + }) + */ return attempts, nil } @@ -512,7 +524,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } for _, as := range ms.addressStates { - as.applyToUnconfirmed(fn, txIDs...) + as.applyToTxs(txmgr.TxUnconfirmed, fn, txIDs...) } return nil @@ -531,7 +543,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } for _, as := range ms.addressStates { - as.applyToAll(fn, txIDs...) + as.applyToTxs(nil, fn, txIDs...) } return nil @@ -543,19 +555,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return attempts, fmt.Errorf("find_tx_attempts_requiring_receipt_fetch: %w", ErrInvalidChainID) } - states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt} filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { attempt := tx.TxAttempts[0] - return attempt.State == txmgrtypes.TxAttemptInsufficientFunds + return attempt.State != txmgrtypes.TxAttemptInsufficientFunds } return false } + states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} attempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} for _, as := range ms.addressStates { attempts = append(as.fetchTxAttempts(states, filterFn), attempts...) } + // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC + // TODO + + return attempts, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) (receiptsPlus []txmgrtypes.ReceiptPlus[R], err error) { diff --git a/common/txmgr/types/tx_store.go b/common/txmgr/types/tx_store.go index 57ecf28d589..2d8d46a4930 100644 --- a/common/txmgr/types/tx_store.go +++ b/common/txmgr/types/tx_store.go @@ -34,6 +34,7 @@ type TxStore[ UnstartedTxQueuePruner TxHistoryReaper[CHAIN_ID] TransactionStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, SEQ, FEE] + InMemoryInitializer[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] // Find confirmed txes beyond the minConfirmations param that require callback but have not yet been signaled FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) (receiptsPlus []ReceiptPlus[R], err error) @@ -55,6 +56,22 @@ type TxStore[ FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []big.Int, states []TxState, chainID *big.Int) (tx []*Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) } +// InMemoryInitializer encapsulates the methods that are used by the txmgr to initialize the in memory tx store. +type InMemoryInitializer[ + ADDR types.Hashable, + CHAIN_ID types.ID, + TX_HASH types.Hashable, + BLOCK_HASH types.Hashable, + R ChainReceipt[TX_HASH, BLOCK_HASH], + SEQ types.Sequence, + FEE feetypes.Fee, +] interface { + UnstartedTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + UnconfirmedTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + ConfirmedTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + ConfirmedMissingReceiptTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) +} + // TransactionStore contains the persistence layer methods needed to manage Txs and TxAttempts type TransactionStore[ ADDR types.Hashable, diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 08d63cc1aaf..452b31349db 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -62,7 +62,7 @@ type TxStoreWebApi interface { FindTxWithAttempts(etxID int64) (etx Tx, err error) } -// TxStoreInMemory encapsulates the methods that are not used by the txmgr and only used by the in memory tx store. +// TxStoreInMemory encapsulates the methods that are used by the txmgr to initialize the in memory tx store. type TxStoreInMemory interface { UnstartedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) UnconfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) From 2efe71b66787d1cdf5c9b925915a6b349c152f99 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 17:26:13 -0500 Subject: [PATCH 27/74] implement findTxesPendingCallback --- common/txmgr/address_state.go | 66 ++++++++++++++++++++++++++- common/txmgr/inmemory_store.go | 81 ++++++++++++++++++++++++---------- 2 files changed, 121 insertions(+), 26 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 1b0c6641c6b..d6682459785 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -234,7 +234,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLa return maxSeq } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToTxs( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyToTxs( txStates []txmgrtypes.TxState, fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), txIDs ...int64, @@ -289,7 +289,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyT } } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxAttempts( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchTxAttempts( txStates []txmgrtypes.TxState, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, txIDs ...int64, @@ -351,6 +351,68 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT return txAttempts } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchTxs( + txStates []txmgrtypes.TxState, + filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txIDs ...int64, +) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + as.RLock() + defer as.RUnlock() + + // if txStates is empty then apply the filter to only the as.allTransactions map + if txStates == nil || len(txStates) == 0 { + return as.fetchTxsFromStorage(as.allTransactions, filter, txIDs...) + } + + var txs []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, txState := range txStates { + switch txState { + case TxInProgress: + if as.inprogress != nil && filter(as.inprogress) { + txs = append(txs, *as.inprogress) + } + case TxUnconfirmed: + txs = append(txs, as.fetchTxsFromStorage(as.unconfirmed, filter, txIDs...)...) + case TxConfirmedMissingReceipt: + txs = append(txs, as.fetchTxsFromStorage(as.confirmedMissingReceipt, filter, txIDs...)...) + case TxConfirmed: + txs = append(txs, as.fetchTxsFromStorage(as.confirmed, filter, txIDs...)...) + } + } + + return txs +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxsFromStorage( + txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txIDs ...int64, +) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + as.RLock() + defer as.RUnlock() + + var txs []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // if txIDs is not empty then only apply the filter to those transactions + if len(txIDs) > 0 { + for _, txID := range txIDs { + tx := txIDsToTx[txID] + if tx != nil && filter(tx) { + txs = append(txs, *tx) + } + } + return txs + } + + // if txIDs is empty then apply the filter to all transactions + for _, tx := range txIDsToTx { + if filter(tx) { + txs = append(txs, *tx) + } + } + + return txs +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.RLock() defer as.RUnlock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 4b85369c8e9..92a98571f55 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -2,6 +2,7 @@ package txmgr import ( "context" + "encoding/json" "fmt" "math/big" "sync" @@ -9,7 +10,6 @@ import ( "github.com/google/uuid" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" - "github.com/smartcontractkit/chainlink/v2/common/txmgr" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/label" @@ -464,7 +464,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr } } for _, as := range ms.addressStates { - as.applyToTxs([]txmgrtypes.TxState{txmgr.TxUnconfirmed}, fn) + as.ApplyToTxs(nil, fn) } return nil @@ -478,10 +478,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { - if tx.ChainID.String() != chainID.String() { - return false - } - return true + return tx.ChainID.String() == chainID.String() } return false @@ -489,22 +486,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} for _, as := range ms.addressStates { - attempts = append(attempts, as.fetchTxAttempts(states, filter)...) + attempts = append(attempts, as.FetchTxAttempts(states, filter)...) } // sort by tx_id ASC, gas_price DESC, gas_tip_cap DESC // TODO - /* - sort.SliceStable(attempts, func(i, j int) bool { - // TODO: THIS IS CURRENTLY TIED TO ETHEREUM WE MIGHT WANT TO MAKE METHODS ON FEE FOR PRICE AND TIP CAP - if attempts[i].TxID == attempts[j].TxID { - if attempts[i].TxFee.GasPrice == attempts[j].TxFee.GasPrice { - return attempts[i].TxFee.GasTipCap > attempts[j].TxFee.GasTipCap - } - return attempts[i].TxFee.GasPrice > attempts[j].TxFee.GasPrice - } - return attempts[i].TxID < attempts[j].TxID - }) - */ return attempts, nil } @@ -524,7 +509,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } for _, as := range ms.addressStates { - as.applyToTxs(txmgr.TxUnconfirmed, fn, txIDs...) + as.ApplyToTxs(nil, fn, txIDs...) } return nil @@ -543,7 +528,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } for _, as := range ms.addressStates { - as.applyToTxs(nil, fn, txIDs...) + as.ApplyToTxs(nil, fn, txIDs...) } return nil @@ -566,7 +551,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} attempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} for _, as := range ms.addressStates { - attempts = append(as.fetchTxAttempts(states, filterFn), attempts...) + attempts = append(attempts, as.FetchTxAttempts(states, filterFn)...) } // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC // TODO @@ -574,9 +559,57 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return attempts, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) (receiptsPlus []txmgrtypes.ReceiptPlus[R], err error) { - return ms.txStore.FindTxesPendingCallback(ctx, blockNum, chainID) +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) ([]txmgrtypes.ReceiptPlus[R], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txes_pending_callback: %w", ErrInvalidChainID) + } + + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + + if tx.TxAttempts[0].Receipts == nil || len(tx.TxAttempts[0].Receipts) == 0 { + return false + } + + if tx.PipelineTaskRunID.Valid && tx.SignalCallback && !tx.CallbackCompleted && + tx.TxAttempts[0].Receipts[0].GetBlockNumber() != nil && + big.NewInt(blockNum-int64(tx.MinConfirmations.Uint32)).Cmp(tx.TxAttempts[0].Receipts[0].GetBlockNumber()) > 0 { + return true + } + + return false + + } + states := []txmgrtypes.TxState{TxConfirmed} + txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + txs = append(txs, as.FetchTxs(states, filterFn)...) + } + + receiptsPlus := make([]txmgrtypes.ReceiptPlus[R], len(txs)) + meta := map[string]interface{}{} + for i, tx := range txs { + if err := json.Unmarshal(json.RawMessage(*tx.Meta), &meta); err != nil { + return nil, err + } + failOnRevert := false + if v, ok := meta["FailOnRevert"].(bool); ok { + failOnRevert = v + } + + receiptsPlus[i] = txmgrtypes.ReceiptPlus[R]{ + ID: tx.PipelineTaskRunID.UUID, + Receipt: (tx.TxAttempts[0].Receipts[0]).(R), + FailOnRevert: failOnRevert, + } + clear(meta) + } + + return receiptsPlus, nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID) error { return ms.txStore.UpdateTxCallbackCompleted(ctx, pipelineTaskRunRid, chainId) } From 39936cfa39c44a9d81e0af0a71cd6f8b170bc940 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 17:30:13 -0500 Subject: [PATCH 28/74] UpdateTxCallbackCompleted --- common/txmgr/inmemory_store.go | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 92a98571f55..f280f913c56 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -611,30 +611,50 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxCallbackCompleted(ctx context.Context, pipelineTaskRunRid uuid.UUID, chainId CHAIN_ID) error { - return ms.txStore.UpdateTxCallbackCompleted(ctx, pipelineTaskRunRid, chainId) + if ms.chainID.String() != chainId.String() { + return fmt.Errorf("update_tx_callback_completed: %w", ErrInvalidChainID) + } + + // Persist to persistent storage + if err := ms.txStore.UpdateTxCallbackCompleted(ctx, pipelineTaskRunRid, chainId); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.PipelineTaskRunID.UUID == pipelineTaskRunRid { + tx.CallbackCompleted = true + } + } + for _, as := range ms.addressStates { + as.ApplyToTxs(nil, fn) + } + + return nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx context.Context, receipts []R, chainID CHAIN_ID) (err error) { + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx context.Context, receipts []R, chainID CHAIN_ID) error { return ms.txStore.SaveFetchedReceipts(ctx, receipts, chainID) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { return ms.txStore.FindTxesByMetaFieldAndStates(ctx, metaField, metaValue, states, chainID) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { return ms.txStore.FindTxesWithMetaFieldByStates(ctx, metaField, states, chainID) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { return ms.txStore.FindTxesWithMetaFieldByReceiptBlockNum(ctx, metaField, blockNum, chainID) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []big.Int, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { return ms.txStore.FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx, ids, states, chainID) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (n int64, err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (int64, error) { return ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapTxHistory(ctx context.Context, minBlockNumberToKeep int64, timeThreshold time.Time, chainID CHAIN_ID) error { return ms.txStore.ReapTxHistory(ctx, minBlockNumberToKeep, timeThreshold, chainID) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState, chainID CHAIN_ID) (count uint32, err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState, chainID CHAIN_ID) (uint32, error) { return ms.txStore.CountTransactionsByState(ctx, state, chainID) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) DeleteInProgressAttempt(ctx context.Context, attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { From d601c85b60051d37da52e99e210ac979d56fd684 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 18:10:00 -0500 Subject: [PATCH 29/74] initial work on SaveFetchedReceipts --- common/txmgr/inmemory_store.go | 49 +++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index f280f913c56..3e35a329db8 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -634,8 +634,55 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveFetchedReceipts(ctx context.Context, receipts []R, chainID CHAIN_ID) error { - return ms.txStore.SaveFetchedReceipts(ctx, receipts, chainID) + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("save_fetched_receipts: %w", ErrInvalidChainID) + } + + // Persist to persistent storage + if err := ms.txStore.SaveFetchedReceipts(ctx, receipts, chainID); err != nil { + return err + } + + // convert receipts to map + receiptsMap := map[TX_HASH]R{} + for _, receipt := range receipts { + receiptsMap[receipt.GetTxHash()] = receipt + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return + } + attempt := tx.TxAttempts[0] + receipt, ok := receiptsMap[attempt.Hash] + if !ok { + return + } + + if attempt.Receipts != nil && len(attempt.Receipts) > 0 && + attempt.Receipts[0].GetBlockNumber() != nil && receipt.GetBlockNumber() != nil && + attempt.Receipts[0].GetBlockNumber().Cmp(receipt.GetBlockNumber()) == 0 { + return + } + // TODO(jtw): this needs to be finished + + attempt.State = txmgrtypes.TxAttemptBroadcast + if attempt.BroadcastBeforeBlockNum == nil { + blocknum := receipt.GetBlockNumber().Int64() + attempt.BroadcastBeforeBlockNum = &blocknum + } + attempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} + + tx.State = TxConfirmed + } + for _, as := range ms.addressStates { + as.ApplyToTxs(nil, fn) + } + + return nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { return ms.txStore.FindTxesByMetaFieldAndStates(ctx, metaField, metaValue, states, chainID) } From f51da913ce6f985f716910112f8f01e2c91ebbde Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 18:52:08 -0500 Subject: [PATCH 30/74] implement FindTxesByMetaFieldAndStates --- common/txmgr/inmemory_store.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 3e35a329db8..f236480d6d2 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -684,7 +684,32 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveF } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - return ms.txStore.FindTxesByMetaFieldAndStates(ctx, metaField, metaValue, states, chainID) + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txes_by_meta_field_and_states: %w", ErrInvalidChainID) + } + + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.Meta == nil { + return false + } + meta := map[string]interface{}{} + if err := json.Unmarshal(json.RawMessage(*tx.Meta), &meta); err != nil { + return false + } + if v, ok := meta[metaField].(string); ok { + return v == metaValue + } + + return false + } + txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + for _, tx := range as.FetchTxs(states, filterFn) { + txs = append(txs, &tx) + } + } + + return txs, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { return ms.txStore.FindTxesWithMetaFieldByStates(ctx, metaField, states, chainID) From bc54da303ba758665421ef9487ae7f5376d48d8d Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 18:54:35 -0500 Subject: [PATCH 31/74] implement FindTxsWithMetaFieldByStates --- common/txmgr/inmemory_store.go | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index f236480d6d2..c61a0d8111b 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -712,8 +712,35 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return txs, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByStates(ctx context.Context, metaField string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - return ms.txStore.FindTxesWithMetaFieldByStates(ctx, metaField, states, chainID) + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txes_with_meta_field_by_states: %w", ErrInvalidChainID) + } + + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.Meta == nil { + return false + } + meta := map[string]interface{}{} + if err := json.Unmarshal(json.RawMessage(*tx.Meta), &meta); err != nil { + return false + } + if _, ok := meta[metaField]; ok { + return true + } + + return false + } + + txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + for _, tx := range as.FetchTxs(states, filterFn) { + txs = append(txs, &tx) + } + } + + return txs, nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { return ms.txStore.FindTxesWithMetaFieldByReceiptBlockNum(ctx, metaField, blockNum, chainID) } From 7906c9ab5fa5768721c51c42a5a665cb22ab7174 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 18:59:00 -0500 Subject: [PATCH 32/74] implement FindTxesWithMetaFieldByReceiptBlockNum --- common/txmgr/inmemory_store.go | 38 +++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index c61a0d8111b..d2a0b333f26 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -742,7 +742,43 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithMetaFieldByReceiptBlockNum(ctx context.Context, metaField string, blockNum int64, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - return ms.txStore.FindTxesWithMetaFieldByReceiptBlockNum(ctx, metaField, blockNum, chainID) + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txes_with_meta_field_by_receipt_block_num: %w", ErrInvalidChainID) + } + + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.Meta == nil { + return false + } + meta := map[string]interface{}{} + if err := json.Unmarshal(json.RawMessage(*tx.Meta), &meta); err != nil { + return false + } + if _, ok := meta[metaField]; !ok { + return false + } + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + return false + } + if attempt.Receipts[0].GetBlockNumber() == nil { + return false + } + + return attempt.Receipts[0].GetBlockNumber().Int64() >= blockNum + } + + txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + for _, tx := range as.FetchTxs(nil, filterFn) { + txs = append(txs, &tx) + } + } + + return txs, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []big.Int, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { return ms.txStore.FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx, ids, states, chainID) From f092364a67cc9a669e046612f4675b9f7c7f30c5 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 19:05:28 -0500 Subject: [PATCH 33/74] implement FindTxesWithAttemptsAndReceiptsByIdsAndState --- common/txmgr/inmemory_store.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index d2a0b333f26..8d99293fdf6 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -780,9 +780,31 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return txs, nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx context.Context, ids []big.Int, states []txmgrtypes.TxState, chainID *big.Int) (tx []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxesWithAttemptsAndReceiptsByIdsAndState(ctx, ids, states, chainID) + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txes_with_attempts_and_receipts_by_ids_and_state: %w", ErrInvalidChainID) + } + + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return true + } + // convert ids to slice of int64 + txIDs := make([]int64, len(ids)) + for i, id := range ids { + txIDs[i] = id.Int64() + } + + txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + for _, tx := range as.FetchTxs(states, filterFn, txIDs...) { + txs = append(txs, &tx) + } + } + + return txs, nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (int64, error) { return ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) } From 74b97c170399fedec3e0dc5a0e64e5b09b9e7a06 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 18 Dec 2023 19:50:06 -0500 Subject: [PATCH 34/74] implement PruneUnstartedTxQueue --- common/txmgr/address_state.go | 7 +++++++ common/txmgr/inmemory_store.go | 28 ++++++++++++++++++++++++++-- common/txmgr/tx_priority_queue.go | 26 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index d6682459785..a4a89fe0efb 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -413,6 +413,13 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT return txs } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(queueSize uint32, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool) int { + as.RLock() + defer as.RUnlock() + + return len(as.unstarted.Prune(int(queueSize), filter)) +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.RLock() defer as.RUnlock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 8d99293fdf6..963408c2916 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -789,7 +789,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return true } - // convert ids to slice of int64 + txIDs := make([]int64, len(ids)) for i, id := range ids { txIDs[i] = id.Int64() @@ -806,7 +806,31 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (int64, error) { - return ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) + // Persist to persistent storage + n, err := ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) + if err != nil { + return 0, err + } + + // Update in memory store + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if !tx.Subject.Valid { + return false + } + + return tx.Subject.UUID == subject + } + var m int + for _, as := range ms.addressStates { + m += as.PruneUnstartedTxQueue(queueSize, filter) + } + + if n != int64(m) { + // TODO: WHAT SHOULD HAPPEN HERE IF THE COUNTS DON'T MATCH? + return n, fmt.Errorf("prune_unstarted_tx_queue: inmemory prune(%d) does not match persistence(%d) ", m, n) + } + + return n, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapTxHistory(ctx context.Context, minBlockNumberToKeep int64, timeThreshold time.Time, chainID CHAIN_ID) error { return ms.txStore.ReapTxHistory(ctx, minBlockNumberToKeep, timeThreshold, chainID) diff --git a/common/txmgr/tx_priority_queue.go b/common/txmgr/tx_priority_queue.go index 71db08a2dab..81dfcb30e16 100644 --- a/common/txmgr/tx_priority_queue.go +++ b/common/txmgr/tx_priority_queue.go @@ -87,6 +87,32 @@ func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Rem return nil } +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prune(maxUnstarted int, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + pq.Lock() + defer pq.Unlock() + + if len(pq.txs) <= maxUnstarted { + return nil + } + + // Remove all transactions that are oldest, unstarted, and match the filter + removed := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for i := 0; i < len(pq.txs); i++ { + tx := pq.txs[i] + if filter(tx) { + removed = append(removed, *tx) + } + if len(pq.txs)-len(removed) <= maxUnstarted { + break + } + } + for _, tx := range removed { + pq.RemoveTxByID(tx.ID) + } + + return removed +} + func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { pq.Lock() defer pq.Unlock() From af5442a9ff13afab52456bf2dbc11aaff25a4efa Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 19 Dec 2023 12:16:36 -0500 Subject: [PATCH 35/74] add fatalerrored --- common/txmgr/address_state.go | 121 ++++++++++++++++++++++----------- common/txmgr/inmemory_store.go | 105 ++++++++++++++++++++++------ 2 files changed, 168 insertions(+), 58 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index a4a89fe0efb..144346e9223 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -32,6 +32,7 @@ type AddressState[ confirmedMissingReceipt map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] confirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] allTransactions map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + fatalErrored map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } // NewAddressState returns a new AddressState instance @@ -180,39 +181,27 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close( clear(as.idempotencyKeyToTx) } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unstartedCount() int { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UnstartedCount() int { as.RLock() defer as.RUnlock() return as.unstarted.Len() } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) unconfirmedCount() int { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UnconfirmedCount() int { as.RLock() defer as.RUnlock() return len(as.unconfirmed) } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findTxWithIdempotencyKey(key string) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithIdempotencyKey(key string) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { as.RLock() defer as.RUnlock() return as.idempotencyKeyToTx[key] } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) confirmedMissingReceiptTxs() []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - as.RLock() - defer as.RUnlock() - - var txAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] - for _, tx := range as.confirmedMissingReceipt { - txAttempts = append(txAttempts, tx.TxAttempts...) - } - - return txAttempts -} - -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) findLatestSequence() SEQ { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence() SEQ { as.RLock() defer as.RUnlock() @@ -243,7 +232,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyT defer as.Unlock() // if txStates is empty then apply the filter to only the as.allTransactions map - if txStates == nil || len(txStates) == 0 { + if len(txStates) == 0 { as.applyToStorage(as.allTransactions, fn, txIDs...) return } @@ -260,6 +249,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyT as.applyToStorage(as.confirmedMissingReceipt, fn, txIDs...) case TxConfirmed: as.applyToStorage(as.confirmed, fn, txIDs...) + case TxFatalError: + as.applyToStorage(as.fatalErrored, fn, txIDs...) } } } @@ -298,7 +289,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT defer as.RUnlock() // if txStates is empty then apply the filter to only the as.allTransactions map - if txStates == nil || len(txStates) == 0 { + if len(txStates) == 0 { return as.fetchTxAttemptsFromStorage(as.allTransactions, filter, txIDs...) } @@ -315,6 +306,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmedMissingReceipt, filter, txIDs...)...) case TxConfirmed: txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmed, filter, txIDs...)...) + case TxFatalError: + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.fatalErrored, filter, txIDs...)...) } } @@ -360,7 +353,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT defer as.RUnlock() // if txStates is empty then apply the filter to only the as.allTransactions map - if txStates == nil || len(txStates) == 0 { + if len(txStates) == 0 { return as.fetchTxsFromStorage(as.allTransactions, filter, txIDs...) } @@ -377,6 +370,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT txs = append(txs, as.fetchTxsFromStorage(as.confirmedMissingReceipt, filter, txIDs...)...) case TxConfirmed: txs = append(txs, as.fetchTxsFromStorage(as.confirmed, filter, txIDs...)...) + case TxFatalError: + txs = append(txs, as.fetchTxsFromStorage(as.fatalErrored, filter, txIDs...)...) } } @@ -414,13 +409,41 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(queueSize uint32, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool) int { - as.RLock() - defer as.RUnlock() + as.Lock() + defer as.Unlock() + + txs := as.unstarted.Prune(int(queueSize), filter) + as.deleteTxs(txs...) + + return len(txs) +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) DeleteTxs(txs ...txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + as.Lock() + defer as.Unlock() - return len(as.unstarted.Prune(int(queueSize), filter)) + as.deleteTxs(txs...) } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deleteTxs(txs ...txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + for _, tx := range txs { + if tx.IdempotencyKey != nil { + delete(as.idempotencyKeyToTx, *tx.IdempotencyKey) + } + txID := tx.ID + if as.inprogress != nil && as.inprogress.ID == txID { + as.inprogress = nil + } + delete(as.allTransactions, txID) + delete(as.unconfirmed, txID) + delete(as.confirmedMissingReceipt, txID) + delete(as.allTransactions, txID) + delete(as.confirmed, txID) + delete(as.fatalErrored, txID) + } +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekNextUnstartedTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.RLock() defer as.RUnlock() @@ -432,7 +455,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekNe return tx, nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekInProgressTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekInProgressTx() (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { as.RLock() defer as.RUnlock() @@ -444,7 +467,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) peekIn return tx, nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) addTxToUnstarted(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTxToUnstarted(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { as.Lock() defer as.Unlock() @@ -461,7 +484,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) addTxT return nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveUnstartedToInProgress(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnstartedToInProgress(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { as.Lock() defer as.Unlock() @@ -486,7 +509,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveUn return nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveInProgressToUnconfirmed( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToUnconfirmed( txAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) error { as.Lock() @@ -519,37 +542,57 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) moveIn return nil } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToFatalError(txError null.String) error { + as.Lock() + defer as.Unlock() + + tx := as.inprogress + if tx == nil { + return fmt.Errorf("move_in_progress_to_fatal_error: no transaction in progress") + } + + tx.State = TxFatalError + tx.Sequence = nil + tx.TxAttempts = nil + tx.InitialBroadcastAt = nil + tx.Error = txError + as.fatalErrored[tx.ID] = tx + as.inprogress = nil + + return nil +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandon() { as.Lock() defer as.Unlock() for as.unstarted.Len() > 0 { tx := as.unstarted.RemoveNextTx() - abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) + as.abandonTx(tx) } if as.inprogress != nil { tx := as.inprogress - abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) + as.abandonTx(tx) as.inprogress = nil } for _, tx := range as.unconfirmed { - abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) + as.abandonTx(tx) } for _, tx := range as.idempotencyKeyToTx { - abandon[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](tx) + as.abandonTx(tx) + } + for _, tx := range as.confirmedMissingReceipt { + as.abandonTx(tx) + } + for _, tx := range as.confirmed { + as.abandonTx(tx) } clear(as.unconfirmed) } -func abandon[ - CHAIN_ID types.ID, - ADDR, TX_HASH, BLOCK_HASH types.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], - SEQ types.Sequence, - FEE feetypes.Fee, -](tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandonTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { if tx == nil { return } @@ -557,4 +600,6 @@ func abandon[ tx.State = TxFatalError tx.Sequence = nil tx.Error = null.NewString("abandoned", true) + + as.fatalErrored[tx.ID] = tx } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 963408c2916..e180de1a2b6 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -118,7 +118,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat // TODO(jtw); HANDLE PRUNING STEP // Add the request to the Unstarted channel to be processed by the Broadcaster - if err := as.addTxToUnstarted(&tx); err != nil { + if err := as.AddTxToUnstarted(&tx); err != nil { return tx, fmt.Errorf("create_transaction: %w", err) } @@ -135,7 +135,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT ms.addressStatesLock.Lock() defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { - if tx := as.findTxWithIdempotencyKey(idempotencyKey); tx != nil { + if tx := as.FindTxWithIdempotencyKey(idempotencyKey); tx != nil { return ms.deepCopyTx(tx), nil } } @@ -157,7 +157,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check return fmt.Errorf("check_tx_queue_capacity: %w", ErrAddressNotFound) } - count := uint64(as.unstartedCount()) + count := uint64(as.UnstartedCount()) if count >= maxQueuedTransactions { return fmt.Errorf("check_tx_queue_capacity: cannot create transaction; too many unstarted transactions in the queue (%v/%v). %s", count, maxQueuedTransactions, label.MaxQueuedTransactionsWarning) } @@ -183,7 +183,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) } - seq = as.findLatestSequence() + seq = as.FindLatestSequence() if seq.Int64() == 0 { return seq, fmt.Errorf("find_latest_sequence: %w", ErrSequenceNotFound) } @@ -203,7 +203,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - return uint32(as.unconfirmedCount()), nil + return uint32(as.UnconfirmedCount()), nil } // CountUnstartedTransactions returns the number of unstarted transactions for a given address. @@ -218,7 +218,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - return uint32(as.unstartedCount()), nil + return uint32(as.UnstartedCount()), nil } // UpdateTxUnstartedToInProgress updates a transaction from unstarted to in_progress. @@ -248,7 +248,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat tx.TxAttempts = append(tx.TxAttempts, *attempt) // Update in address state in memory - if err := as.moveUnstartedToInProgress(tx); err != nil { + if err := as.MoveUnstartedToInProgress(tx); err != nil { return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) } @@ -262,7 +262,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx return nil, fmt.Errorf("get_tx_in_progress: %w", ErrAddressNotFound) } - tx, err := as.peekInProgressTx() + tx, err := as.PeekInProgressTx() if tx == nil { return nil, fmt.Errorf("get_tx_in_progress: %w", err) } @@ -311,7 +311,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat if !ok { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", ErrAddressNotFound) } - if err := as.moveInProgressToUnconfirmed(attempt); err != nil { + if err := as.MoveInProgressToUnconfirmed(attempt); err != nil { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } @@ -335,7 +335,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN } var err error - tx, err = as.peekNextUnstartedTx() + tx, err = as.PeekNextUnstartedTx() if tx == nil { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", err) } @@ -356,17 +356,18 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR return fmt.Errorf("save_replacement_in_progress_attempt: expected oldattempt to have an ID") } + // Check if fromaddress enabled + as, ok := ms.addressStates[oldAttempt.Tx.FromAddress] + if !ok { + return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) + } + // Persist to persistent storage if err := ms.txStore.SaveReplacementInProgressAttempt(ctx, oldAttempt, replacementAttempt); err != nil { return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) } - // Update in memory store - as, ok := ms.addressStates[oldAttempt.Tx.FromAddress] - if !ok { - return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) - } - tx, err := as.peekInProgressTx() + tx, err := as.PeekInProgressTx() if tx == nil { return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) } @@ -393,15 +394,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat if !tx.Error.Valid { return fmt.Errorf("update_tx_fatal_error: expected error field to be set") } + // Check if fromaddress enabled + as, ok := ms.addressStates[tx.FromAddress] + if !ok { + return fmt.Errorf("update_tx_fatal_error: %w", ErrAddressNotFound) + } // Persist to persistent storage if err := ms.txStore.UpdateTxFatalError(ctx, tx); err != nil { return fmt.Errorf("update_tx_fatal_error: %w", err) } - // Ensure that the tx state is updated to fatal_error since this is a chain agnostic operation - tx.Sequence = nil - tx.State = TxFatalError + // Update in memory store + if err := as.MoveInProgressToFatalError(tx.Error); err != nil { + return fmt.Errorf("update_tx_fatal_error: %w", err) + } return fmt.Errorf("update_tx_fatal_error: not implemented") } @@ -832,8 +839,66 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prune return n, nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapTxHistory(ctx context.Context, minBlockNumberToKeep int64, timeThreshold time.Time, chainID CHAIN_ID) error { - return ms.txStore.ReapTxHistory(ctx, minBlockNumberToKeep, timeThreshold, chainID) + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("reap_tx_history: %w", ErrInvalidChainID) + } + + // Persist to persistent storage + if err := ms.txStore.ReapTxHistory(ctx, minBlockNumberToKeep, timeThreshold, chainID); err != nil { + return err + } + + // Update in memory store + states := []txmgrtypes.TxState{TxConfirmed} + filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + return false + } + if attempt.Receipts[0].GetBlockNumber() == nil { + return false + } + if attempt.Receipts[0].GetBlockNumber().Int64() >= minBlockNumberToKeep { + return false + } + if tx.CreatedAt.After(timeThreshold) { + return false + } + if tx.State != TxConfirmed { + return false + } + return true + } + + wg := sync.WaitGroup{} + for _, as := range ms.addressStates { + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + as.DeleteTxs(as.FetchTxs(states, filterFn)...) + wg.Done() + }(as) + } + wg.Wait() + + filterFn = func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.State == TxFatalError && tx.CreatedAt.Before(timeThreshold) + } + states = []txmgrtypes.TxState{TxFatalError} + for _, as := range ms.addressStates { + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + as.DeleteTxs(as.FetchTxs(states, filterFn)...) + wg.Done() + }(as) + } + wg.Wait() + + return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState, chainID CHAIN_ID) (uint32, error) { return ms.txStore.CountTransactionsByState(ctx, state, chainID) From 12209c07688ea04b5415654f6bb90ae3539e3919 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 19 Dec 2023 14:29:57 -0500 Subject: [PATCH 36/74] implement DeleteInProgressAttempt --- common/txmgr/address_state.go | 25 ++++++++++++----- common/txmgr/inmemory_store.go | 51 ++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 144346e9223..d2e206dd264 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -181,17 +181,28 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close( clear(as.idempotencyKeyToTx) } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UnstartedCount() int { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(txState txmgrtypes.TxState) int { as.RLock() defer as.RUnlock() - return as.unstarted.Len() -} -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UnconfirmedCount() int { - as.RLock() - defer as.RUnlock() + switch txState { + case TxUnstarted: + return as.unstarted.Len() + case TxInProgress: + if as.inprogress != nil { + return 1 + } + case TxUnconfirmed: + return len(as.unconfirmed) + case TxConfirmedMissingReceipt: + return len(as.confirmedMissingReceipt) + case TxConfirmed: + return len(as.confirmed) + case TxFatalError: + return len(as.fatalErrored) + } - return len(as.unconfirmed) + return 0 } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithIdempotencyKey(key string) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index e180de1a2b6..1b98a13a1ab 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -157,7 +157,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check return fmt.Errorf("check_tx_queue_capacity: %w", ErrAddressNotFound) } - count := uint64(as.UnstartedCount()) + count := uint64(as.CountTransactionsByState(TxUnstarted)) if count >= maxQueuedTransactions { return fmt.Errorf("check_tx_queue_capacity: cannot create transaction; too many unstarted transactions in the queue (%v/%v). %s", count, maxQueuedTransactions, label.MaxQueuedTransactionsWarning) } @@ -203,7 +203,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - return uint32(as.UnconfirmedCount()), nil + return uint32(as.CountTransactionsByState(TxUnconfirmed)), nil } // CountUnstartedTransactions returns the number of unstarted transactions for a given address. @@ -218,7 +218,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) } - return uint32(as.UnstartedCount()), nil + return uint32(as.CountTransactionsByState(TxUnstarted)), nil } // UpdateTxUnstartedToInProgress updates a transaction from unstarted to in_progress. @@ -900,12 +900,51 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapT return nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(ctx context.Context, state txmgrtypes.TxState, chainID CHAIN_ID) (uint32, error) { - return ms.txStore.CountTransactionsByState(ctx, state, chainID) +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(_ context.Context, state txmgrtypes.TxState, chainID CHAIN_ID) (uint32, error) { + if ms.chainID.String() != chainID.String() { + return 0, fmt.Errorf("count_transactions_by_state: %w", ErrInvalidChainID) + } + + var total int + for _, as := range ms.addressStates { + total += as.CountTransactionsByState(state) + } + + return uint32(total), nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) DeleteInProgressAttempt(ctx context.Context, attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - return ms.txStore.DeleteInProgressAttempt(ctx, attempt) + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("delete_in_progress_attempt: expected attempt to be in_progress") + } + if attempt.ID == 0 { + return fmt.Errorf("delete_in_progress_attempt: expected attempt to have an ID") + } + + // Check if fromaddress enabled + as, ok := ms.addressStates[attempt.Tx.FromAddress] + if !ok { + return fmt.Errorf("delete_in_progress_attempt: %w", ErrAddressNotFound) + } + + // Persist to persistent storage + if err := ms.txStore.DeleteInProgressAttempt(ctx, attempt); err != nil { + return fmt.Errorf("delete_in_progress_attempt: %w", err) + } + + // Update in memory store + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + + return tx.TxAttempts[0].ID == attempt.ID + } + as.DeleteTxs(as.FetchTxs(nil, filter)...) + + return nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { return ms.txStore.FindTxsRequiringGasBump(ctx, address, blockNum, gasBumpThreshold, depth, chainID) } From da30a83a946aa66c0f200de057e2afd9bfa0689b Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 19 Dec 2023 16:22:07 -0500 Subject: [PATCH 37/74] implement more methods --- common/txmgr/inmemory_store.go | 413 ++++++++++++++++++++++++-- core/chains/evm/txmgr/evm_tx_store.go | 51 +++- 2 files changed, 428 insertions(+), 36 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 1b98a13a1ab..d0b17f9a482 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "math/big" + "sort" "sync" "time" @@ -945,44 +946,314 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Delet return nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxsRequiringGasBump(ctx, address, blockNum, gasBumpThreshold, depth, chainID) -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringResubmissionDueToInsufficientFunds(ctx context.Context, address ADDR, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxsRequiringResubmissionDueToInsufficientFunds(ctx, address, chainID) +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringResubmissionDueToInsufficientFunds(_ context.Context, address ADDR, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txs_requiring_resubmission_due_to_insufficient_funds: %w", ErrInvalidChainID) + } + + as, ok := ms.addressStates[address] + if !ok { + return nil, fmt.Errorf("find_txs_requiring_resubmission_due_to_insufficient_funds: %w", ErrAddressNotFound) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + + return attempt.State == txmgrtypes.TxAttemptInsufficientFunds + } + states := []txmgrtypes.TxState{TxUnconfirmed} + txs := as.FetchTxs(states, filter) + // sort by sequence ASC + sort.Slice(txs, func(i, j int) bool { + return (*txs[i].Sequence).Int64() < (*txs[j].Sequence).Int64() + }) + + etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) + for i, tx := range txs { + etxs[i] = &tx + } + + return etxs, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringResend(ctx context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxAttemptsRequiringResend(ctx, olderThan, maxInFlightTransactions, chainID, address) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringResend(_ context.Context, olderThan time.Time, maxInFlightTransactions uint32, chainID CHAIN_ID, address ADDR) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrInvalidChainID) + } + + as, ok := ms.addressStates[address] + if !ok { + return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrAddressNotFound) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + if attempt.State == txmgrtypes.TxAttemptInProgress { + return false + } + if tx.BroadcastAt.After(olderThan) { + return false + } + + return false + } + states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} + attempts := as.FetchTxAttempts(states, filter) + // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC + // TODO + + // LIMIT by maxInFlightTransactions + if len(attempts) > int(maxInFlightTransactions) { + attempts = attempts[:maxInFlightTransactions] + } + + return attempts, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithSequence(ctx context.Context, fromAddress ADDR, seq SEQ) (etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTxWithSequence(ctx, fromAddress, seq) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithSequence(_ context.Context, fromAddress ADDR, seq SEQ) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + as, ok := ms.addressStates[fromAddress] + if !ok { + return nil, fmt.Errorf("find_tx_with_sequence: %w", ErrAddressNotFound) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.Sequence == nil { + return false + } + + return (*tx.Sequence).String() == seq.String() + } + states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt, TxUnconfirmed} + txs := as.FetchTxs(states, filter) + if len(txs) == 0 { + return nil, nil + } + + return &txs[0], nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTransactionsConfirmedInBlockRange(ctx context.Context, highBlockNumber, lowBlockNumber int64, chainID CHAIN_ID) (etxs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.FindTransactionsConfirmedInBlockRange(ctx, highBlockNumber, lowBlockNumber, chainID) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTransactionsConfirmedInBlockRange(_ context.Context, highBlockNumber, lowBlockNumber int64, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_transactions_confirmed_in_block_range: %w", ErrInvalidChainID) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + if attempt.State != txmgrtypes.TxAttemptBroadcast { + return false + } + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + return false + } + if attempt.Receipts[0].GetBlockNumber() == nil { + return false + } + blockNum := attempt.Receipts[0].GetBlockNumber().Int64() + return blockNum >= lowBlockNumber && blockNum <= highBlockNumber + } + states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt} + txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + txs = append(txs, as.FetchTxs(states, filter)...) + } + // sort by sequence ASC + sort.Slice(txs, func(i, j int) bool { + return (*txs[i].Sequence).Int64() < (*txs[j].Sequence).Int64() + }) + + etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) + for i, tx := range txs { + etxs[i] = &tx + } + + return etxs, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindEarliestUnconfirmedBroadcastTime(ctx context.Context, chainID CHAIN_ID) (null.Time, error) { - return ms.txStore.FindEarliestUnconfirmedBroadcastTime(ctx, chainID) + if ms.chainID.String() != chainID.String() { + return null.Time{}, fmt.Errorf("find_earliest_unconfirmed_broadcast_time: %w", ErrInvalidChainID) + } + + // TODO(jtw): this is super niave and might be slow + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.InitialBroadcastAt != nil + } + states := []txmgrtypes.TxState{TxUnconfirmed} + txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + txs = append(txs, as.FetchTxs(states, filter)...) + } + + var minInitialBroadcastAt time.Time + for _, tx := range txs { + if tx.InitialBroadcastAt.Before(minInitialBroadcastAt) { + minInitialBroadcastAt = *tx.InitialBroadcastAt + } + } + + return null.TimeFrom(minInitialBroadcastAt), nil } + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindEarliestUnconfirmedTxAttemptBlock(ctx context.Context, chainID CHAIN_ID) (null.Int, error) { - return ms.txStore.FindEarliestUnconfirmedTxAttemptBlock(ctx, chainID) + if ms.chainID.String() != chainID.String() { + return null.Int{}, fmt.Errorf("find_earliest_unconfirmed_broadcast_time: %w", ErrInvalidChainID) + } + + // TODO(jtw): this is super niave and might be slow + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + return attempt.BroadcastBeforeBlockNum != nil + } + states := []txmgrtypes.TxState{TxUnconfirmed} + txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + txs = append(txs, as.FetchTxs(states, filter)...) + } + + var minBroadcastBeforeBlockNum int64 + for _, tx := range txs { + if *tx.TxAttempts[0].BroadcastBeforeBlockNum < minBroadcastBeforeBlockNum { + minBroadcastBeforeBlockNum = *tx.TxAttempts[0].BroadcastBeforeBlockNum + } + } + + return null.IntFrom(minBroadcastBeforeBlockNum), nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetInProgressTxAttempts(ctx context.Context, address ADDR, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.GetInProgressTxAttempts(ctx, address, chainID) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetInProgressTxAttempts(ctx context.Context, address ADDR, chainID CHAIN_ID) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrInvalidChainID) + } + + as, ok := ms.addressStates[address] + if !ok { + return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrAddressNotFound) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + return attempt.State == txmgrtypes.TxAttemptInProgress + } + states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt, TxUnconfirmed} + attempts := as.FetchTxAttempts(states, filter) + + return attempts, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNonFatalTransactions(ctx context.Context, chainID CHAIN_ID) (txs []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.GetNonFatalTransactions(ctx, chainID) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNonFatalTransactions(ctx context.Context, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("get_non_fatal_transactions: %w", ErrInvalidChainID) + } + + // TODO(jtw): this is niave ... it might be better to just use all states excluding fatal + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.State != TxFatalError + } + txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, as := range ms.addressStates { + txs = append(txs, as.FetchTxs(nil, filter)...) + } + + etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) + for i, tx := range txs { + etxs[i] = &tx + } + + return etxs, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxByID(ctx context.Context, id int64) (tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { - return ms.txStore.GetTxByID(ctx, id) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxByID(_ context.Context, id int64) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.ID == id + } + for _, as := range ms.addressStates { + txs := as.FetchTxs(nil, filter, id) + if len(txs) > 0 { + return &txs[0], nil + } + } + + return nil, fmt.Errorf("failed to get tx with id: %v", id) + } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasInProgressTransaction(ctx context.Context, account ADDR, chainID CHAIN_ID) (exists bool, err error) { - return ms.txStore.HasInProgressTransaction(ctx, account, chainID) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasInProgressTransaction(_ context.Context, account ADDR, chainID CHAIN_ID) (bool, error) { + if ms.chainID.String() != chainID.String() { + return false, fmt.Errorf("has_in_progress_transaction: %w", ErrInvalidChainID) + } + + as, ok := ms.addressStates[account] + if !ok { + return false, fmt.Errorf("has_in_progress_transaction: %w", ErrAddressNotFound) + } + + n := as.CountTransactionsByState(TxInProgress) + + return n > 0, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadTxAttempts(ctx context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - return ms.txStore.LoadTxAttempts(ctx, etx) + +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadTxAttempts(_ context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + as, ok := ms.addressStates[etx.FromAddress] + if !ok { + return fmt.Errorf("load_tx_attempts: %w", ErrAddressNotFound) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.ID == etx.ID + } + txAttempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + for _, tx := range as.FetchTxs(nil, filter, etx.ID) { + txAttempts = append(txAttempts, tx.TxAttempts...) + } + etx.TxAttempts = txAttempts + + return nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) (err error) { - return ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID) +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) error { + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("mark_all_confirmed_missing_receipt: %w", ErrInvalidChainID) + } + + // TODO(jtw): need to complete + + /* + // Persist to persistent storage + if err := ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.State != TxUnconfirmed { + return + } + if tx.Sequence >= maxSequence { + return + } + + tx.State = TxConfirmedMissingReceipt + } + states := []txmgrtypes.TxState{TxUnconfirmed} + for _, as := range ms.addressStates { + as.ApplyToTxs(states, fn) + } + */ + + return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { return ms.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, finalityDepth, chainID) @@ -994,21 +1265,105 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveC return ms.txStore.SaveConfirmedMissingReceiptAttempt(ctx, timeout, attempt, broadcastAt) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInProgressAttempt(ctx context.Context, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - return ms.txStore.SaveInProgressAttempt(ctx, attempt) + _, ok := ms.addressStates[attempt.Tx.FromAddress] + if !ok { + return fmt.Errorf("save_in_progress_attempt: %w", ErrAddressNotFound) + } + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("SaveInProgressAttempt failed: attempt state must be in_progress") + } + + // Persist to persistent storage + if err := ms.txStore.SaveInProgressAttempt(ctx, attempt); err != nil { + return err + } + + // Update in memory store + // TODO + + return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInsufficientFundsAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { - return ms.txStore.SaveInsufficientFundsAttempt(ctx, timeout, attempt, broadcastAt) + as, ok := ms.addressStates[attempt.Tx.FromAddress] + if !ok { + return fmt.Errorf("save_insufficient_funds_attempt: %w", ErrAddressNotFound) + } + if !(attempt.State == txmgrtypes.TxAttemptInProgress || attempt.State == txmgrtypes.TxAttemptInsufficientFunds) { + return fmt.Errorf("expected state to be in_progress or insufficient_funds") + } + + // Persist to persistent storage + if err := ms.txStore.SaveInsufficientFundsAttempt(ctx, timeout, attempt, broadcastAt); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.ID != attempt.TxID { + return + } + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return + } + if tx.BroadcastAt.Before(broadcastAt) { + tx.BroadcastAt = &broadcastAt + } + + tx.TxAttempts[0].State = txmgrtypes.TxAttemptInsufficientFunds + } + as.ApplyToTxs(nil, fn, attempt.TxID) + + return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveSentAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { - return ms.txStore.SaveSentAttempt(ctx, timeout, attempt, broadcastAt) + as, ok := ms.addressStates[attempt.Tx.FromAddress] + if !ok { + return fmt.Errorf("save_sent_attempt: %w", ErrAddressNotFound) + } + + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("expected state to be in_progress") + } + + // Persist to persistent storage + if err := ms.txStore.SaveSentAttempt(ctx, timeout, attempt, broadcastAt); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.ID != attempt.TxID { + return + } + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return + } + if tx.BroadcastAt.Before(broadcastAt) { + tx.BroadcastAt = &broadcastAt + } + + tx.TxAttempts[0].State = txmgrtypes.TxAttemptBroadcast + } + as.ApplyToTxs(nil, fn, attempt.TxID) + + return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { return ms.txStore.UpdateTxForRebroadcast(ctx, etx, etxAttempt) } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxFinalized(ctx context.Context, blockHeight int64, txID int64, chainID CHAIN_ID) (finalized bool, err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxFinalized(ctx context.Context, blockHeight int64, txID int64, chainID CHAIN_ID) (bool, error) { return ms.txStore.IsTxFinalized(ctx, blockHeight, txID, chainID) } +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + if ms.chainID.String() != chainID.String() { + return nil, fmt.Errorf("find_txs_requiring_gas_bump: %w", ErrInvalidChainID) + } + + // TODO + return nil, nil +} + func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { etx := *tx etx.TxAttempts = make([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(tx.TxAttempts)) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 452b31349db..152834f3a5b 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -698,7 +698,11 @@ func (o *evmTxStore) LoadTxesAttempts(etxs []*Tx, qopts ...pg.QOpt) error { ethTxesM[etx.ID] = etxs[i] } var dbTxAttempts []DbEthTxAttempt - if err := qq.Select(&dbTxAttempts, `SELECT * FROM evm.tx_attempts WHERE eth_tx_id = ANY($1) ORDER BY evm.tx_attempts.gas_price DESC, evm.tx_attempts.gas_tip_cap DESC`, pq.Array(ethTxIDs)); err != nil { + if err := qq.Select(&dbTxAttempts, ` + SELECT * + FROM evm.tx_attempts + WHERE eth_tx_id = ANY($1) + ORDER BY evm.tx_attempts.gas_price DESC, evm.tx_attempts.gas_tip_cap DESC`, pq.Array(ethTxIDs)); err != nil { return pkgerrors.Wrap(err, "loadEthTxesAttempts failed to load evm.tx_attempts") } for _, dbAttempt := range dbTxAttempts { @@ -1037,7 +1041,8 @@ func (o *evmTxStore) GetInProgressTxAttempts(ctx context.Context, address common var dbAttempts []DbEthTxAttempt err = tx.Select(&dbAttempts, ` SELECT evm.tx_attempts.* FROM evm.tx_attempts -INNER JOIN evm.txes ON evm.txes.id = evm.tx_attempts.eth_tx_id AND evm.txes.state in ('confirmed', 'confirmed_missing_receipt', 'unconfirmed') +INNER JOIN evm.txes ON evm.txes.id = evm.tx_attempts.eth_tx_id + AND evm.txes.state in ('confirmed', 'confirmed_missing_receipt', 'unconfirmed') WHERE evm.tx_attempts.state = 'in_progress' AND evm.txes.from_address = $1 AND evm.txes.evm_chain_id = $2 `, address, chainID.String()) if err != nil { @@ -1214,7 +1219,11 @@ func (o *evmTxStore) FindEarliestUnconfirmedBroadcastTime(ctx context.Context, c defer cancel() qq := o.q.WithOpts(pg.WithParentCtx(ctx)) err = qq.Transaction(func(tx pg.Queryer) error { - if err = qq.QueryRowContext(ctx, `SELECT min(initial_broadcast_at) FROM evm.txes WHERE state = 'unconfirmed' AND evm_chain_id = $1`, chainID.String()).Scan(&broadcastAt); err != nil { + if err = qq.QueryRowContext(ctx, ` + SELECT + min(initial_broadcast_at) + FROM evm.txes + WHERE state = 'unconfirmed' AND evm_chain_id = $1`, chainID.String()).Scan(&broadcastAt); err != nil { return fmt.Errorf("failed to query for unconfirmed eth_tx count: %w", err) } return nil @@ -1443,9 +1452,30 @@ func (o *evmTxStore) FindTxsRequiringGasBump(ctx context.Context, address common err = qq.Transaction(func(tx pg.Queryer) error { stmt := ` SELECT evm.txes.* FROM evm.txes -LEFT JOIN evm.tx_attempts ON evm.txes.id = evm.tx_attempts.eth_tx_id AND (broadcast_before_block_num > $4 OR broadcast_before_block_num IS NULL OR evm.tx_attempts.state != 'broadcast') -WHERE evm.txes.state = 'unconfirmed' AND evm.tx_attempts.id IS NULL AND evm.txes.from_address = $1 AND evm.txes.evm_chain_id = $2 - AND (($3 = 0) OR (evm.txes.id IN (SELECT id FROM evm.txes WHERE state = 'unconfirmed' AND from_address = $1 ORDER BY nonce ASC LIMIT $3))) +LEFT JOIN evm.tx_attempts ON evm.txes.id = evm.tx_attempts.eth_tx_id + AND ( + broadcast_before_block_num > $4 + OR broadcast_before_block_num IS NULL + OR evm.tx_attempts.state != 'broadcast' + ) +WHERE + evm.txes.state = 'unconfirmed' + AND evm.tx_attempts.id IS NULL + AND evm.txes.from_address = $1 + AND evm.txes.evm_chain_id = $2 + AND ( + ($3 = 0) + OR ( + evm.txes.id IN ( + SELECT id + FROM evm.txes + WHERE + state = 'unconfirmed' + AND from_address = $1 + ORDER BY nonce ASC LIMIT $3 + ) + ) + ) ORDER BY nonce ASC ` var dbEtxs []DbEthTx @@ -1804,7 +1834,14 @@ func (o *evmTxStore) HasInProgressTransaction(ctx context.Context, account commo ctx, cancel = o.mergeContexts(ctx) defer cancel() qq := o.q.WithOpts(pg.WithParentCtx(ctx)) - err = qq.Get(&exists, `SELECT EXISTS(SELECT 1 FROM evm.txes WHERE state = 'in_progress' AND from_address = $1 AND evm_chain_id = $2)`, account, chainID.String()) + err = qq.Get(&exists, ` + SELECT EXISTS( + SELECT 1 + FROM evm.txes + WHERE + state = 'in_progress' + AND from_address = $1 + AND evm_chain_id = $2)`, account, chainID.String()) return exists, pkgerrors.Wrap(err, "hasInProgressTransaction failed") } From bffbdfaab6daf6a663e01c6e660952e52eac5e91 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 19 Dec 2023 21:41:50 -0500 Subject: [PATCH 38/74] implement more methods --- common/txmgr/inmemory_store.go | 183 ++++++++++++++++++++++++++------- 1 file changed, 148 insertions(+), 35 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index d0b17f9a482..e97dc8ddc4e 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -17,6 +17,10 @@ import ( "gopkg.in/guregu/null.v4" ) +// BIG TODO LIST +// TODO: make sure that all state transitions are handled by the address state to ensure that the in-memory store is always in a consistent state +// TODO: figure out if multiple tx attempts are actually stored in the db for each tx + var ( // ErrInvalidChainID is returned when the chain ID is invalid ErrInvalidChainID = fmt.Errorf("invalid chain ID") @@ -1223,49 +1227,69 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadT return nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) error { - if ms.chainID.String() != chainID.String() { - return fmt.Errorf("mark_all_confirmed_missing_receipt: %w", ErrInvalidChainID) +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PreloadTxes(_ context.Context, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + if len(attempts) == 0 { + return nil } - // TODO(jtw): need to complete + as, ok := ms.addressStates[attempts[0].Tx.FromAddress] + if !ok { + return fmt.Errorf("preload_txes: %w", ErrAddressNotFound) + } - /* - // Persist to persistent storage - if err := ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID); err != nil { - return err + txIDs := make([]int64, len(attempts)) + for i, attempt := range attempts { + txIDs[i] = attempt.TxID + } + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return true + } + txs := as.FetchTxs(nil, filter, txIDs...) + for i, attempt := range attempts { + for _, tx := range txs { + if tx.ID == attempt.TxID { + attempts[i].Tx = tx + } } + } - // Update in memory store - fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - if tx.State != TxUnconfirmed { - return - } - if tx.Sequence >= maxSequence { - return - } + return nil +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + as, ok := ms.addressStates[attempt.Tx.FromAddress] + if !ok { + return fmt.Errorf("save_confirmed_missing_receipt_attempt: %w", ErrAddressNotFound) + } + if attempt.State != txmgrtypes.TxAttemptInProgress { + return fmt.Errorf("expected state to be in_progress") + } - tx.State = TxConfirmedMissingReceipt + // Persist to persistent storage + if err := ms.txStore.SaveConfirmedMissingReceiptAttempt(ctx, timeout, attempt, broadcastAt); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.ID != attempt.TxID { + return } - states := []txmgrtypes.TxState{TxUnconfirmed} - for _, as := range ms.addressStates { - as.ApplyToTxs(states, fn) + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return } - */ + if tx.BroadcastAt.Before(broadcastAt) { + tx.BroadcastAt = &broadcastAt + } + + tx.TxAttempts[0].State = txmgrtypes.TxAttemptBroadcast + tx.State = TxConfirmedMissingReceipt + } + as.ApplyToTxs(nil, fn, attempt.TxID) return nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { - return ms.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, finalityDepth, chainID) -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PreloadTxes(ctx context.Context, attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - return ms.txStore.PreloadTxes(ctx, attempts) -} -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { - return ms.txStore.SaveConfirmedMissingReceiptAttempt(ctx, timeout, attempt, broadcastAt) -} func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInProgressAttempt(ctx context.Context, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - _, ok := ms.addressStates[attempt.Tx.FromAddress] + as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_in_progress_attempt: %w", ErrAddressNotFound) } @@ -1278,8 +1302,16 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI return err } - // Update in memory store - // TODO + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.ID != attempt.TxID { + return + } + if tx.TxAttempts == nil { + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + } + tx.TxAttempts = append(tx.TxAttempts, *attempt) + } + as.ApplyToTxs(nil, fn, attempt.TxID) return nil } @@ -1349,10 +1381,55 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveS return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - return ms.txStore.UpdateTxForRebroadcast(ctx, etx, etxAttempt) + as, ok := ms.addressStates[etx.FromAddress] + if !ok { + return fmt.Errorf("update_tx_for_rebroadcast: %w", ErrAddressNotFound) + } + + // Persist to persistent storage + if err := ms.txStore.UpdateTxForRebroadcast(ctx, etx, etxAttempt); err != nil { + return err + } + + // Update in memory store + + // TODO + // delete receipts + // update tx unconfirmed + // update tx_attempt unbroadcast + + return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxFinalized(ctx context.Context, blockHeight int64, txID int64, chainID CHAIN_ID) (bool, error) { - return ms.txStore.IsTxFinalized(ctx, blockHeight, txID, chainID) + if ms.chainID.String() != chainID.String() { + return false, fmt.Errorf("is_tx_finalized: %w", ErrInvalidChainID) + } + + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.ID != txID { + return false + } + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + return false + } + if attempt.Receipts[0].GetBlockNumber() == nil { + return false + } + + return attempt.Receipts[0].GetBlockNumber().Int64() <= (blockHeight - int64(tx.MinConfirmations.Uint32)) + } + for _, as := range ms.addressStates { + txas := as.FetchTxAttempts(nil, fn, txID) + if len(txas) > 0 { + return true, nil + } + } + + return false, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxsRequiringGasBump(ctx context.Context, address ADDR, blockNum, gasBumpThreshold, depth int64, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { @@ -1363,6 +1440,42 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT // TODO return nil, nil } +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) error { + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("mark_all_confirmed_missing_receipt: %w", ErrInvalidChainID) + } + + // TODO(jtw): need to complete + + /* + // Persist to persistent storage + if err := ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID); err != nil { + return err + } + + // Update in memory store + fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { + if tx.State != TxUnconfirmed { + return + } + if tx.Sequence >= maxSequence { + return + } + + tx.State = TxConfirmedMissingReceipt + } + states := []txmgrtypes.TxState{TxUnconfirmed} + for _, as := range ms.addressStates { + as.ApplyToTxs(states, fn) + } + */ + + return nil +} +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { + // TODO + return ms.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, finalityDepth, chainID) +} func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { etx := *tx From b7b8f3df3955432d6eb97ebd6aa3515dc032090b Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 20 Dec 2023 22:10:05 -0500 Subject: [PATCH 39/74] cleanup --- common/txmgr/address_state.go | 109 ++++++++++++++++++------ common/txmgr/inmemory_store.go | 149 ++++++++++++++++----------------- 2 files changed, 155 insertions(+), 103 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index d2e206dd264..f83098b7bbf 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -217,12 +217,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLa defer as.RUnlock() var maxSeq SEQ - if as.inprogress != nil && as.inprogress.Sequence != nil { - if (*as.inprogress.Sequence).Int64() > maxSeq.Int64() { - maxSeq = *as.inprogress.Sequence - } - } - for _, tx := range as.unconfirmed { + for _, tx := range as.allTransactions { if tx.Sequence == nil { continue } @@ -271,9 +266,6 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyT fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), txIDs ...int64, ) { - as.Lock() - defer as.Unlock() - // if txIDs is not empty then only apply the filter to those transactions if len(txIDs) > 0 { for _, txID := range txIDs { @@ -495,7 +487,9 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTxT return nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnstartedToInProgress(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnstartedToInProgress( + tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) error { as.Lock() defer as.Unlock() @@ -505,7 +499,6 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUn if tx != nil { // if tx is not nil then remove the tx from the unstarted queue - // TODO(jtw): what should be the unique idenitifier for each transaction? ID is being set by the postgres DB tx = as.unstarted.RemoveTxByID(tx.ID) } else { // if tx is nil then pop the next unstarted transaction @@ -520,6 +513,24 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUn return nil } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveConfirmedMissingReceiptToUnconfirmed( + txID int64, +) error { + as.Lock() + defer as.Unlock() + + tx, ok := as.confirmedMissingReceipt[txID] + if !ok || tx == nil { + return fmt.Errorf("move_confirmed_missing_receipt_to_unconfirmed: no confirmed_missing_receipt transaction with ID %d: %w", txID, ErrTxnNotFound) + } + + tx.State = TxUnconfirmed + as.unconfirmed[tx.ID] = tx + delete(as.confirmedMissingReceipt, tx.ID) + + return nil +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToUnconfirmed( txAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) error { @@ -530,25 +541,66 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveIn if tx == nil { return fmt.Errorf("move_in_progress_to_unconfirmed: no transaction in progress") } + + txAttempt.TxID = tx.ID + txAttempt.State = txmgrtypes.TxAttemptBroadcast tx.State = TxUnconfirmed + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{txAttempt} - var found bool - for i := 0; i < len(tx.TxAttempts); i++ { - if tx.TxAttempts[i].ID == txAttempt.ID { - tx.TxAttempts[i] = txAttempt - found = true - break + as.unconfirmed[tx.ID] = tx + as.inprogress = nil + + return nil +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnconfirmedToConfirmed( + receipt txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], +) error { + as.Lock() + defer as.Unlock() + + for _, tx := range as.unconfirmed { + if tx.TxAttempts == nil { + continue + } + for i := 0; i < len(tx.TxAttempts); i++ { + txAttempt := tx.TxAttempts[i] + if receipt.GetTxHash() == txAttempt.Hash { + // TODO(jtw): not sure how to set blocknumber, transactionindex, and receipt on conflict + txAttempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} + txAttempt.State = txmgrtypes.TxAttemptBroadcast + if txAttempt.BroadcastBeforeBlockNum == nil { + blockNum := receipt.GetBlockNumber().Int64() + txAttempt.BroadcastBeforeBlockNum = &blockNum + } + + tx.State = TxConfirmed + return nil + } } } - if !found { - // NOTE(jtw): this would mean that the TxAttempt did not exist for the Tx - // NOTE(jtw): should this log a warning? - // NOTE(jtw): can this happen? - tx.TxAttempts = append(tx.TxAttempts, txAttempt) + + return fmt.Errorf("move_unconfirmed_to_confirmed: no unconfirmed transaction with receipt %v: %w", receipt, ErrTxnNotFound) +} + +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnstartedToFatalError( + etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + txError null.String, +) error { + as.Lock() + defer as.Unlock() + + tx := as.unstarted.RemoveTxByID(etx.ID) + if tx == nil { + return fmt.Errorf("move_unstarted_to_fatal_error: no unstarted transaction with ID %d", etx.ID) } - as.unconfirmed[tx.ID] = tx - as.inprogress = nil + tx.State = TxFatalError + tx.Sequence = nil + tx.TxAttempts = nil + tx.InitialBroadcastAt = nil + tx.Error = txError + as.fatalErrored[tx.ID] = tx return nil } @@ -573,6 +625,15 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveIn return nil } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnconfirmedToConfirmedMissingReceipt() error { + // TODO + return nil +} +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToConfirmedMissingReceipt() error { + // TODO + return nil +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandon() { as.Lock() defer as.Unlock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index e97dc8ddc4e..81ec3fca544 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -10,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/smartcontractkit/chainlink-common/pkg/logger" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" @@ -20,6 +21,9 @@ import ( // BIG TODO LIST // TODO: make sure that all state transitions are handled by the address state to ensure that the in-memory store is always in a consistent state // TODO: figure out if multiple tx attempts are actually stored in the db for each tx +// TODO: check that txns are deep copied when returned from the in-memory store +// TODO: need a way to get id for a tx attempt. since there are some methods where the persistent store creates a tx attempt and doesnt returns it +// TODO: make sure all address states are locked when updating the in-memory store var ( // ErrInvalidChainID is returned when the chain ID is invalid @@ -58,6 +62,7 @@ type InMemoryStore[ SEQ types.Sequence, FEE feetypes.Fee, ] struct { + lggr logger.Logger chainID CHAIN_ID keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] @@ -75,11 +80,13 @@ func NewInMemoryStore[ SEQ types.Sequence, FEE feetypes.Fee, ]( + lggr logger.Logger, chainID CHAIN_ID, keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], ) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { ms := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + lggr: lggr, chainID: chainID, keyStore: keyStore, txStore: txStore, @@ -120,7 +127,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat return tx, fmt.Errorf("create_transaction: %w", err) } - // TODO(jtw); HANDLE PRUNING STEP + // Prune the in-memory txs + pruned, err := txRequest.Strategy.PruneQueue(ctx, ms) + if err != nil { + return tx, fmt.Errorf("CreateTransaction failed to prune in-memory txs: %w", err) + } + if pruned > 0 { + ms.lggr.Warnf("Dropped %d old transactions from transaction queue", pruned) + } // Add the request to the Unstarted channel to be processed by the Broadcaster if err := as.AddTxToUnstarted(&tx); err != nil { @@ -227,6 +241,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count } // UpdateTxUnstartedToInProgress updates a transaction from unstarted to in_progress. +// TODO THIS HAS SOME INCONSISTENCIES WITH THE PERSISTENT STORE func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxUnstartedToInProgress( ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], @@ -251,6 +266,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) } tx.TxAttempts = append(tx.TxAttempts, *attempt) + // TODO: DOES THIS ATTEMPT HAVE AN ID? IF NOT, HOW DO WE GET IT? // Update in address state in memory if err := as.MoveUnstartedToInProgress(tx); err != nil { @@ -272,10 +288,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx return nil, fmt.Errorf("get_tx_in_progress: %w", err) } - // NOTE(jtw): should this exist in the in-memory store? or just the persistent store? - // NOTE(jtw): where should this live? if len(tx.TxAttempts) != 1 || tx.TxAttempts[0].State != txmgrtypes.TxAttemptInProgress { - return nil, fmt.Errorf("get_tx_in_progress: expected in_progress transaction %v to have exactly one unsent attempt. "+ + return nil, fmt.Errorf("get_tx_in_progress: invariant violation: expected in_progress transaction %v to have exactly one unsent attempt. "+ "Your database is in an inconsistent state and this node will not function correctly until the problem is resolved", tx.ID) } @@ -283,6 +297,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx } // UpdateTxAttemptInProgressToBroadcast updates a transaction attempt from in_progress to broadcast. +// It also updates the transaction state to unconfirmed. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxAttemptInProgressToBroadcast( ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], @@ -305,17 +320,17 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: new attempt state must be broadcast, got: %s", newAttemptState) } + as, ok := ms.addressStates[tx.FromAddress] + if !ok { + return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", ErrAddressNotFound) + } + // Persist to persistent storage if err := ms.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, tx, attempt, newAttemptState); err != nil { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } - // Ensure that the tx state is updated to unconfirmed since this is a chain agnostic operation attempt.State = newAttemptState - as, ok := ms.addressStates[tx.FromAddress] - if !ok { - return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", ErrAddressNotFound) - } if err := as.MoveInProgressToUnconfirmed(attempt); err != nil { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } @@ -324,7 +339,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // FindNextUnstartedTransactionFromAddress returns the next unstarted transaction for a given address. -// NOTE(jtw): method signature is different from most other signatures where the tx is passed in and updated func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindNextUnstartedTransactionFromAddress(_ context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fromAddress ADDR, chainID CHAIN_ID) error { if ms.chainID.String() != chainID.String() { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) @@ -339,11 +353,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN return fmt.Errorf("find_next_unstarted_transaction_from_address: address %s is already busy with a transaction in progress", fromAddress) } - var err error - tx, err = as.PeekNextUnstartedTx() - if tx == nil { + etx, err := as.PeekNextUnstartedTx() + if err != nil || etx == nil { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", err) } + tx = ms.deepCopyTx(etx) return nil } @@ -361,7 +375,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR return fmt.Errorf("save_replacement_in_progress_attempt: expected oldattempt to have an ID") } - // Check if fromaddress enabled as, ok := ms.addressStates[oldAttempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) @@ -376,30 +389,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR if tx == nil { return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) } - var found bool - for i := 0; i < len(tx.TxAttempts); i++ { - if tx.TxAttempts[i].ID == oldAttempt.ID { - tx.TxAttempts[i] = *replacementAttempt - found = true - } - } - if !found { - tx.TxAttempts = append(tx.TxAttempts, *replacementAttempt) - // NOTE(jtw): should this log a warning? - } + // TODO: DOES THIS ATTEMPT HAVE AN ID? IF NOT, HOW DO WE GET IT? + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{*replacementAttempt} return nil } // UpdateTxFatalError updates a transaction to fatal_error. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxFatalError(ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - if tx.State != TxInProgress { + if tx.State != TxInProgress && tx.State != TxUnstarted { return fmt.Errorf("update_tx_fatal_error: can only transition to fatal_error from in_progress, transaction is currently %s", tx.State) } if !tx.Error.Valid { return fmt.Errorf("update_tx_fatal_error: expected error field to be set") } - // Check if fromaddress enabled + as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_fatal_error: %w", ErrAddressNotFound) @@ -411,11 +415,18 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Update in memory store - if err := as.MoveInProgressToFatalError(tx.Error); err != nil { - return fmt.Errorf("update_tx_fatal_error: %w", err) + switch tx.State { + case TxInProgress: + if err := as.MoveInProgressToFatalError(tx.Error); err != nil { + return fmt.Errorf("update_tx_fatal_error: %w", err) + } + case TxUnstarted: + if err := as.MoveUnstartedToFatalError(*tx, tx.Error); err != nil { + return fmt.Errorf("update_tx_fatal_error: %w", err) + } } - return fmt.Errorf("update_tx_fatal_error: not implemented") + return nil } // Close closes the InMemoryStore @@ -468,11 +479,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return } - // TODO(jtw): how many tx_attempts are actually stored in the db for each tx? It looks like its only 1 - attempt := tx.TxAttempts[0] - if attempt.State == txmgrtypes.TxAttemptBroadcast && attempt.BroadcastBeforeBlockNum == nil && - tx.ChainID.String() == chainID.String() { - tx.TxAttempts[0].BroadcastBeforeBlockNum = &blockNum + + for i := 0; i < len(tx.TxAttempts); i++ { + attempt := tx.TxAttempts[i] + if attempt.State == txmgrtypes.TxAttemptBroadcast && attempt.BroadcastBeforeBlockNum == nil { + tx.TxAttempts[i].BroadcastBeforeBlockNum = &blockNum + } } } for _, as := range ms.addressStates { @@ -489,11 +501,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { - if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { - return tx.ChainID.String() == chainID.String() - } - - return false + return tx.TxAttempts != nil && len(tx.TxAttempts) > 0 } states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} @@ -515,7 +523,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat // Update in memory store fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - if tx.BroadcastAt != nil { + if tx.BroadcastAt != nil && tx.BroadcastAt.Before(now) { tx.BroadcastAt = &now } } @@ -535,13 +543,19 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Update in memory store - fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - tx.State = TxUnconfirmed - } - + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - as.ApplyToTxs(nil, fn, txIDs...) + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + for _, txID := range txIDs { + if err := as.MoveConfirmedMissingReceiptToUnconfirmed(txID); err != nil { + continue + } + } + wg.Done() + }(as) } + wg.Wait() return nil } @@ -655,42 +669,18 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveF return err } - // convert receipts to map - receiptsMap := map[TX_HASH]R{} - for _, receipt := range receipts { - receiptsMap[receipt.GetTxHash()] = receipt - } - // Update in memory store - fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { - return - } - attempt := tx.TxAttempts[0] - receipt, ok := receiptsMap[attempt.Hash] - if !ok { - return - } - - if attempt.Receipts != nil && len(attempt.Receipts) > 0 && - attempt.Receipts[0].GetBlockNumber() != nil && receipt.GetBlockNumber() != nil && - attempt.Receipts[0].GetBlockNumber().Cmp(receipt.GetBlockNumber()) == 0 { - return - } - // TODO(jtw): this needs to be finished - - attempt.State = txmgrtypes.TxAttemptBroadcast - if attempt.BroadcastBeforeBlockNum == nil { - blocknum := receipt.GetBlockNumber().Int64() - attempt.BroadcastBeforeBlockNum = &blocknum - } - attempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} - - tx.State = TxConfirmed - } + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - as.ApplyToTxs(nil, fn) + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + for _, receipt := range receipts { + as.MoveUnconfirmedToConfirmed(receipt) + } + wg.Done() + }(as) } + wg.Wait() return nil } @@ -1270,6 +1260,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveC } // Update in memory store + // TODO: WHERE LEFT OFF fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { if tx.ID != attempt.TxID { return From 52e0c2adb4056babfc6609fbf292e6332b6f7540 Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 21 Dec 2023 15:27:29 -0500 Subject: [PATCH 40/74] updates --- common/txmgr/address_state.go | 41 ++++++++++++++++++++++++++++++---- common/txmgr/inmemory_store.go | 32 ++++++++++++-------------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index f83098b7bbf..7a5e8fc665e 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" @@ -625,12 +626,44 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveIn return nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnconfirmedToConfirmedMissingReceipt() error { - // TODO +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnconfirmedToConfirmedMissingReceipt(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + as.Lock() + defer as.Unlock() + + tx, ok := as.unconfirmed[attempt.TxID] + if !ok || tx == nil { + return fmt.Errorf("move_unconfirmed_to_confirmed_missing_receipt: no unconfirmed transaction with ID %d: %w", attempt.TxID, ErrTxnNotFound) + } + if tx.BroadcastAt.Before(broadcastAt) { + tx.BroadcastAt = &broadcastAt + } + tx.State = TxConfirmedMissingReceipt + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{attempt} + tx.TxAttempts[0].State = txmgrtypes.TxAttemptBroadcast + + as.confirmedMissingReceipt[tx.ID] = tx + delete(as.unconfirmed, tx.ID) + return nil } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToConfirmedMissingReceipt() error { - // TODO +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToConfirmedMissingReceipt(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + as.Lock() + defer as.Unlock() + + tx := as.inprogress + if tx == nil { + return fmt.Errorf("move_in_progress_to_confirmed_missing_receipt: no transaction in progress") + } + if tx.BroadcastAt.Before(broadcastAt) { + tx.BroadcastAt = &broadcastAt + } + tx.State = TxConfirmedMissingReceipt + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{attempt} + tx.TxAttempts[0].State = txmgrtypes.TxAttemptBroadcast + + as.confirmedMissingReceipt[tx.ID] = tx + as.inprogress = nil + return nil } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 81ec3fca544..76851bb6b9d 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -1260,22 +1260,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveC } // Update in memory store - // TODO: WHERE LEFT OFF - fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - if tx.ID != attempt.TxID { - return - } - if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { - return - } - if tx.BroadcastAt.Before(broadcastAt) { - tx.BroadcastAt = &broadcastAt - } - - tx.TxAttempts[0].State = txmgrtypes.TxAttemptBroadcast - tx.State = TxConfirmedMissingReceipt + if err := as.MoveInProgressToConfirmedMissingReceipt(*attempt, broadcastAt); err != nil { + return err } - as.ApplyToTxs(nil, fn, attempt.TxID) return nil } @@ -1293,14 +1280,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI return err } + // Update in memory store fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { if tx.ID != attempt.TxID { return } - if tx.TxAttempts == nil { - tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { + for i := 0; i < len(tx.TxAttempts); i++ { + if tx.TxAttempts[i].ID == attempt.ID { + tx.TxAttempts[i].State = txmgrtypes.TxAttemptInProgress + tx.TxAttempts[i].BroadcastBeforeBlockNum = attempt.BroadcastBeforeBlockNum + return + } + } } - tx.TxAttempts = append(tx.TxAttempts, *attempt) + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{*attempt} + return + } as.ApplyToTxs(nil, fn, attempt.TxID) From a47f98acab63170db25565a7ed52588c5e9140cc Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 4 Jan 2024 11:26:59 -0500 Subject: [PATCH 41/74] implement updateTxForRebroadcast --- common/txmgr/address_state.go | 26 ++++++++++++++++++++++++++ common/txmgr/inmemory_store.go | 8 +++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 7a5e8fc665e..21f54627c3a 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -666,6 +666,32 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveIn return nil } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveConfirmedToUnconfirmed(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + as.Lock() + defer as.Unlock() + + if attempt.State != txmgrtypes.TxAttemptBroadcast { + return fmt.Errorf("move_confirmed_to_unconfirmed: attempt must be in broadcast state") + } + + tx, ok := as.confirmed[attempt.TxID] + if !ok || tx == nil { + return fmt.Errorf("move_confirmed_to_unconfirmed: no confirmed transaction with ID %d: %w", attempt.TxID, ErrTxnNotFound) + } + tx.State = TxUnconfirmed + + // Delete the receipt from the attempt + attempt.Receipts = nil + // Reset the broadcast information for the attempt + attempt.State = txmgrtypes.TxAttemptInProgress + attempt.BroadcastBeforeBlockNum = nil + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{attempt} + + as.unconfirmed[tx.ID] = tx + delete(as.confirmed, tx.ID) + + return nil +} func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abandon() { as.Lock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 76851bb6b9d..2be4cad47bf 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -1379,11 +1379,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Update in memory store - - // TODO - // delete receipts - // update tx unconfirmed - // update tx_attempt unbroadcast + if err := as.MoveConfirmedToUnconfirmed(etxAttempt); err != nil { + return err + } return nil } From 121c98bfece16e0f0d2f1cf37c06b73e6db5b42d Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 4 Jan 2024 11:51:39 -0500 Subject: [PATCH 42/74] implement FindTxsRequiringGasBump --- common/txmgr/inmemory_store.go | 47 ++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 2be4cad47bf..4cb0bdd7b6e 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -1421,9 +1421,52 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT if ms.chainID.String() != chainID.String() { return nil, fmt.Errorf("find_txs_requiring_gas_bump: %w", ErrInvalidChainID) } + if gasBumpThreshold == 0 { + return nil, nil + } - // TODO - return nil, nil + as, ok := ms.addressStates[address] + if !ok { + return nil, fmt.Errorf("find_txs_requiring_gas_bump: %w", ErrAddressNotFound) + } + + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + attempt := tx.TxAttempts[0] + if *attempt.BroadcastBeforeBlockNum <= blockNum || + attempt.State == txmgrtypes.TxAttemptBroadcast { + return false + } + + if tx.State != TxUnconfirmed || + attempt.ID != 0 { + return false + } + + return true + } + states := []txmgrtypes.TxState{TxUnconfirmed} + txs := as.FetchTxs(states, filter) + // sort by sequence ASC + sort.Slice(txs, func(i, j int) bool { + return (*txs[i].Sequence).Int64() < (*txs[j].Sequence).Int64() + }) + + if depth > 0 { + // LIMIT by depth + if len(txs) > int(depth) { + txs = txs[:depth] + } + } + + etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) + for i, tx := range txs { + etxs[i] = ms.deepCopyTx(&tx) + } + + return etxs, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkAllConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) error { if ms.chainID.String() != chainID.String() { From 3f4619419859085bccbff14eb873996cdb22d8ea Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 4 Jan 2024 13:18:07 -0500 Subject: [PATCH 43/74] finish initial implementation of core methods of txStore --- common/txmgr/address_state.go | 41 +++++++- common/txmgr/inmemory_store.go | 166 +++++++++++++++++++++++++++------ 2 files changed, 175 insertions(+), 32 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 21f54627c3a..b299bb1f9fa 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -213,7 +213,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTx return as.idempotencyKeyToTx[key] } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence() SEQ { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LatestSequence() SEQ { as.RLock() defer as.RUnlock() @@ -230,6 +230,23 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLa return maxSeq } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MaxConfirmedSequence() SEQ { + as.RLock() + defer as.RUnlock() + + var maxSeq SEQ + for _, tx := range as.confirmed { + if tx.Sequence == nil { + continue + } + if (*tx.Sequence).Int64() > maxSeq.Int64() { + maxSeq = *tx.Sequence + } + } + + return maxSeq +} + func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyToTxs( txStates []txmgrtypes.TxState, fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), @@ -625,6 +642,28 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveIn return nil } +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveConfirmedMissingReceiptToFatalError( + etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + txError null.String, +) error { + as.Lock() + defer as.Unlock() + + tx, ok := as.confirmedMissingReceipt[etx.ID] + if !ok || tx == nil { + return fmt.Errorf("move_confirmed_missing_receipt_to_fatal_error: no confirmed_missing_receipt transaction with ID %d: %w", etx.ID, ErrTxnNotFound) + } + + tx.State = TxFatalError + tx.Sequence = nil + tx.TxAttempts = nil + tx.InitialBroadcastAt = nil + tx.Error = txError + as.fatalErrored[tx.ID] = tx + delete(as.confirmedMissingReceipt, tx.ID) + + return nil +} func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnconfirmedToConfirmedMissingReceipt(attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { as.Lock() diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 4cb0bdd7b6e..64c7868ade6 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -3,6 +3,7 @@ package txmgr import ( "context" "encoding/json" + "errors" "fmt" "math/big" "sort" @@ -27,15 +28,18 @@ import ( var ( // ErrInvalidChainID is returned when the chain ID is invalid - ErrInvalidChainID = fmt.Errorf("invalid chain ID") + ErrInvalidChainID = errors.New("invalid chain ID") // ErrTxnNotFound is returned when a transaction is not found - ErrTxnNotFound = fmt.Errorf("transaction not found") + ErrTxnNotFound = errors.New("transaction not found") // ErrExistingIdempotencyKey is returned when a transaction with the same idempotency key already exists - ErrExistingIdempotencyKey = fmt.Errorf("transaction with idempotency key already exists") + ErrExistingIdempotencyKey = errors.New("transaction with idempotency key already exists") // ErrAddressNotFound is returned when an address is not found - ErrAddressNotFound = fmt.Errorf("address not found") + ErrAddressNotFound = errors.New("address not found") // ErrSequenceNotFound is returned when a sequence is not found - ErrSequenceNotFound = fmt.Errorf("sequence not found") + ErrSequenceNotFound = errors.New("sequence not found") + // ErrCouldNotGetReceipt is the error string we save if we reach our finality depth for a confirmed transaction without ever getting a receipt + // This most likely happened because an external wallet used the account for this nonce + ErrCouldNotGetReceipt = errors.New("could not get receipt") ) type PersistentTxStore[ @@ -62,7 +66,7 @@ type InMemoryStore[ SEQ types.Sequence, FEE feetypes.Fee, ] struct { - lggr logger.Logger + lggr logger.SugaredLogger chainID CHAIN_ID keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] @@ -80,7 +84,7 @@ func NewInMemoryStore[ SEQ types.Sequence, FEE feetypes.Fee, ]( - lggr logger.Logger, + lggr logger.SugaredLogger, chainID CHAIN_ID, keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], @@ -202,7 +206,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) } - seq = as.FindLatestSequence() + seq = as.LatestSequence() if seq.Int64() == 0 { return seq, fmt.Errorf("find_latest_sequence: %w", ErrSequenceNotFound) } @@ -1473,39 +1477,139 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkA return fmt.Errorf("mark_all_confirmed_missing_receipt: %w", ErrInvalidChainID) } - // TODO(jtw): need to complete + // Persist to persistent storage + if err := ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID); err != nil { + return err + } - /* - // Persist to persistent storage - if err := ms.txStore.MarkAllConfirmedMissingReceipt(ctx, chainID); err != nil { - return err - } + // Update in memory store + wg := sync.WaitGroup{} + errsLock := sync.Mutex{} + var errs error + for _, as := range ms.addressStates { + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + maxConfirmedSequence := as.MaxConfirmedSequence() + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.Sequence == nil { + return false + } + if tx.State != TxUnconfirmed { + return false + } - // Update in memory store - fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { - if tx.State != TxUnconfirmed { - return + return (*tx.Sequence).Int64() < maxConfirmedSequence.Int64() } - if tx.Sequence >= maxSequence { - return + states := []txmgrtypes.TxState{TxUnconfirmed} + txs := as.FetchTxs(states, filter) + for _, tx := range txs { + attempt := tx.TxAttempts[0] + + if err := as.MoveUnconfirmedToConfirmedMissingReceipt(attempt, *tx.BroadcastAt); err != nil { + err = fmt.Errorf("mark_all_confirmed_missing_receipt: address: %s: %w", as.fromAddress, err) + errsLock.Lock() + errs = errors.Join(errs, err) + errsLock.Unlock() + } } + wg.Done() + }(as) + } + wg.Wait() - tx.State = TxConfirmedMissingReceipt - } - states := []txmgrtypes.TxState{TxUnconfirmed} - for _, as := range ms.addressStates { - as.ApplyToTxs(states, fn) - } - */ - - return nil + return errs } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkOldTxesMissingReceiptAsErrored(ctx context.Context, blockNum int64, finalityDepth uint32, chainID CHAIN_ID) error { - // TODO - return ms.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, finalityDepth, chainID) + if ms.chainID.String() != chainID.String() { + return fmt.Errorf("mark_old_txes_missing_receipt_as_errored: %w", ErrInvalidChainID) + } + + // Persist to persistent storage + if err := ms.txStore.MarkOldTxesMissingReceiptAsErrored(ctx, blockNum, finalityDepth, chainID); err != nil { + return err + } + + // Update in memory store + type result struct { + ID int64 + Sequence SEQ + FromAddress ADDR + MaxBroadcastBeforeBlockNum int64 + TxHashes []TX_HASH + } + var resultsLock sync.Mutex + var results []result + wg := sync.WaitGroup{} + errsLock := sync.Mutex{} + var errs error + for _, as := range ms.addressStates { + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { + return false + } + if tx.State != TxConfirmedMissingReceipt { + return false + } + attempt := tx.TxAttempts[0] + if attempt.BroadcastBeforeBlockNum == nil { + return false + } + + return *attempt.BroadcastBeforeBlockNum < blockNum-int64(finalityDepth) + } + states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} + txs := as.FetchTxs(states, filter) + for _, tx := range txs { + if err := as.MoveConfirmedMissingReceiptToFatalError(tx, null.StringFrom(ErrCouldNotGetReceipt.Error())); err != nil { + err = fmt.Errorf("mark_old_txes_missing_receipt_as_errored: address: %s: %w", as.fromAddress, err) + errsLock.Lock() + errs = errors.Join(errs, err) + errsLock.Unlock() + continue + } + hashes := make([]TX_HASH, len(tx.TxAttempts)) + maxBroadcastBeforeBlockNum := int64(0) + for i, attempt := range tx.TxAttempts { + hashes[i] = attempt.Hash + if attempt.BroadcastBeforeBlockNum != nil { + if *attempt.BroadcastBeforeBlockNum > maxBroadcastBeforeBlockNum { + maxBroadcastBeforeBlockNum = *attempt.BroadcastBeforeBlockNum + } + } + } + result := result{ + ID: tx.ID, + Sequence: *tx.Sequence, + FromAddress: tx.FromAddress, + MaxBroadcastBeforeBlockNum: maxBroadcastBeforeBlockNum, + TxHashes: hashes, + } + resultsLock.Lock() + results = append(results, result) + resultsLock.Unlock() + } + wg.Done() + }(as) + } + wg.Wait() + + for _, r := range results { + ms.lggr.Criticalw(fmt.Sprintf("eth_tx with ID %v expired without ever getting a receipt for any of our attempts. "+ + "Current block height is %v, transaction was broadcast before block height %v. This transaction may not have not been sent and will be marked as fatally errored. "+ + "This can happen if there is another instance of chainlink running that is using the same private key, or if "+ + "an external wallet has been used to send a transaction from account %s with nonce %v."+ + " Please note that Chainlink requires exclusive ownership of it's private keys and sharing keys across multiple"+ + " chainlink instances, or using the chainlink keys with an external wallet is NOT SUPPORTED and WILL lead to missed transactions", + r.ID, blockNum, r.MaxBroadcastBeforeBlockNum, r.FromAddress, r.Sequence), "ethTxID", r.ID, "sequence", r.Sequence, "fromAddress", r.FromAddress, "txHashes", r.TxHashes) + } + + return errs } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + // TODO: COMPLETE DEEP COPY WORK etx := *tx etx.TxAttempts = make([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(tx.TxAttempts)) copy(etx.TxAttempts, tx.TxAttempts) From 325fb1241a834929b70f9c893b5ccd8ad96b1055 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 5 Jan 2024 09:34:02 -0500 Subject: [PATCH 44/74] remove testing since this will be in future work --- core/chains/tx_store_test.go | 142 ----------------------------------- 1 file changed, 142 deletions(-) delete mode 100644 core/chains/tx_store_test.go diff --git a/core/chains/tx_store_test.go b/core/chains/tx_store_test.go deleted file mode 100644 index c1fc3383d0a..00000000000 --- a/core/chains/tx_store_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package chains_test - -import ( - "context" - "fmt" - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" - "github.com/smartcontractkit/chainlink/v2/common/txmgr" - txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" - commontxmmocks "github.com/smartcontractkit/chainlink/v2/common/txmgr/types/mocks" - "github.com/smartcontractkit/chainlink/v2/common/types" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" - "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" - "github.com/smartcontractkit/chainlink/v2/core/services/pg/datatypes" -) - -type TestingTxStore[ - ADDR types.Hashable, - CHAIN_ID types.ID, - TX_HASH types.Hashable, - BLOCK_HASH types.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], - SEQ types.Sequence, - FEE feetypes.Fee, -] interface { - CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, gas.EvmFee], err error) - Close() -} - -type txStoreFunc func(*testing.T, chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) - -var txStoresFuncs = map[string]txStoreFunc{ - "evm_postgres_tx_store": evmTxStore, - "evm_in_memory_tx_store": inmemoryTxStore, -} - -func TestTxStore_CreateTransaction(t *testing.T) { - cfg := configtest.NewGeneralConfig(t, nil) - - for n, f := range txStoresFuncs { - t.Run(n, func(t *testing.T) { - txStore, fromAddress, chainID := f(t, cfg) - defer txStore.Close() - - subject := uuid.New() - strategy := commontxmmocks.NewTxStrategy(t) - strategy.On("Subject").Return(uuid.NullUUID{UUID: subject, Valid: true}) - strategy.On("PruneQueue", mock.Anything, mock.AnythingOfType("*txmgr.evmTxStore")).Return(int64(0), nil) - ctx := context.Background() - idempotencyKey := "11" - - tts := []struct { - scenario string - createTransactionInput createTransactionInput - createTransactionOutputCheck func(*testing.T, txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) - }{ - { - scenario: "success", - createTransactionInput: createTransactionInput{ - txRequest: txmgrtypes.TxRequest[common.Address, common.Hash]{ - IdempotencyKey: &idempotencyKey, - FromAddress: fromAddress, - ToAddress: common.BytesToAddress([]byte("test")), - EncodedPayload: []byte{1, 2, 3}, - FeeLimit: uint32(1000), - Meta: nil, - Strategy: strategy, - }, - chainID: chainID, - }, - createTransactionOutputCheck: func(t *testing.T, tx txmgrtypes.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], err error) { - funcName := "CreateTransaction" - require.NoError(t, err, fmt.Sprintf("%s: expected err to be nil", funcName)) - assert.Equal(t, &idempotencyKey, tx.IdempotencyKey, fmt.Sprintf("%s: expected idempotencyKey to match actual idempotencyKey", funcName)) - // Check CreatedAt is within 1 second of now - assert.WithinDuration(t, time.Now().UTC(), tx.CreatedAt, time.Second, fmt.Sprintf("%s: expected time to be within 1 second of actual time", funcName)) - assert.Equal(t, txmgr.TxUnstarted, tx.State, fmt.Sprintf("%s: expected state to match actual state", funcName)) - assert.Equal(t, chainID, tx.ChainID, fmt.Sprintf("%s: expected chainID to match actual chainID", funcName)) - assert.Equal(t, fromAddress, tx.FromAddress, fmt.Sprintf("%s: expected fromAddress to match actual fromAddress", funcName)) - assert.Equal(t, common.BytesToAddress([]byte("test")), tx.ToAddress, fmt.Sprintf("%s: expected toAddress to match actual toAddress", funcName)) - assert.Equal(t, []byte{1, 2, 3}, tx.EncodedPayload, fmt.Sprintf("%s: expected encodedPayload to match actual encodedPayload", funcName)) - assert.Equal(t, uint32(1000), tx.FeeLimit, fmt.Sprintf("%s: expected feeLimit to match actual feeLimit", funcName)) - var expMeta *datatypes.JSON - assert.Equal(t, expMeta, tx.Meta, fmt.Sprintf("%s: expected meta to match actual meta", funcName)) - assert.Equal(t, uuid.NullUUID{UUID: subject, Valid: true}, tx.Subject, fmt.Sprintf("%s: expected subject to match actual subject", funcName)) - }, - }, - } - - for _, tt := range tts { - t.Run(tt.scenario, func(t *testing.T) { - actTx, actErr := txStore.CreateTransaction(ctx, tt.createTransactionInput.txRequest, tt.createTransactionInput.chainID) - tt.createTransactionOutputCheck(t, actTx, actErr) - }) - } - }) - } -} - -type createTransactionInput struct { - txRequest txmgrtypes.TxRequest[common.Address, common.Hash] - chainID *big.Int -} - -func evmTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { - db := pgtest.NewSqlxDB(t) - keyStore := cltest.NewKeyStore(t, db, cfg.Database()) - _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - chainID := ethClient.ConfiguredChainID() - - return cltest.NewTestTxStore(t, db, cfg.Database()), fromAddress, chainID -} -func inmemoryTxStore(t *testing.T, cfg chainlink.GeneralConfig) (TestingTxStore[common.Address, *big.Int, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee], common.Address, *big.Int) { - db := pgtest.NewSqlxDB(t) - txStore := cltest.NewTestTxStore(t, db, cfg.Database()) - keyStore := cltest.NewKeyStore(t, db, cfg.Database()) - _, fromAddress := cltest.MustInsertRandomKey(t, keyStore.Eth()) - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - chainID := ethClient.ConfiguredChainID() - - ims, err := txmgr.NewInMemoryStore[ - *big.Int, common.Address, common.Hash, common.Hash, *evmtypes.Receipt, evmtypes.Nonce, gas.EvmFee, - ](chainID, keyStore.Eth(), txStore) - require.NoError(t, err) - - return ims, fromAddress, chainID -} From 18a4355c4fdfa926d9ffabe9aff14dd141406b26 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 5 Jan 2024 10:22:29 -0500 Subject: [PATCH 45/74] improve deepCopy methods --- common/txmgr/inmemory_store.go | 67 +++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 64c7868ade6..fc83c8a5a2e 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -1608,11 +1608,66 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkO return errs } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { - // TODO: COMPLETE DEEP COPY WORK - etx := *tx - etx.TxAttempts = make([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(tx.TxAttempts)) - copy(etx.TxAttempts, tx.TxAttempts) +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx( + tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + copyTx := txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{ + ID: tx.ID, + IdempotencyKey: tx.IdempotencyKey, + Sequence: tx.Sequence, + FromAddress: tx.FromAddress, + ToAddress: tx.ToAddress, + EncodedPayload: make([]byte, len(tx.EncodedPayload)), + Value: *new(big.Int).Set(&tx.Value), + FeeLimit: tx.FeeLimit, + Error: tx.Error, + BroadcastAt: tx.BroadcastAt, + InitialBroadcastAt: tx.InitialBroadcastAt, + CreatedAt: tx.CreatedAt, + State: tx.State, + TxAttempts: make([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(tx.TxAttempts)), + Meta: tx.Meta, + Subject: tx.Subject, + ChainID: tx.ChainID, + PipelineTaskRunID: tx.PipelineTaskRunID, + MinConfirmations: tx.MinConfirmations, + TransmitChecker: tx.TransmitChecker, + SignalCallback: tx.SignalCallback, + CallbackCompleted: tx.CallbackCompleted, + } + + // Copy the EncodedPayload + copy(copyTx.EncodedPayload, tx.EncodedPayload) + + // Copy the TxAttempts + for i, attempt := range tx.TxAttempts { + copyTx.TxAttempts[i] = ms.deepCopyTxAttempt(copyTx, attempt) + } + + return ©Tx +} - return &etx +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTxAttempt( + tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + attempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], +) txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { + copyAttempt := txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{ + ID: attempt.ID, + TxID: attempt.TxID, + Tx: tx, + TxFee: attempt.TxFee, + ChainSpecificFeeLimit: attempt.ChainSpecificFeeLimit, + SignedRawTx: make([]byte, len(attempt.SignedRawTx)), + Hash: attempt.Hash, + CreatedAt: attempt.CreatedAt, + BroadcastBeforeBlockNum: attempt.BroadcastBeforeBlockNum, + State: attempt.State, + Receipts: make([]txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], len(attempt.Receipts)), + TxType: attempt.TxType, + } + + copy(copyAttempt.SignedRawTx, attempt.SignedRawTx) + copy(copyAttempt.Receipts, attempt.Receipts) + + return copyAttempt } From 0091ac312aa90c2ad88a33fa48ed8a645aac9219 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 5 Jan 2024 16:05:35 -0500 Subject: [PATCH 46/74] deep copy returning txns and txAttempts --- common/txmgr/inmemory_store.go | 228 ++++++++++++++++++++++++++------- 1 file changed, 179 insertions(+), 49 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index fc83c8a5a2e..1551035da0e 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -116,7 +116,14 @@ func NewInMemoryStore[ } // CreateTransaction creates a new transaction for a given txRequest. -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction(ctx context.Context, txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], chainID CHAIN_ID) (tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTransaction( + ctx context.Context, + txRequest txmgrtypes.TxRequest[ADDR, TX_HASH], + chainID CHAIN_ID, +) ( + tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + err error, +) { if ms.chainID.String() != chainID.String() { return tx, fmt.Errorf("create_transaction: %w", ErrInvalidChainID) } @@ -142,10 +149,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat // Add the request to the Unstarted channel to be processed by the Broadcaster if err := as.AddTxToUnstarted(&tx); err != nil { - return tx, fmt.Errorf("create_transaction: %w", err) + return *ms.deepCopyTx(tx), fmt.Errorf("create_transaction: %w", err) } - return tx, nil + return *ms.deepCopyTx(tx), nil } // FindTxWithIdempotencyKey returns a transaction with the given idempotency key @@ -159,7 +166,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { if tx := as.FindTxWithIdempotencyKey(idempotencyKey); tx != nil { - return ms.deepCopyTx(tx), nil + return ms.deepCopyTx(*tx), nil } } @@ -297,7 +304,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx "Your database is in an inconsistent state and this node will not function correctly until the problem is resolved", tx.ID) } - return ms.deepCopyTx(tx), nil + return ms.deepCopyTx(*tx), nil } // UpdateTxAttemptInProgressToBroadcast updates a transaction attempt from in_progress to broadcast. @@ -361,7 +368,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN if err != nil || etx == nil { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", err) } - tx = ms.deepCopyTx(etx) + tx = ms.deepCopyTx(*etx) return nil } @@ -499,7 +506,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr } // FindTxAttemptsConfirmedMissingReceipt returns all transactions that are confirmed but missing a receipt -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) ([]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsConfirmedMissingReceipt(ctx context.Context, chainID CHAIN_ID) ( + []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + error, +) { if ms.chainID.String() != chainID.String() { return nil, fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } @@ -513,9 +523,17 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT attempts = append(attempts, as.FetchTxAttempts(states, filter)...) } // sort by tx_id ASC, gas_price DESC, gas_tip_cap DESC - // TODO + sort.SliceStable(attempts, func(i, j int) bool { + return attempts[i].TxID < attempts[j].TxID + }) + + // deep copy the attempts + var eAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, attempt := range attempts { + eAttempts = append(eAttempts, ms.deepCopyTxAttempt(attempt.Tx, attempt)) + } - return attempts, nil + return eAttempts, nil } // UpdateBroadcastAts updates the broadcast_at time for a given set of attempts @@ -565,7 +583,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // FindTxAttemptsRequiringReceiptFetch returns all transactions that are missing a receipt -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) (attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], err error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxAttemptsRequiringReceiptFetch(ctx context.Context, chainID CHAIN_ID) ( + attempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + err error, +) { if ms.chainID.String() != chainID.String() { return attempts, fmt.Errorf("find_tx_attempts_requiring_receipt_fetch: %w", ErrInvalidChainID) } @@ -584,12 +605,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT attempts = append(attempts, as.FetchTxAttempts(states, filterFn)...) } // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC - // TODO + sort.Slice(attempts, func(i, j int) bool { + return (*attempts[i].Tx.Sequence).Int64() < (*attempts[j].Tx.Sequence).Int64() + }) + + // deep copy the attempts + var eAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, attempt := range attempts { + eAttempts = append(eAttempts, ms.deepCopyTxAttempt(attempt.Tx, attempt)) + } - return attempts, nil + return eAttempts, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) ([]txmgrtypes.ReceiptPlus[R], error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesPendingCallback(ctx context.Context, blockNum int64, chainID CHAIN_ID) ( + []txmgrtypes.ReceiptPlus[R], + error, +) { if ms.chainID.String() != chainID.String() { return nil, fmt.Errorf("find_txes_pending_callback: %w", ErrInvalidChainID) } @@ -656,9 +688,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat tx.CallbackCompleted = true } } + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - as.ApplyToTxs(nil, fn) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + as.ApplyToTxs(nil, fn) + wg.Done() + }(as) } + wg.Wait() return nil } @@ -674,22 +711,31 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveF } // Update in memory store + errsLock := sync.Mutex{} + var errs error wg := sync.WaitGroup{} for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { for _, receipt := range receipts { - as.MoveUnconfirmedToConfirmed(receipt) + if err := as.MoveUnconfirmedToConfirmed(receipt); err != nil { + errsLock.Lock() + errs = errors.Join(errs, err) + errsLock.Unlock() + } } wg.Done() }(as) } wg.Wait() - return nil + return errs } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxesByMetaFieldAndStates(ctx context.Context, metaField string, metaValue string, states []txmgrtypes.TxState, chainID *big.Int) ( + []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + error, +) { if ms.chainID.String() != chainID.String() { return nil, fmt.Errorf("find_txes_by_meta_field_and_states: %w", ErrInvalidChainID) } @@ -708,12 +754,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return false } + txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - for _, tx := range as.FetchTxs(states, filterFn) { - txs = append(txs, &tx) - } + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + for _, tx := range as.FetchTxs(states, filterFn) { + txsLock.Lock() + txs = append(txs, ms.deepCopyTx(tx)) + txsLock.Unlock() + } + wg.Done() + }(as) } + wg.Wait() return txs, nil } @@ -737,12 +792,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return false } + txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - for _, tx := range as.FetchTxs(states, filterFn) { - txs = append(txs, &tx) - } + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + for _, tx := range as.FetchTxs(states, filterFn) { + txsLock.Lock() + txs = append(txs, ms.deepCopyTx(tx)) + txsLock.Unlock() + } + wg.Done() + }(as) } + wg.Wait() return txs, nil } @@ -777,12 +841,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return attempt.Receipts[0].GetBlockNumber().Int64() >= blockNum } + txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - for _, tx := range as.FetchTxs(nil, filterFn) { - txs = append(txs, &tx) - } + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + for _, tx := range as.FetchTxs(nil, filterFn) { + txsLock.Lock() + txs = append(txs, ms.deepCopyTx(tx)) + txsLock.Unlock() + } + wg.Done() + }(as) } + wg.Wait() return txs, nil } @@ -801,12 +874,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txIDs[i] = id.Int64() } + txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - for _, tx := range as.FetchTxs(states, filterFn, txIDs...) { - txs = append(txs, &tx) - } + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + for _, tx := range as.FetchTxs(states, filterFn, txIDs...) { + txsLock.Lock() + txs = append(txs, ms.deepCopyTx(tx)) + txsLock.Unlock() + } + wg.Done() + }(as) } + wg.Wait() return txs, nil } @@ -971,7 +1053,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) for i, tx := range txs { - etxs[i] = &tx + etxs[i] = ms.deepCopyTx(tx) } return etxs, nil @@ -1004,14 +1086,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} attempts := as.FetchTxAttempts(states, filter) // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC - // TODO - + sort.Slice(attempts, func(i, j int) bool { + return (*attempts[i].Tx.Sequence).Int64() < (*attempts[j].Tx.Sequence).Int64() + }) // LIMIT by maxInFlightTransactions if len(attempts) > int(maxInFlightTransactions) { attempts = attempts[:maxInFlightTransactions] } - return attempts, nil + // deep copy the attempts + var eAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, attempt := range attempts { + eAttempts = append(eAttempts, ms.deepCopyTxAttempt(attempt.Tx, attempt)) + } + + return eAttempts, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithSequence(_ context.Context, fromAddress ADDR, seq SEQ) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { @@ -1033,7 +1122,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, nil } - return &txs[0], nil + return ms.deepCopyTx(txs[0]), nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTransactionsConfirmedInBlockRange(_ context.Context, highBlockNumber, lowBlockNumber int64, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { @@ -1059,10 +1148,19 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return blockNum >= lowBlockNumber && blockNum <= highBlockNumber } states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt} + txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - txs = append(txs, as.FetchTxs(states, filter)...) + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + txsLock.Lock() + txs = append(txs, as.FetchTxs(states, filter)...) + txsLock.Unlock() + wg.Done() + }(as) } + wg.Wait() // sort by sequence ASC sort.Slice(txs, func(i, j int) bool { return (*txs[i].Sequence).Int64() < (*txs[j].Sequence).Int64() @@ -1070,7 +1168,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) for i, tx := range txs { - etxs[i] = &tx + etxs[i] = ms.deepCopyTx(tx) } return etxs, nil @@ -1085,10 +1183,19 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE return tx.InitialBroadcastAt != nil } states := []txmgrtypes.TxState{TxUnconfirmed} + txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - txs = append(txs, as.FetchTxs(states, filter)...) + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + txsLock.Lock() + txs = append(txs, as.FetchTxs(states, filter)...) + txsLock.Unlock() + wg.Done() + }(as) } + wg.Wait() var minInitialBroadcastAt time.Time for _, tx := range txs { @@ -1114,10 +1221,19 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE return attempt.BroadcastBeforeBlockNum != nil } states := []txmgrtypes.TxState{TxUnconfirmed} + txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - txs = append(txs, as.FetchTxs(states, filter)...) + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + txsLock.Lock() + txs = append(txs, as.FetchTxs(states, filter)...) + txsLock.Unlock() + wg.Done() + }(as) } + wg.Wait() var minBroadcastBeforeBlockNum int64 for _, tx := range txs { @@ -1149,7 +1265,13 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetIn states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt, TxUnconfirmed} attempts := as.FetchTxAttempts(states, filter) - return attempts, nil + // deep copy the attempts + var eAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + for _, attempt := range attempts { + eAttempts = append(eAttempts, ms.deepCopyTxAttempt(attempt.Tx, attempt)) + } + + return eAttempts, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNonFatalTransactions(ctx context.Context, chainID CHAIN_ID) ([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { @@ -1157,18 +1279,26 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNo return nil, fmt.Errorf("get_non_fatal_transactions: %w", ErrInvalidChainID) } - // TODO(jtw): this is niave ... it might be better to just use all states excluding fatal filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return tx.State != TxFatalError } + txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + wg := sync.WaitGroup{} for _, as := range ms.addressStates { - txs = append(txs, as.FetchTxs(nil, filter)...) + wg.Add(1) + go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + txsLock.Lock() + txs = append(txs, as.FetchTxs(nil, filter)...) + txsLock.Unlock() + wg.Done() + }(as) } + wg.Wait() etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) for i, tx := range txs { - etxs[i] = &tx + etxs[i] = ms.deepCopyTx(tx) } return etxs, nil @@ -1181,7 +1311,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx for _, as := range ms.addressStates { txs := as.FetchTxs(nil, filter, id) if len(txs) > 0 { - return &txs[0], nil + return ms.deepCopyTx(txs[0]), nil } } @@ -1215,7 +1345,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadT } txAttempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} for _, tx := range as.FetchTxs(nil, filter, etx.ID) { - txAttempts = append(txAttempts, tx.TxAttempts...) + for _, txAttempt := range tx.TxAttempts { + txAttempts = append(txAttempts, ms.deepCopyTxAttempt(*etx, txAttempt)) + } } etx.TxAttempts = txAttempts @@ -1242,7 +1374,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prelo for i, attempt := range attempts { for _, tx := range txs { if tx.ID == attempt.TxID { - attempts[i].Tx = tx + attempts[i].Tx = *ms.deepCopyTx(tx) } } } @@ -1299,8 +1431,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI } } tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{*attempt} - return - } as.ApplyToTxs(nil, fn, attempt.TxID) @@ -1467,7 +1597,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT etxs := make([]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)) for i, tx := range txs { - etxs[i] = ms.deepCopyTx(&tx) + etxs[i] = ms.deepCopyTx(tx) } return etxs, nil @@ -1609,7 +1739,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkO } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) deepCopyTx( - tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + tx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { copyTx := txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{ ID: tx.ID, From 1fe825c5eb0227fe04abdde3fed0437c9bfdfd76 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 5 Jan 2024 16:16:40 -0500 Subject: [PATCH 47/74] add locks for any instance where the txstore is working with addressStates --- common/txmgr/inmemory_store.go | 109 ++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 1551035da0e..854f378e4c0 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -22,9 +22,7 @@ import ( // BIG TODO LIST // TODO: make sure that all state transitions are handled by the address state to ensure that the in-memory store is always in a consistent state // TODO: figure out if multiple tx attempts are actually stored in the db for each tx -// TODO: check that txns are deep copied when returned from the in-memory store // TODO: need a way to get id for a tx attempt. since there are some methods where the persistent store creates a tx attempt and doesnt returns it -// TODO: make sure all address states are locked when updating the in-memory store var ( // ErrInvalidChainID is returned when the chain ID is invalid @@ -127,6 +125,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat if ms.chainID.String() != chainID.String() { return tx, fmt.Errorf("create_transaction: %w", ErrInvalidChainID) } + + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return tx, fmt.Errorf("create_transaction: %w", ErrAddressNotFound) @@ -182,6 +183,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check if ms.chainID.String() != chainID.String() { return fmt.Errorf("check_tx_queue_capacity: %w", ErrInvalidChainID) } + + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return fmt.Errorf("check_tx_queue_capacity: %w", ErrAddressNotFound) @@ -208,6 +212,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL if ms.chainID.String() != chainID.String() { return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) } + + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) @@ -228,6 +235,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count if ms.chainID.String() != chainID.String() { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } + + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) @@ -243,6 +253,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count if ms.chainID.String() != chainID.String() { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } + + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) @@ -267,6 +280,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat if attempt.State != txmgrtypes.TxAttemptInProgress { return fmt.Errorf("update_tx_unstarted_to_in_progress: attempt state must be in_progress") } + + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", ErrAddressNotFound) @@ -289,6 +305,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat // GetTxInProgress returns the in_progress transaction for a given address. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxInProgress(ctx context.Context, fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return nil, fmt.Errorf("get_tx_in_progress: %w", ErrAddressNotFound) @@ -331,6 +349,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: new attempt state must be broadcast, got: %s", newAttemptState) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", ErrAddressNotFound) @@ -354,6 +374,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN if ms.chainID.String() != chainID.String() { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) @@ -386,6 +408,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR return fmt.Errorf("save_replacement_in_progress_attempt: expected oldattempt to have an ID") } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[oldAttempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) @@ -415,6 +439,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_fatal_error: expected error field to be set") } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_fatal_error: %w", ErrAddressNotFound) @@ -466,6 +492,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband } // check that the address exists in the unstarted transactions + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[addr] if !ok { return fmt.Errorf("abandon: %w", ErrAddressNotFound) @@ -498,6 +526,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr } } } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { as.ApplyToTxs(nil, fn) } @@ -519,6 +549,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { attempts = append(attempts, as.FetchTxAttempts(states, filter)...) } @@ -550,6 +582,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { as.ApplyToTxs(nil, fn, txIDs...) } @@ -565,6 +599,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Update in memory store + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() wg := sync.WaitGroup{} for _, as := range ms.addressStates { wg.Add(1) @@ -601,6 +637,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} attempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { attempts = append(attempts, as.FetchTxAttempts(states, filterFn)...) } @@ -646,6 +684,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } states := []txmgrtypes.TxState{TxConfirmed} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { txs = append(txs, as.FetchTxs(states, filterFn)...) } @@ -689,7 +729,10 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } } wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { + wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { as.ApplyToTxs(nil, fn) wg.Done() @@ -714,6 +757,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveF errsLock := sync.Mutex{} var errs error wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -757,6 +802,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -795,6 +842,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -844,6 +893,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -877,6 +928,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -909,6 +962,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prune return tx.Subject.UUID == subject } var m int + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { m += as.PruneUnstartedTxQueue(queueSize, filter) } @@ -957,6 +1012,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapT } wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -970,6 +1027,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapT return tx.State == TxFatalError && tx.CreatedAt.Before(timeThreshold) } states = []txmgrtypes.TxState{TxFatalError} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -987,6 +1046,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count } var total int + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { total += as.CountTransactionsByState(state) } @@ -1003,6 +1064,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Delet } // Check if fromaddress enabled + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("delete_in_progress_attempt: %w", ErrAddressNotFound) @@ -1031,6 +1094,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_txs_requiring_resubmission_due_to_insufficient_funds: %w", ErrInvalidChainID) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("find_txs_requiring_resubmission_due_to_insufficient_funds: %w", ErrAddressNotFound) @@ -1064,6 +1129,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrInvalidChainID) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrAddressNotFound) @@ -1104,6 +1171,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithSequence(_ context.Context, fromAddress ADDR, seq SEQ) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[fromAddress] if !ok { return nil, fmt.Errorf("find_tx_with_sequence: %w", ErrAddressNotFound) @@ -1151,6 +1220,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1186,6 +1257,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1224,6 +1297,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1250,6 +1325,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetIn return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrInvalidChainID) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrAddressNotFound) @@ -1285,6 +1362,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNo txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1308,6 +1387,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return tx.ID == id } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { txs := as.FetchTxs(nil, filter, id) if len(txs) > 0 { @@ -1324,6 +1405,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasIn return false, fmt.Errorf("has_in_progress_transaction: %w", ErrInvalidChainID) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[account] if !ok { return false, fmt.Errorf("has_in_progress_transaction: %w", ErrAddressNotFound) @@ -1335,6 +1418,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasIn } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadTxAttempts(_ context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[etx.FromAddress] if !ok { return fmt.Errorf("load_tx_attempts: %w", ErrAddressNotFound) @@ -1358,6 +1443,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prelo return nil } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[attempts[0].Tx.FromAddress] if !ok { return fmt.Errorf("preload_txes: %w", ErrAddressNotFound) @@ -1382,6 +1469,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prelo return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_confirmed_missing_receipt_attempt: %w", ErrAddressNotFound) @@ -1403,6 +1492,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveC return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInProgressAttempt(ctx context.Context, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_in_progress_attempt: %w", ErrAddressNotFound) @@ -1437,6 +1528,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInsufficientFundsAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_insufficient_funds_attempt: %w", ErrAddressNotFound) @@ -1469,6 +1562,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveSentAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_sent_attempt: %w", ErrAddressNotFound) @@ -1502,6 +1597,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveS return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[etx.FromAddress] if !ok { return fmt.Errorf("update_tx_for_rebroadcast: %w", ErrAddressNotFound) @@ -1541,6 +1638,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxF return attempt.Receipts[0].GetBlockNumber().Int64() <= (blockHeight - int64(tx.MinConfirmations.Uint32)) } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { txas := as.FetchTxAttempts(nil, fn, txID) if len(txas) > 0 { @@ -1559,6 +1658,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, nil } + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("find_txs_requiring_gas_bump: %w", ErrAddressNotFound) @@ -1616,6 +1717,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkA wg := sync.WaitGroup{} errsLock := sync.Mutex{} var errs error + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1672,6 +1775,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkO wg := sync.WaitGroup{} errsLock := sync.Mutex{} var errs error + ms.addressStatesLock.Lock() + defer ms.addressStatesLock.Unlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { From 9ed165108b1e516ada5142d7b412c7022d06d814 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 5 Jan 2024 16:39:02 -0500 Subject: [PATCH 48/74] cleanup --- common/txmgr/address_state.go | 3 --- common/txmgr/inmemory_store.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index b299bb1f9fa..1ce8d21c43c 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -172,9 +172,6 @@ func NewAddressState[ } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close() { - as.Lock() - defer as.Unlock() - as.unstarted.Close() as.unstarted = nil as.inprogress = nil diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 854f378e4c0..0433948e9fb 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -20,9 +20,9 @@ import ( ) // BIG TODO LIST -// TODO: make sure that all state transitions are handled by the address state to ensure that the in-memory store is always in a consistent state // TODO: figure out if multiple tx attempts are actually stored in the db for each tx // TODO: need a way to get id for a tx attempt. since there are some methods where the persistent store creates a tx attempt and doesnt returns it +// TODO: should txAttempt state transitions be handled by the address state manager? var ( // ErrInvalidChainID is returned when the chain ID is invalid From 204d5ffcbb0af84024e7c322cb8e30015c7908d2 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 9 Jan 2024 15:54:43 -0500 Subject: [PATCH 49/74] address some feedback --- common/txmgr/address_state.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 1ce8d21c43c..f61628af889 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -63,9 +63,6 @@ func NewAddressState[ allTransactions: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } - as.Lock() - defer as.Unlock() - // Load all unstarted transactions from persistent storage offset := 0 limit := 50 @@ -172,11 +169,24 @@ func NewAddressState[ } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) close() { + clear(as.idempotencyKeyToTx) + as.unstarted.Close() as.unstarted = nil as.inprogress = nil + clear(as.unconfirmed) - clear(as.idempotencyKeyToTx) + clear(as.confirmedMissingReceipt) + clear(as.confirmed) + clear(as.allTransactions) + clear(as.fatalErrored) + + as.idempotencyKeyToTx = nil + as.unconfirmed = nil + as.confirmedMissingReceipt = nil + as.confirmed = nil + as.allTransactions = nil + as.fatalErrored = nil } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CountTransactionsByState(txState txmgrtypes.TxState) int { From 2535e27cf2f12f4db52d6c7f8244b1ecc9d2dbd2 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 9 Jan 2024 16:32:32 -0500 Subject: [PATCH 50/74] address more comments --- common/txmgr/address_state.go | 9 --------- common/txmgr/broadcaster.go | 2 +- common/txmgr/txmgr.go | 3 +-- core/chains/evm/txmgr/evm_tx_store.go | 10 ++++++++-- core/chains/evm/txmgr/evm_tx_store_test.go | 3 +-- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index f61628af889..e75b9b0622a 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -756,15 +756,6 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) abando for _, tx := range as.unconfirmed { as.abandonTx(tx) } - for _, tx := range as.idempotencyKeyToTx { - as.abandonTx(tx) - } - for _, tx := range as.confirmedMissingReceipt { - as.abandonTx(tx) - } - for _, tx := range as.confirmed { - as.abandonTx(tx) - } clear(as.unconfirmed) } diff --git a/common/txmgr/broadcaster.go b/common/txmgr/broadcaster.go index 4a2cec75179..1a23c4aaa24 100644 --- a/common/txmgr/broadcaster.go +++ b/common/txmgr/broadcaster.go @@ -698,7 +698,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) next defer cancel() etx := &txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} if err := eb.txStore.FindNextUnstartedTransactionFromAddress(ctx, etx, fromAddress, eb.chainID); err != nil { - if errors.Is(err, sql.ErrNoRows) || errors.Is(err, ErrTxnNotFound) { + if errors.Is(err, ErrTxnNotFound) { // Finish. No more transactions left to process. Hoorah! return nil, nil } diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index ca12e5ffaae..19ffd7942fd 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -2,7 +2,6 @@ package txmgr import ( "context" - "database/sql" "errors" "fmt" "math/big" @@ -477,7 +476,7 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTran if txRequest.IdempotencyKey != nil { var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) - if err != nil && !errors.Is(err, sql.ErrNoRows) && !errors.Is(err, ErrTxnNotFound) { + if err != nil && !errors.Is(err, ErrTxnNotFound) { return tx, fmt.Errorf("Failed to search for transaction with IdempotencyKey: %w", err) } if existingTx != nil { diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 152834f3a5b..b3c2f925868 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -476,7 +476,7 @@ func (o *evmTxStore) UnstartedTransactions(offset, limit int, fromAddress common return } - sql = `SELECT * FROM evm.txes WHERE state = 'unstarted' AND from_address = $1 AND evm_chain_id = $2 ORDER BY value ASC, created_at ASC, id ASC LIMIT $3 OFFSET $4` + sql = `SELECT * FROM evm.txes WHERE state = 'unstarted' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` var dbTxs []DbEthTx if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { return @@ -1653,8 +1653,14 @@ func (o *evmTxStore) FindNextUnstartedTransactionFromAddress(ctx context.Context qq := o.q.WithOpts(pg.WithParentCtx(ctx)) var dbEtx DbEthTx err := qq.Get(&dbEtx, `SELECT * FROM evm.txes WHERE from_address = $1 AND state = 'unstarted' AND evm_chain_id = $2 ORDER BY value ASC, created_at ASC, id ASC`, fromAddress, chainID.String()) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return txmgr.ErrTxnNotFound + } + return pkgerrors.Wrap(err, "failed to FindNextUnstartedTransactionFromAddress") + } dbEtx.ToTx(etx) - return pkgerrors.Wrap(err, "failed to FindNextUnstartedTransactionFromAddress") + return nil } func (o *evmTxStore) UpdateTxFatalError(ctx context.Context, etx *Tx) error { diff --git a/core/chains/evm/txmgr/evm_tx_store_test.go b/core/chains/evm/txmgr/evm_tx_store_test.go index 0b4287b6f6c..54f9512a763 100644 --- a/core/chains/evm/txmgr/evm_tx_store_test.go +++ b/core/chains/evm/txmgr/evm_tx_store_test.go @@ -1,7 +1,6 @@ package txmgr_test import ( - "database/sql" "fmt" "math/big" "testing" @@ -1264,7 +1263,7 @@ func TestORM_FindNextUnstartedTransactionFromAddress(t *testing.T) { resultEtx := new(txmgr.Tx) err := txStore.FindNextUnstartedTransactionFromAddress(testutils.Context(t), resultEtx, fromAddress, ethClient.ConfiguredChainID()) - assert.ErrorIs(t, err, sql.ErrNoRows) + assert.ErrorIs(t, err, txmgrcommon.ErrTxnNotFound) }) t.Run("finds unstarted tx", func(t *testing.T) { From b1bfd67ba45b9d3500f00096da46060f7c282e17 Mon Sep 17 00:00:00 2001 From: James Walker Date: Tue, 9 Jan 2024 16:55:56 -0500 Subject: [PATCH 51/74] some cleanup in inmemory storage --- common/txmgr/inmemory_store.go | 43 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 0433948e9fb..43e5b691338 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -23,6 +23,7 @@ import ( // TODO: figure out if multiple tx attempts are actually stored in the db for each tx // TODO: need a way to get id for a tx attempt. since there are some methods where the persistent store creates a tx attempt and doesnt returns it // TODO: should txAttempt state transitions be handled by the address state manager? +// TODO: add RLock and RUnlock to address state usage where applicable var ( // ErrInvalidChainID is returned when the chain ID is invalid @@ -808,8 +809,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { for _, tx := range as.FetchTxs(states, filterFn) { + etx := ms.deepCopyTx(tx) txsLock.Lock() - txs = append(txs, ms.deepCopyTx(tx)) + txs = append(txs, etx) txsLock.Unlock() } wg.Done() @@ -848,8 +850,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { for _, tx := range as.FetchTxs(states, filterFn) { + etx := ms.deepCopyTx(tx) txsLock.Lock() - txs = append(txs, ms.deepCopyTx(tx)) + txs = append(txs, etx) txsLock.Unlock() } wg.Done() @@ -899,8 +902,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { for _, tx := range as.FetchTxs(nil, filterFn) { + etx := ms.deepCopyTx(tx) txsLock.Lock() - txs = append(txs, ms.deepCopyTx(tx)) + txs = append(txs, etx) txsLock.Unlock() } wg.Done() @@ -934,8 +938,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { for _, tx := range as.FetchTxs(states, filterFn, txIDs...) { + etx := ms.deepCopyTx(tx) txsLock.Lock() - txs = append(txs, ms.deepCopyTx(tx)) + txs = append(txs, etx) txsLock.Unlock() } wg.Done() @@ -1207,7 +1212,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT if attempt.State != txmgrtypes.TxAttemptBroadcast { return false } - if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + if len(attempt.Receipts) == 0 { return false } if attempt.Receipts[0].GetBlockNumber() == nil { @@ -1225,8 +1230,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + ts := as.FetchTxs(states, filter) txsLock.Lock() - txs = append(txs, as.FetchTxs(states, filter)...) + txs = append(txs, ts...) txsLock.Unlock() wg.Done() }(as) @@ -1262,8 +1268,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + etxs := as.FetchTxs(states, filter) txsLock.Lock() - txs = append(txs, as.FetchTxs(states, filter)...) + txs = append(txs, etxs...) txsLock.Unlock() wg.Done() }(as) @@ -1302,8 +1309,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + etxs := as.FetchTxs(states, filter) txsLock.Lock() - txs = append(txs, as.FetchTxs(states, filter)...) + txs = append(txs, etxs...) txsLock.Unlock() wg.Done() }(as) @@ -1367,8 +1375,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNo for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + etxs := as.FetchTxs(nil, filter) txsLock.Lock() - txs = append(txs, as.FetchTxs(nil, filter)...) + txs = append(txs, etxs...) txsLock.Unlock() wg.Done() }(as) @@ -1485,11 +1494,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveC } // Update in memory store - if err := as.MoveInProgressToConfirmedMissingReceipt(*attempt, broadcastAt); err != nil { - return err - } - - return nil + return as.MoveInProgressToConfirmedMissingReceipt(*attempt, broadcastAt) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInProgressAttempt(ctx context.Context, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { ms.addressStatesLock.Lock() @@ -1610,11 +1615,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Update in memory store - if err := as.MoveConfirmedToUnconfirmed(etxAttempt); err != nil { - return err - } - - return nil + return as.MoveConfirmedToUnconfirmed(etxAttempt) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxFinalized(ctx context.Context, blockHeight int64, txID int64, chainID CHAIN_ID) (bool, error) { if ms.chainID.String() != chainID.String() { @@ -1814,7 +1815,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkO } } } - result := result{ + rr := result{ ID: tx.ID, Sequence: *tx.Sequence, FromAddress: tx.FromAddress, @@ -1822,7 +1823,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkO TxHashes: hashes, } resultsLock.Lock() - results = append(results, result) + results = append(results, rr) resultsLock.Unlock() } wg.Done() From f4d9b0f94591bae20ff79abde41273a6206e04b2 Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 11 Jan 2024 15:33:38 -0500 Subject: [PATCH 52/74] change address state locks to read locks where applicable --- common/txmgr/inmemory_store.go | 204 ++++++++++++++++----------------- 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 43e5b691338..aa2d3ac9f99 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -127,8 +127,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat return tx, fmt.Errorf("create_transaction: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return tx, fmt.Errorf("create_transaction: %w", ErrAddressNotFound) @@ -164,8 +164,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } // Check if the transaction is in the pending queue of all address states - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { if tx := as.FindTxWithIdempotencyKey(idempotencyKey); tx != nil { return ms.deepCopyTx(*tx), nil @@ -185,8 +185,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check return fmt.Errorf("check_tx_queue_capacity: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return fmt.Errorf("check_tx_queue_capacity: %w", ErrAddressNotFound) @@ -214,8 +214,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindL return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) @@ -237,8 +237,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) @@ -255,8 +255,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return 0, fmt.Errorf("count_unstarted_transactions: %w", ErrAddressNotFound) @@ -282,8 +282,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_unstarted_to_in_progress: attempt state must be in_progress") } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", ErrAddressNotFound) @@ -306,8 +306,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat // GetTxInProgress returns the in_progress transaction for a given address. func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTxInProgress(ctx context.Context, fromAddress ADDR) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return nil, fmt.Errorf("get_tx_in_progress: %w", ErrAddressNotFound) @@ -350,8 +350,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: new attempt state must be broadcast, got: %s", newAttemptState) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", ErrAddressNotFound) @@ -375,8 +375,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN if ms.chainID.String() != chainID.String() { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) @@ -409,8 +409,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR return fmt.Errorf("save_replacement_in_progress_attempt: expected oldattempt to have an ID") } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[oldAttempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_replacement_in_progress_attempt: %w", ErrAddressNotFound) @@ -440,8 +440,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_fatal_error: expected error field to be set") } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[tx.FromAddress] if !ok { return fmt.Errorf("update_tx_fatal_error: %w", ErrAddressNotFound) @@ -493,8 +493,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Aband } // check that the address exists in the unstarted transactions - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[addr] if !ok { return fmt.Errorf("abandon: %w", ErrAddressNotFound) @@ -527,8 +527,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr } } } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { as.ApplyToTxs(nil, fn) } @@ -550,8 +550,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { attempts = append(attempts, as.FetchTxAttempts(states, filter)...) } @@ -583,8 +583,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { as.ApplyToTxs(nil, fn, txIDs...) } @@ -600,8 +600,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } // Update in memory store - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() wg := sync.WaitGroup{} for _, as := range ms.addressStates { wg.Add(1) @@ -638,8 +638,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} attempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { attempts = append(attempts, as.FetchTxAttempts(states, filterFn)...) } @@ -685,8 +685,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } states := []txmgrtypes.TxState{TxConfirmed} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { txs = append(txs, as.FetchTxs(states, filterFn)...) } @@ -730,8 +730,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat } } wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -758,8 +758,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveF errsLock := sync.Mutex{} var errs error wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -803,8 +803,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -844,8 +844,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -896,8 +896,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -932,8 +932,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -967,8 +967,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prune return tx.Subject.UUID == subject } var m int - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { m += as.PruneUnstartedTxQueue(queueSize, filter) } @@ -1017,8 +1017,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapT } wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1032,8 +1032,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapT return tx.State == TxFatalError && tx.CreatedAt.Before(timeThreshold) } states = []txmgrtypes.TxState{TxFatalError} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1051,8 +1051,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count } var total int - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { total += as.CountTransactionsByState(state) } @@ -1069,8 +1069,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Delet } // Check if fromaddress enabled - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("delete_in_progress_attempt: %w", ErrAddressNotFound) @@ -1099,8 +1099,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_txs_requiring_resubmission_due_to_insufficient_funds: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("find_txs_requiring_resubmission_due_to_insufficient_funds: %w", ErrAddressNotFound) @@ -1134,8 +1134,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrAddressNotFound) @@ -1176,8 +1176,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTxWithSequence(_ context.Context, fromAddress ADDR, seq SEQ) (*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[fromAddress] if !ok { return nil, fmt.Errorf("find_tx_with_sequence: %w", ErrAddressNotFound) @@ -1225,8 +1225,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1263,8 +1263,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1304,8 +1304,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1333,8 +1333,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetIn return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrAddressNotFound) @@ -1370,8 +1370,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetNo txsLock := sync.Mutex{} txs := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} wg := sync.WaitGroup{} - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1396,8 +1396,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetTx filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return tx.ID == id } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { txs := as.FetchTxs(nil, filter, id) if len(txs) > 0 { @@ -1414,8 +1414,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasIn return false, fmt.Errorf("has_in_progress_transaction: %w", ErrInvalidChainID) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[account] if !ok { return false, fmt.Errorf("has_in_progress_transaction: %w", ErrAddressNotFound) @@ -1427,8 +1427,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) HasIn } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LoadTxAttempts(_ context.Context, etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[etx.FromAddress] if !ok { return fmt.Errorf("load_tx_attempts: %w", ErrAddressNotFound) @@ -1452,8 +1452,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prelo return nil } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[attempts[0].Tx.FromAddress] if !ok { return fmt.Errorf("preload_txes: %w", ErrAddressNotFound) @@ -1478,8 +1478,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prelo return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveConfirmedMissingReceiptAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_confirmed_missing_receipt_attempt: %w", ErrAddressNotFound) @@ -1497,8 +1497,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveC return as.MoveInProgressToConfirmedMissingReceipt(*attempt, broadcastAt) } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInProgressAttempt(ctx context.Context, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_in_progress_attempt: %w", ErrAddressNotFound) @@ -1533,8 +1533,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveInsufficientFundsAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_insufficient_funds_attempt: %w", ErrAddressNotFound) @@ -1567,8 +1567,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveSentAttempt(ctx context.Context, timeout time.Duration, attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], broadcastAt time.Time) error { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[attempt.Tx.FromAddress] if !ok { return fmt.Errorf("save_sent_attempt: %w", ErrAddressNotFound) @@ -1602,8 +1602,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveS return nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxForRebroadcast(ctx context.Context, etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], etxAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) error { - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[etx.FromAddress] if !ok { return fmt.Errorf("update_tx_for_rebroadcast: %w", ErrAddressNotFound) @@ -1639,8 +1639,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxF return attempt.Receipts[0].GetBlockNumber().Int64() <= (blockHeight - int64(tx.MinConfirmations.Uint32)) } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { txas := as.FetchTxAttempts(nil, fn, txID) if len(txas) > 0 { @@ -1659,8 +1659,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, nil } - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() as, ok := ms.addressStates[address] if !ok { return nil, fmt.Errorf("find_txs_requiring_gas_bump: %w", ErrAddressNotFound) @@ -1718,8 +1718,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkA wg := sync.WaitGroup{} errsLock := sync.Mutex{} var errs error - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { @@ -1776,8 +1776,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkO wg := sync.WaitGroup{} errsLock := sync.Mutex{} var errs error - ms.addressStatesLock.Lock() - defer ms.addressStatesLock.Unlock() + ms.addressStatesLock.RLock() + defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { From a20f47720d9edece5e706d167f79f679edc82c86 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 17 Jan 2024 13:43:19 -0500 Subject: [PATCH 53/74] update comments --- common/txmgr/inmemory_store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index aa2d3ac9f99..7cc861b9e7f 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -23,7 +23,6 @@ import ( // TODO: figure out if multiple tx attempts are actually stored in the db for each tx // TODO: need a way to get id for a tx attempt. since there are some methods where the persistent store creates a tx attempt and doesnt returns it // TODO: should txAttempt state transitions be handled by the address state manager? -// TODO: add RLock and RUnlock to address state usage where applicable var ( // ErrInvalidChainID is returned when the chain ID is invalid From 90a4c9d118c46cf162c5e695ba53073e984d0dc3 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 17 Jan 2024 15:41:26 -0500 Subject: [PATCH 54/74] cleanup line break changes --- core/chains/evm/txmgr/evm_tx_store.go | 50 ++++----------------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index b3c2f925868..a9a61920344 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -698,11 +698,7 @@ func (o *evmTxStore) LoadTxesAttempts(etxs []*Tx, qopts ...pg.QOpt) error { ethTxesM[etx.ID] = etxs[i] } var dbTxAttempts []DbEthTxAttempt - if err := qq.Select(&dbTxAttempts, ` - SELECT * - FROM evm.tx_attempts - WHERE eth_tx_id = ANY($1) - ORDER BY evm.tx_attempts.gas_price DESC, evm.tx_attempts.gas_tip_cap DESC`, pq.Array(ethTxIDs)); err != nil { + if err := qq.Select(&dbTxAttempts, `SELECT * FROM evm.tx_attempts WHERE eth_tx_id = ANY($1) ORDER BY evm.tx_attempts.gas_price DESC, evm.tx_attempts.gas_tip_cap DESC`, pq.Array(ethTxIDs)); err != nil { return pkgerrors.Wrap(err, "loadEthTxesAttempts failed to load evm.tx_attempts") } for _, dbAttempt := range dbTxAttempts { @@ -1041,8 +1037,7 @@ func (o *evmTxStore) GetInProgressTxAttempts(ctx context.Context, address common var dbAttempts []DbEthTxAttempt err = tx.Select(&dbAttempts, ` SELECT evm.tx_attempts.* FROM evm.tx_attempts -INNER JOIN evm.txes ON evm.txes.id = evm.tx_attempts.eth_tx_id - AND evm.txes.state in ('confirmed', 'confirmed_missing_receipt', 'unconfirmed') +INNER JOIN evm.txes ON evm.txes.id = evm.tx_attempts.eth_tx_id AND evm.txes.state in ('confirmed', 'confirmed_missing_receipt', 'unconfirmed') WHERE evm.tx_attempts.state = 'in_progress' AND evm.txes.from_address = $1 AND evm.txes.evm_chain_id = $2 `, address, chainID.String()) if err != nil { @@ -1219,11 +1214,7 @@ func (o *evmTxStore) FindEarliestUnconfirmedBroadcastTime(ctx context.Context, c defer cancel() qq := o.q.WithOpts(pg.WithParentCtx(ctx)) err = qq.Transaction(func(tx pg.Queryer) error { - if err = qq.QueryRowContext(ctx, ` - SELECT - min(initial_broadcast_at) - FROM evm.txes - WHERE state = 'unconfirmed' AND evm_chain_id = $1`, chainID.String()).Scan(&broadcastAt); err != nil { + if err = qq.QueryRowContext(ctx, `SELECT min(initial_broadcast_at) FROM evm.txes WHERE state = 'unconfirmed' AND evm_chain_id = $1`, chainID.String()).Scan(&broadcastAt); err != nil { return fmt.Errorf("failed to query for unconfirmed eth_tx count: %w", err) } return nil @@ -1452,30 +1443,9 @@ func (o *evmTxStore) FindTxsRequiringGasBump(ctx context.Context, address common err = qq.Transaction(func(tx pg.Queryer) error { stmt := ` SELECT evm.txes.* FROM evm.txes -LEFT JOIN evm.tx_attempts ON evm.txes.id = evm.tx_attempts.eth_tx_id - AND ( - broadcast_before_block_num > $4 - OR broadcast_before_block_num IS NULL - OR evm.tx_attempts.state != 'broadcast' - ) -WHERE - evm.txes.state = 'unconfirmed' - AND evm.tx_attempts.id IS NULL - AND evm.txes.from_address = $1 - AND evm.txes.evm_chain_id = $2 - AND ( - ($3 = 0) - OR ( - evm.txes.id IN ( - SELECT id - FROM evm.txes - WHERE - state = 'unconfirmed' - AND from_address = $1 - ORDER BY nonce ASC LIMIT $3 - ) - ) - ) +LEFT JOIN evm.tx_attempts ON evm.txes.id = evm.tx_attempts.eth_tx_id AND (broadcast_before_block_num > $4 OR broadcast_before_block_num IS NULL OR evm.tx_attempts.state != 'broadcast') +WHERE evm.txes.state = 'unconfirmed' AND evm.tx_attempts.id IS NULL AND evm.txes.from_address = $1 AND evm.txes.evm_chain_id = $2 + AND (($3 = 0) OR (evm.txes.id IN (SELECT id FROM evm.txes WHERE state = 'unconfirmed' AND from_address = $1 ORDER BY nonce ASC LIMIT $3))) ORDER BY nonce ASC ` var dbEtxs []DbEthTx @@ -1841,13 +1811,7 @@ func (o *evmTxStore) HasInProgressTransaction(ctx context.Context, account commo defer cancel() qq := o.q.WithOpts(pg.WithParentCtx(ctx)) err = qq.Get(&exists, ` - SELECT EXISTS( - SELECT 1 - FROM evm.txes - WHERE - state = 'in_progress' - AND from_address = $1 - AND evm_chain_id = $2)`, account, chainID.String()) + SELECT EXISTS(SELECT 1 FROM evm.txes WHERE state = 'in_progress' AND from_address = $1 AND evm_chain_id = $2)`, account, chainID.String()) return exists, pkgerrors.Wrap(err, "hasInProgressTransaction failed") } From 3a66a05eec1787662a8f058c3331ab123dd7e1a9 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 17 Jan 2024 18:06:42 -0500 Subject: [PATCH 55/74] create AllTransactions db query --- common/txmgr/address_state.go | 110 +++++--------------------- common/txmgr/inmemory_store.go | 5 +- common/txmgr/types/tx_store.go | 5 +- core/chains/evm/txmgr/evm_tx_store.go | 76 +++--------------- 4 files changed, 31 insertions(+), 165 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index e75b9b0622a..eed7047e79c 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -63,106 +63,32 @@ func NewAddressState[ allTransactions: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } - // Load all unstarted transactions from persistent storage - offset := 0 - limit := 50 - for { - txs, count, err := txStore.UnstartedTransactions(offset, limit, as.fromAddress, as.chainID) - if err != nil { - return nil, fmt.Errorf("address_state: initialization: %w", err) - } - for i := 0; i < len(txs); i++ { - tx := txs[i] - as.unstarted.AddTx(&tx) - as.allTransactions[tx.ID] = &tx - if tx.IdempotencyKey != nil { - as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx - } - } - if count <= offset+limit { - break - } - offset += limit - } - - // Load all in progress transactions from persistent storage + // Load all transactions from persistent storage ctx := context.Background() - tx, err := txStore.GetTxInProgress(ctx, as.fromAddress) + txs, err := txStore.AllTransactions(ctx, as.fromAddress, as.chainID) if err != nil { return nil, fmt.Errorf("address_state: initialization: %w", err) } - as.inprogress = tx - if tx != nil { - if tx.IdempotencyKey != nil { - as.idempotencyKeyToTx[*tx.IdempotencyKey] = tx - } - as.allTransactions[tx.ID] = tx - } - - // Load all unconfirmed transactions from persistent storage - offset = 0 - limit = 50 - for { - txs, count, err := txStore.UnconfirmedTransactions(offset, limit, as.fromAddress, as.chainID) - if err != nil { - return nil, fmt.Errorf("address_state: initialization: %w", err) - } - for i := 0; i < len(txs); i++ { - tx := txs[i] + for i := 0; i < len(txs); i++ { + tx := txs[i] + switch tx.State { + case TxUnstarted: + as.unstarted.AddTx(&tx) + case TxInProgress: + as.inprogress = &tx + case TxUnconfirmed: as.unconfirmed[tx.ID] = &tx - as.allTransactions[tx.ID] = &tx - if tx.IdempotencyKey != nil { - as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx - } - } - if count <= offset+limit { - break - } - offset += limit - } - - // Load all confirmed transactions from persistent storage - offset = 0 - limit = 50 - for { - txs, count, err := txStore.ConfirmedTransactions(offset, limit, as.fromAddress, as.chainID) - if err != nil { - return nil, fmt.Errorf("address_state: initialization: %w", err) - } - for i := 0; i < len(txs); i++ { - tx := txs[i] - as.confirmed[tx.ID] = &tx - as.allTransactions[tx.ID] = &tx - if tx.IdempotencyKey != nil { - as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx - } - } - if count <= offset+limit { - break - } - offset += limit - } - - // Load all unconfirmed transactions from persistent storage - offset = 0 - limit = 50 - for { - txs, count, err := txStore.ConfirmedMissingReceiptTransactions(offset, limit, as.fromAddress, as.chainID) - if err != nil { - return nil, fmt.Errorf("address_state: initialization: %w", err) - } - for i := 0; i < len(txs); i++ { - tx := txs[i] + case TxConfirmedMissingReceipt: as.confirmedMissingReceipt[tx.ID] = &tx - as.allTransactions[tx.ID] = &tx - if tx.IdempotencyKey != nil { - as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx - } + case TxConfirmed: + as.confirmed[tx.ID] = &tx + case TxFatalError: + as.fatalErrored[tx.ID] = &tx } - if count <= offset+limit { - break + as.allTransactions[tx.ID] = &tx + if tx.IdempotencyKey != nil { + as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx } - offset += limit } return &as, nil diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 7cc861b9e7f..89323f72946 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -51,10 +51,7 @@ type PersistentTxStore[ ] interface { txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - UnstartedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - UnconfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - ConfirmedTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - ConfirmedMissingReceiptTransactions(limit, offset int, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + AllTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) } type InMemoryStore[ diff --git a/common/txmgr/types/tx_store.go b/common/txmgr/types/tx_store.go index 2d8d46a4930..9a7fb7f97e8 100644 --- a/common/txmgr/types/tx_store.go +++ b/common/txmgr/types/tx_store.go @@ -66,10 +66,7 @@ type InMemoryInitializer[ SEQ types.Sequence, FEE feetypes.Fee, ] interface { - UnstartedTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - UnconfirmedTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - ConfirmedTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) - ConfirmedMissingReceiptTransactions(offset, limit int, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], int, error) + AllTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) ([]Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) } // TransactionStore contains the persistence layer methods needed to manage Txs and TxAttempts diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index a9a61920344..39c966e16ea 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -64,10 +64,7 @@ type TxStoreWebApi interface { // TxStoreInMemory encapsulates the methods that are used by the txmgr to initialize the in memory tx store. type TxStoreInMemory interface { - UnstartedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) - UnconfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) - ConfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) - ConfirmedMissingReceiptTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) + AllTransactions(ctx context.Context, fromAddress common.Address, chainID *big.Int) (txs []Tx, err error) } type TestEvmTxStore interface { @@ -469,69 +466,18 @@ func (o *evmTxStore) TransactionsWithAttempts(offset, limit int) (txs []Tx, coun return } -// UnstartedTransactions returns all eth transactions that have no attempts. -func (o *evmTxStore) UnstartedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { - sql := `SELECT count(*) FROM evm.txes WHERE state = 'unstarted' AND from_address = $1 AND evm_chain_id = $2` - if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { - return - } - - sql = `SELECT * FROM evm.txes WHERE state = 'unstarted' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` - var dbTxs []DbEthTx - if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { - return - } - txs = dbEthTxsToEvmEthTxs(dbTxs) - return -} - -// UnconfirmedTransactions returns all eth transactions that have at least one attempt and in the unconfirmed state. -func (o *evmTxStore) UnconfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { - sql := `SELECT count(*) FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'unconfirmed' AND from_address = $1 AND evm_chain_id = $2` - if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { - return - } - - sql = `SELECT * FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'unconfirmed' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` - var dbTxs []DbEthTx - if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { - return - } - txs = dbEthTxsToEvmEthTxs(dbTxs) - err = o.preloadTxAttempts(txs) - return -} - -// ConfirmedTransactions returns all eth transactions that have at least one attempt and in the confirmed state. -func (o *evmTxStore) ConfirmedTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { - sql := `SELECT count(*) FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed' AND from_address = $1 AND evm_chain_id = $2` - if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { - return - } - - sql = `SELECT * FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` - var dbTxs []DbEthTx - if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { - return - } - txs = dbEthTxsToEvmEthTxs(dbTxs) - err = o.preloadTxAttempts(txs) - return -} - -// ConfirmedMissingReceiptTransactions returns all eth transactions that have at least one attempt and in the confirmed_missing_receipt state. -func (o *evmTxStore) ConfirmedMissingReceiptTransactions(offset, limit int, fromAddress common.Address, chainID *big.Int) (txs []Tx, count int, err error) { - sql := `SELECT count(*) FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed_missing_receipt' AND from_address = $1 AND evm_chain_id = $2` - if err = o.q.Get(&count, sql, fromAddress, chainID.String()); err != nil { - return - } - - sql = `SELECT * FROM evm.txes WHERE id IN (SELECT DISTINCT eth_tx_id FROM evm.tx_attempts) AND state = 'confirmed_missing_receipt' AND from_address = $1 AND evm_chain_id = $2 ORDER BY id desc LIMIT $3 OFFSET $4` - var dbTxs []DbEthTx - if err = o.q.Select(&dbTxs, sql, fromAddress, chainID.String(), limit, offset); err != nil { +// AllTransactions returns all eth transactions +func (o *evmTxStore) AllTransactions(ctx context.Context, fromAddress common.Address, chainID *big.Int) (txs []Tx, err error) { + var cancel context.CancelFunc + ctx, cancel = o.mergeContexts(ctx) + defer cancel() + qq := o.q.WithOpts(pg.WithParentCtx(ctx)) + var dbEtxs []DbEthTx + sql := `SELECT * FROM evm.txes WHERE from_address = $1 AND evm_chain_id = $2 ORDER BY id desc` + if err = qq.Select(&dbEtxs, sql, fromAddress, chainID.String()); err != nil { return } - txs = dbEthTxsToEvmEthTxs(dbTxs) + txs = dbEthTxsToEvmEthTxs(dbEtxs) err = o.preloadTxAttempts(txs) return } From da1df83d57cec612a88ad7c452d4d3aa703b841d Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 19 Jan 2024 15:13:57 -0500 Subject: [PATCH 56/74] add more comments to priority queue, fix naming issues --- common/txmgr/address_state.go | 14 +++---- common/txmgr/inmemory_store.go | 12 +++--- common/txmgr/tx_priority_queue.go | 67 ++++++++++++++++++------------- 3 files changed, 53 insertions(+), 40 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index eed7047e79c..c54666eee2b 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -180,7 +180,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MaxCon return maxSeq } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyToTxs( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyToTxsByState( txStates []txmgrtypes.TxState, fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), txIDs ...int64, @@ -190,7 +190,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyT // if txStates is empty then apply the filter to only the as.allTransactions map if len(txStates) == 0 { - as.applyToStorage(as.allTransactions, fn, txIDs...) + as.applyToTxs(as.allTransactions, fn, txIDs...) return } @@ -201,18 +201,18 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyT fn(as.inprogress) } case TxUnconfirmed: - as.applyToStorage(as.unconfirmed, fn, txIDs...) + as.applyToTxs(as.unconfirmed, fn, txIDs...) case TxConfirmedMissingReceipt: - as.applyToStorage(as.confirmedMissingReceipt, fn, txIDs...) + as.applyToTxs(as.confirmedMissingReceipt, fn, txIDs...) case TxConfirmed: - as.applyToStorage(as.confirmed, fn, txIDs...) + as.applyToTxs(as.confirmed, fn, txIDs...) case TxFatalError: - as.applyToStorage(as.fatalErrored, fn, txIDs...) + as.applyToTxs(as.fatalErrored, fn, txIDs...) } } } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToStorage( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyToTxs( txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), txIDs ...int64, diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 89323f72946..9e98692cae1 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -526,7 +526,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SetBr ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { - as.ApplyToTxs(nil, fn) + as.ApplyToTxsByState(nil, fn) } return nil @@ -582,7 +582,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { - as.ApplyToTxs(nil, fn, txIDs...) + as.ApplyToTxsByState(nil, fn, txIDs...) } return nil @@ -731,7 +731,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { - as.ApplyToTxs(nil, fn) + as.ApplyToTxsByState(nil, fn) wg.Done() }(as) } @@ -1524,7 +1524,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI } tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{*attempt} } - as.ApplyToTxs(nil, fn, attempt.TxID) + as.ApplyToTxsByState(nil, fn, attempt.TxID) return nil } @@ -1558,7 +1558,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveI tx.TxAttempts[0].State = txmgrtypes.TxAttemptInsufficientFunds } - as.ApplyToTxs(nil, fn, attempt.TxID) + as.ApplyToTxsByState(nil, fn, attempt.TxID) return nil } @@ -1593,7 +1593,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveS tx.TxAttempts[0].State = txmgrtypes.TxAttemptBroadcast } - as.ApplyToTxs(nil, fn, attempt.TxID) + as.ApplyToTxsByState(nil, fn, attempt.TxID) return nil } diff --git a/common/txmgr/tx_priority_queue.go b/common/txmgr/tx_priority_queue.go index 81dfcb30e16..5b644017fbb 100644 --- a/common/txmgr/tx_priority_queue.go +++ b/common/txmgr/tx_priority_queue.go @@ -38,45 +38,23 @@ func NewTxPriorityQueue[ return &pq } -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Cap() int { - return cap(pq.txs) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Len() int { - return len(pq.txs) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Less(i, j int) bool { - // We want Pop to give us the oldest, not newest, transaction based on creation time - return pq.txs[i].CreatedAt.Before(pq.txs[j].CreatedAt) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Swap(i, j int) { - pq.txs[i], pq.txs[j] = pq.txs[j], pq.txs[i] - pq.idToIndex[pq.txs[i].ID] = j - pq.idToIndex[pq.txs[j].ID] = i -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Push(tx any) { - pq.txs = append(pq.txs, tx.(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) -} -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pop() any { - old := pq.txs - n := len(old) - tx := old[n-1] - old[n-1] = nil // avoid memory leak - pq.txs = old[0 : n-1] - return tx -} - +// AddTx adds a transaction to the queue func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTx(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { pq.Lock() defer pq.Unlock() heap.Push(pq, tx) } + +// RemoveNextTx removes the next transaction to be processed from the queue func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { pq.Lock() defer pq.Unlock() return heap.Pop(pq).(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) } + +// RemoveTxByID removes the transaction with the given ID from the queue func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) RemoveTxByID(id int64) *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { pq.Lock() defer pq.Unlock() @@ -87,6 +65,8 @@ func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Rem return nil } + +// Prune removes the transactions that match the filter func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prune(maxUnstarted int, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { pq.Lock() defer pq.Unlock() @@ -113,6 +93,7 @@ func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pru return removed } +// PeekNextTx returns the next transaction to be processed without removing it from the queue func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PeekNextTx() *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { pq.Lock() defer pq.Unlock() @@ -122,9 +103,41 @@ func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pee } return pq.txs[0] } + +// Close clears the queue func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Close() { pq.Lock() defer pq.Unlock() clear(pq.txs) } + +// Cap returns the capacity of the queue +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Cap() int { + return cap(pq.txs) +} + +// Len, Less, Swap, Push, and Pop methods implement the heap interface +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Len() int { + return len(pq.txs) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Less(i, j int) bool { + // We want Pop to give us the oldest, not newest, transaction based on creation time + return pq.txs[i].CreatedAt.Before(pq.txs[j].CreatedAt) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Swap(i, j int) { + pq.txs[i], pq.txs[j] = pq.txs[j], pq.txs[i] + pq.idToIndex[pq.txs[i].ID] = j + pq.idToIndex[pq.txs[j].ID] = i +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Push(tx any) { + pq.txs = append(pq.txs, tx.(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE])) +} +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Pop() any { + old := pq.txs + n := len(old) + tx := old[n-1] + old[n-1] = nil // avoid memory leak + pq.txs = old[0 : n-1] + return tx +} From ba5ef07441d8c0d043b3b516977506e036f1450d Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 19 Jan 2024 15:20:44 -0500 Subject: [PATCH 57/74] fix save replacement in progress attempt --- common/txmgr/inmemory_store.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 9e98692cae1..2b88161fec9 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -417,12 +417,22 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) SaveR return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) } + // Update in memory store tx, err := as.PeekInProgressTx() if tx == nil { return fmt.Errorf("save_replacement_in_progress_attempt: %w", err) } - // TODO: DOES THIS ATTEMPT HAVE AN ID? IF NOT, HOW DO WE GET IT? - tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{*replacementAttempt} + + var found bool + for i := 0; i < len(tx.TxAttempts); i++ { + if tx.TxAttempts[i].ID == oldAttempt.ID { + tx.TxAttempts[i] = *replacementAttempt + found = true + } + } + if !found { + tx.TxAttempts = append(tx.TxAttempts, *replacementAttempt) + } return nil } From 9108232b69f72cebb1f7c816e802c2534b232e33 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 22 Jan 2024 10:45:48 -0500 Subject: [PATCH 58/74] remove txstore req of address state --- common/txmgr/address_state.go | 12 ++---------- common/txmgr/inmemory_store.go | 7 ++++++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index c54666eee2b..9f7fe5833f8 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -1,7 +1,6 @@ package txmgr import ( - "context" "fmt" "sync" "time" @@ -22,7 +21,6 @@ type AddressState[ ] struct { chainID CHAIN_ID fromAddress ADDR - txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] sync.RWMutex idempotencyKeyToTx map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] @@ -47,12 +45,11 @@ func NewAddressState[ chainID CHAIN_ID, fromAddress ADDR, maxUnstarted int, - txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + txs []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) (*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ chainID: chainID, fromAddress: fromAddress, - txStore: txStore, idempotencyKeyToTx: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), @@ -63,12 +60,7 @@ func NewAddressState[ allTransactions: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, } - // Load all transactions from persistent storage - ctx := context.Background() - txs, err := txStore.AllTransactions(ctx, as.fromAddress, as.chainID) - if err != nil { - return nil, fmt.Errorf("address_state: initialization: %w", err) - } + // Load all transactions supplied for i := 0; i < len(txs); i++ { tx := txs[i] switch tx.State { diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 2b88161fec9..2acbc24c2b8 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -79,6 +79,7 @@ func NewInMemoryStore[ SEQ types.Sequence, FEE feetypes.Fee, ]( + ctx context.Context, lggr logger.SugaredLogger, chainID CHAIN_ID, keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], @@ -99,7 +100,11 @@ func NewInMemoryStore[ return nil, fmt.Errorf("new_in_memory_store: %w", err) } for _, fromAddr := range addresses { - as, err := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](chainID, fromAddr, maxUnstarted, txStore) + txs, err := txStore.AllTransactions(ctx, fromAddr, chainID) + if err != nil { + return nil, fmt.Errorf("address_state: initialization: %w", err) + } + as, err := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](chainID, fromAddr, maxUnstarted, txs) if err != nil { return nil, fmt.Errorf("new_in_memory_store: %w", err) } From f44ea22aa2b095d4ed0de1475f08797878b74ec8 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 22 Jan 2024 10:55:44 -0500 Subject: [PATCH 59/74] use counts to initialize map sizes --- common/txmgr/address_state.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 9f7fe5833f8..a2dd8932ad7 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -47,17 +47,31 @@ func NewAddressState[ maxUnstarted int, txs []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) (*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { + // Count the number of transactions in each state to reduce the number of map resizes + counts := map[txmgrtypes.TxState]int{ + TxUnstarted: 0, + TxInProgress: 0, + TxUnconfirmed: 0, + TxConfirmedMissingReceipt: 0, + TxConfirmed: 0, + TxFatalError: 0, + } + for _, tx := range txs { + counts[tx.State]++ + } + as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ chainID: chainID, fromAddress: fromAddress, - idempotencyKeyToTx: map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + idempotencyKeyToTx: make(map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)), unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), inprogress: nil, - unconfirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - confirmedMissingReceipt: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - confirmed: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, - allTransactions: map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{}, + unconfirmed: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxUnconfirmed]), + confirmedMissingReceipt: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxConfirmedMissingReceipt]), + confirmed: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxConfirmed]), + allTransactions: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)), + fatalErrored: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxFatalError]), } // Load all transactions supplied From 041bf61cf42b392127175f63f788d5774e046cc3 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 22 Jan 2024 11:21:04 -0500 Subject: [PATCH 60/74] cleanup reliance on sequences --- common/txmgr/address_state.go | 29 ++++++--------------------- common/txmgr/inmemory_store.go | 28 ++++++-------------------- core/chains/evm/txmgr/evm_tx_store.go | 7 +++++-- 3 files changed, 17 insertions(+), 47 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index a2dd8932ad7..6619cb47398 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -152,23 +152,6 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTx return as.idempotencyKeyToTx[key] } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) LatestSequence() SEQ { - as.RLock() - defer as.RUnlock() - - var maxSeq SEQ - for _, tx := range as.allTransactions { - if tx.Sequence == nil { - continue - } - if (*tx.Sequence).Int64() > maxSeq.Int64() { - maxSeq = *tx.Sequence - } - } - - return maxSeq -} - func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MaxConfirmedSequence() SEQ { as.RLock() defer as.RUnlock() @@ -314,7 +297,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT // if txStates is empty then apply the filter to only the as.allTransactions map if len(txStates) == 0 { - return as.fetchTxsFromStorage(as.allTransactions, filter, txIDs...) + return as.fetchTxs(as.allTransactions, filter, txIDs...) } var txs []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] @@ -325,20 +308,20 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT txs = append(txs, *as.inprogress) } case TxUnconfirmed: - txs = append(txs, as.fetchTxsFromStorage(as.unconfirmed, filter, txIDs...)...) + txs = append(txs, as.fetchTxs(as.unconfirmed, filter, txIDs...)...) case TxConfirmedMissingReceipt: - txs = append(txs, as.fetchTxsFromStorage(as.confirmedMissingReceipt, filter, txIDs...)...) + txs = append(txs, as.fetchTxs(as.confirmedMissingReceipt, filter, txIDs...)...) case TxConfirmed: - txs = append(txs, as.fetchTxsFromStorage(as.confirmed, filter, txIDs...)...) + txs = append(txs, as.fetchTxs(as.confirmed, filter, txIDs...)...) case TxFatalError: - txs = append(txs, as.fetchTxsFromStorage(as.fatalErrored, filter, txIDs...)...) + txs = append(txs, as.fetchTxs(as.fatalErrored, filter, txIDs...)...) } } return txs } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxsFromStorage( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxs( txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, txIDs ...int64, diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 2acbc24c2b8..551e0c51077 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -206,28 +206,11 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check ///// // FindLatestSequence returns the latest sequence number for a given address -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (seq SEQ, err error) { - // query the persistent storage since this method only gets called when the broadcaster is starting up. - // It is used to initialize the in-memory sequence map in the broadcaster - // NOTE(jtw): should the nextSequenceMap be moved to the in-memory store? - - if ms.chainID.String() != chainID.String() { - return seq, fmt.Errorf("find_latest_sequence: %w", ErrInvalidChainID) - } - - ms.addressStatesLock.RLock() - defer ms.addressStatesLock.RUnlock() - as, ok := ms.addressStates[fromAddress] - if !ok { - return seq, fmt.Errorf("find_latest_sequence: %w", ErrAddressNotFound) - } - - seq = as.LatestSequence() - if seq.Int64() == 0 { - return seq, fmt.Errorf("find_latest_sequence: %w", ErrSequenceNotFound) - } - - return seq, nil +// It is used to initialize the in-memory sequence map in the broadcaster +// NOTE(jtw): this is until we have a abstracted Sequencer Component which can be used instead +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) { + // Query the persistent store + return ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) } // CountUnconfirmedTransactions returns the number of unconfirmed transactions for a given address. @@ -1734,6 +1717,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkA for _, as := range ms.addressStates { wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { + // TODO(jtw): THIS IS EVM SPECIFIC THIS SHOULD BE GENERALIZED maxConfirmedSequence := as.MaxConfirmedSequence() filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.Sequence == nil { diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 39c966e16ea..132ce2b38f2 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -1035,8 +1035,11 @@ func (o *evmTxStore) FindLatestSequence(ctx context.Context, fromAddress common. ctx, cancel = o.mergeContexts(ctx) defer cancel() qq := o.q.WithOpts(pg.WithParentCtx(ctx)) - sql := `SELECT nonce FROM evm.txes WHERE from_address = $1 AND evm_chain_id = $2 AND nonce IS NOT NULL ORDER BY nonce DESC LIMIT 1` - err = qq.Get(&nonce, sql, fromAddress, chainId.String()) + stmt := `SELECT nonce FROM evm.txes WHERE from_address = $1 AND evm_chain_id = $2 AND nonce IS NOT NULL ORDER BY nonce DESC LIMIT 1` + err = qq.Get(&nonce, stmt, fromAddress, chainId.String()) + if errors.Is(err, sql.ErrNoRows) { + return nonce, txmgr.ErrSequenceNotFound + } return } From 970db2b0c098313dc2ae46cabeaf9d1e2595e58e Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 22 Jan 2024 11:30:26 -0500 Subject: [PATCH 61/74] move MaxConfirmedSequence logic from addressState to caller --- common/txmgr/address_state.go | 17 ----------------- common/txmgr/inmemory_store.go | 22 ++++++++++++++++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 6619cb47398..79557cf13f5 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -152,23 +152,6 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindTx return as.idempotencyKeyToTx[key] } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MaxConfirmedSequence() SEQ { - as.RLock() - defer as.RUnlock() - - var maxSeq SEQ - for _, tx := range as.confirmed { - if tx.Sequence == nil { - continue - } - if (*tx.Sequence).Int64() > maxSeq.Int64() { - maxSeq = *tx.Sequence - } - } - - return maxSeq -} - func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ApplyToTxsByState( txStates []txmgrtypes.TxState, fn func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]), diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 551e0c51077..4cfcf484b94 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -1718,8 +1718,22 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkA wg.Add(1) go func(as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) { // TODO(jtw): THIS IS EVM SPECIFIC THIS SHOULD BE GENERALIZED - maxConfirmedSequence := as.MaxConfirmedSequence() - filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + // Get the max confirmed sequence + filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return true } + states := []txmgrtypes.TxState{TxConfirmed} + txs := as.FetchTxs(states, filter) + var maxConfirmedSequence SEQ + for _, tx := range txs { + if tx.Sequence == nil { + continue + } + if (*tx.Sequence).Int64() > maxConfirmedSequence.Int64() { + maxConfirmedSequence = *tx.Sequence + } + } + + // Mark all unconfirmed txs with a sequence less than the max confirmed sequence as confirmed_missing_receipt + filter = func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.Sequence == nil { return false } @@ -1729,8 +1743,8 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MarkA return (*tx.Sequence).Int64() < maxConfirmedSequence.Int64() } - states := []txmgrtypes.TxState{TxUnconfirmed} - txs := as.FetchTxs(states, filter) + states = []txmgrtypes.TxState{TxUnconfirmed} + txs = as.FetchTxs(states, filter) for _, tx := range txs { attempt := tx.TxAttempts[0] From b62a43eccae9e566313d2cc1fe40bedb40a67c2c Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 22 Jan 2024 11:52:16 -0500 Subject: [PATCH 62/74] cleanup --- common/txmgr/inmemory_store.go | 2 +- core/chains/evm/txmgr/evm_tx_store.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 4cfcf484b94..6158971413e 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -207,7 +207,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check // FindLatestSequence returns the latest sequence number for a given address // It is used to initialize the in-memory sequence map in the broadcaster -// NOTE(jtw): this is until we have a abstracted Sequencer Component which can be used instead +// TODO(jtw): this is until we have a abstracted Sequencer Component which can be used instead func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindLatestSequence(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) (SEQ, error) { // Query the persistent store return ms.txStore.FindLatestSequence(ctx, fromAddress, chainID) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index 132ce2b38f2..58181ce02fd 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -1759,8 +1759,7 @@ func (o *evmTxStore) HasInProgressTransaction(ctx context.Context, account commo ctx, cancel = o.mergeContexts(ctx) defer cancel() qq := o.q.WithOpts(pg.WithParentCtx(ctx)) - err = qq.Get(&exists, ` - SELECT EXISTS(SELECT 1 FROM evm.txes WHERE state = 'in_progress' AND from_address = $1 AND evm_chain_id = $2)`, account, chainID.String()) + err = qq.Get(&exists, `SELECT EXISTS(SELECT 1 FROM evm.txes WHERE state = 'in_progress' AND from_address = $1 AND evm_chain_id = $2)`, account, chainID.String()) return exists, pkgerrors.Wrap(err, "hasInProgressTransaction failed") } From 1709cae60f4bdb96d237a763c3a6ea1144c99a16 Mon Sep 17 00:00:00 2001 From: James Walker Date: Mon, 22 Jan 2024 14:54:26 -0500 Subject: [PATCH 63/74] small cleanup --- common/txmgr/inmemory_store.go | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 6158971413e..8aec5380c1d 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -40,20 +40,6 @@ var ( ErrCouldNotGetReceipt = errors.New("could not get receipt") ) -type PersistentTxStore[ - ADDR types.Hashable, - CHAIN_ID types.ID, - TX_HASH types.Hashable, - BLOCK_HASH types.Hashable, - R txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH], - SEQ types.Sequence, - FEE feetypes.Fee, -] interface { - txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] - - AllTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) -} - type InMemoryStore[ CHAIN_ID types.ID, ADDR, TX_HASH, BLOCK_HASH types.Hashable, @@ -65,7 +51,7 @@ type InMemoryStore[ chainID CHAIN_ID keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ] - txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE] addressStatesLock sync.RWMutex addressStates map[ADDR]*AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE] @@ -83,7 +69,7 @@ func NewInMemoryStore[ lggr logger.SugaredLogger, chainID CHAIN_ID, keyStore txmgrtypes.KeyStore[ADDR, CHAIN_ID, SEQ], - txStore PersistentTxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], + txStore txmgrtypes.TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE], ) (*InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE], error) { ms := InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ lggr: lggr, @@ -201,10 +187,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Check return nil } -///// -// BROADCASTER FUNCTIONS -///// - // FindLatestSequence returns the latest sequence number for a given address // It is used to initialize the in-memory sequence map in the broadcaster // TODO(jtw): this is until we have a abstracted Sequencer Component which can be used instead From 2274b2a66a897e59224d3dd55611d9f1b53cfa3e Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 25 Jan 2024 16:32:04 -0500 Subject: [PATCH 64/74] update pruning and create transaction work --- common/txmgr/address_state.go | 6 ++---- common/txmgr/inmemory_store.go | 32 +++++-------------------------- common/txmgr/tx_priority_queue.go | 20 ++++--------------- 3 files changed, 11 insertions(+), 47 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 79557cf13f5..5bf1df50e41 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -334,14 +334,12 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT return txs } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(queueSize uint32, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool) int { +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ids []int64) { as.Lock() defer as.Unlock() - txs := as.unstarted.Prune(int(queueSize), filter) + txs := as.unstarted.PruneByTxIDs(ids) as.deleteTxs(txs...) - - return len(txs) } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) DeleteTxs(txs ...txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) { diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 8aec5380c1d..72351734cd3 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -127,15 +127,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat return tx, fmt.Errorf("create_transaction: %w", err) } - // Prune the in-memory txs - pruned, err := txRequest.Strategy.PruneQueue(ctx, ms) - if err != nil { - return tx, fmt.Errorf("CreateTransaction failed to prune in-memory txs: %w", err) - } - if pruned > 0 { - ms.lggr.Warnf("Dropped %d old transactions from transaction queue", pruned) - } - // Add the request to the Unstarted channel to be processed by the Broadcaster if err := as.AddTxToUnstarted(&tx); err != nil { return *ms.deepCopyTx(tx), fmt.Errorf("create_transaction: %w", err) @@ -927,34 +918,21 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return txs, nil } -func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) (int64, error) { +func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneUnstartedTxQueue(ctx context.Context, queueSize uint32, subject uuid.UUID) ([]int64, error) { // Persist to persistent storage - n, err := ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) + ids, err := ms.txStore.PruneUnstartedTxQueue(ctx, queueSize, subject) if err != nil { - return 0, err + return ids, err } // Update in memory store - filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { - if !tx.Subject.Valid { - return false - } - - return tx.Subject.UUID == subject - } - var m int ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { - m += as.PruneUnstartedTxQueue(queueSize, filter) - } - - if n != int64(m) { - // TODO: WHAT SHOULD HAPPEN HERE IF THE COUNTS DON'T MATCH? - return n, fmt.Errorf("prune_unstarted_tx_queue: inmemory prune(%d) does not match persistence(%d) ", m, n) + as.PruneUnstartedTxQueue(ids) } - return n, nil + return ids, nil } func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapTxHistory(ctx context.Context, minBlockNumberToKeep int64, timeThreshold time.Time, chainID CHAIN_ID) error { diff --git a/common/txmgr/tx_priority_queue.go b/common/txmgr/tx_priority_queue.go index 5b644017fbb..a72d7422c62 100644 --- a/common/txmgr/tx_priority_queue.go +++ b/common/txmgr/tx_priority_queue.go @@ -66,28 +66,16 @@ func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Rem return nil } -// Prune removes the transactions that match the filter -func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Prune(maxUnstarted int, filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { +// PruneByTxIDs removes the transactions with the given IDs from the queue +func (pq *TxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) PruneByTxIDs(ids []int64) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { pq.Lock() defer pq.Unlock() - if len(pq.txs) <= maxUnstarted { - return nil - } - - // Remove all transactions that are oldest, unstarted, and match the filter removed := []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} - for i := 0; i < len(pq.txs); i++ { - tx := pq.txs[i] - if filter(tx) { + for _, id := range ids { + if tx := pq.RemoveTxByID(id); tx != nil { removed = append(removed, *tx) } - if len(pq.txs)-len(removed) <= maxUnstarted { - break - } - } - for _, tx := range removed { - pq.RemoveTxByID(tx.ID) } return removed From c31911cc97e4cc3694bfb891d43d71da4b3c6639 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 7 Feb 2024 15:49:14 -0500 Subject: [PATCH 65/74] update MoveUnstartedToInProgress --- common/txmgr/address_state.go | 34 ++++++++++++++++++++++++++-------- common/txmgr/inmemory_store.go | 8 +++----- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 5bf1df50e41..ab5e41344f3 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/smartcontractkit/chainlink-common/pkg/logger" feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" txmgrtypes "github.com/smartcontractkit/chainlink/v2/common/txmgr/types" "github.com/smartcontractkit/chainlink/v2/common/types" @@ -19,6 +20,7 @@ type AddressState[ SEQ types.Sequence, FEE feetypes.Fee, ] struct { + lggr logger.SugaredLogger chainID CHAIN_ID fromAddress ADDR @@ -42,6 +44,7 @@ func NewAddressState[ SEQ types.Sequence, FEE feetypes.Fee, ]( + lggr logger.SugaredLogger, chainID CHAIN_ID, fromAddress ADDR, maxUnstarted int, @@ -61,6 +64,7 @@ func NewAddressState[ } as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ + lggr: lggr, chainID: chainID, fromAddress: fromAddress, @@ -409,7 +413,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AddTxT } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnstartedToInProgress( - tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + etx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], + txAttempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) error { as.Lock() defer as.Unlock() @@ -418,19 +423,32 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUn return fmt.Errorf("move_unstarted_to_in_progress: address %s already has a transaction in progress", as.fromAddress) } - if tx != nil { - // if tx is not nil then remove the tx from the unstarted queue - tx = as.unstarted.RemoveTxByID(tx.ID) - } else { - // if tx is nil then pop the next unstarted transaction - tx = as.unstarted.RemoveNextTx() - } + tx := as.unstarted.RemoveTxByID(etx.ID) if tx == nil { return fmt.Errorf("move_unstarted_to_in_progress: no unstarted transaction to move to in_progress") } tx.State = TxInProgress as.inprogress = tx + newTxAttempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} + affectedTxAttempts := 0 + for i := 0; i < len(tx.TxAttempts); i++ { + // Remove any previous attempts that are in a fatal error state which share the same hash + if tx.TxAttempts[i].Hash == txAttempt.Hash && + tx.State == TxFatalError && tx.Error == null.NewString("abandoned", true) { + affectedTxAttempts++ + continue + } + newTxAttempts = append(newTxAttempts, tx.TxAttempts[i]) + } + if affectedTxAttempts > 0 { + as.lggr.Debugf("Replacing abandoned tx with tx hash %s with tx_id=%d with identical tx hash", txAttempt.Hash, txAttempt.TxID) + } + tx.TxAttempts = append(newTxAttempts, *txAttempt) + tx.Sequence = etx.Sequence + tx.BroadcastAt = etx.BroadcastAt + tx.InitialBroadcastAt = etx.InitialBroadcastAt + return nil } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 72351734cd3..41ca2496e47 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -90,7 +90,7 @@ func NewInMemoryStore[ if err != nil { return nil, fmt.Errorf("address_state: initialization: %w", err) } - as, err := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](chainID, fromAddr, maxUnstarted, txs) + as, err := NewAddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](lggr, chainID, fromAddr, maxUnstarted, txs) if err != nil { return nil, fmt.Errorf("new_in_memory_store: %w", err) } @@ -127,6 +127,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat return tx, fmt.Errorf("create_transaction: %w", err) } + // Update in memory store // Add the request to the Unstarted channel to be processed by the Broadcaster if err := as.AddTxToUnstarted(&tx); err != nil { return *ms.deepCopyTx(tx), fmt.Errorf("create_transaction: %w", err) @@ -223,7 +224,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Count } // UpdateTxUnstartedToInProgress updates a transaction from unstarted to in_progress. -// TODO THIS HAS SOME INCONSISTENCIES WITH THE PERSISTENT STORE func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) UpdateTxUnstartedToInProgress( ctx context.Context, tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], @@ -250,11 +250,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat if err := ms.txStore.UpdateTxUnstartedToInProgress(ctx, tx, attempt); err != nil { return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) } - tx.TxAttempts = append(tx.TxAttempts, *attempt) - // TODO: DOES THIS ATTEMPT HAVE AN ID? IF NOT, HOW DO WE GET IT? // Update in address state in memory - if err := as.MoveUnstartedToInProgress(tx); err != nil { + if err := as.MoveUnstartedToInProgress(tx, attempt); err != nil { return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) } From c2cb7bd06c7ebad60a5175bb74caca3e0e9a4fdd Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 7 Feb 2024 16:13:34 -0500 Subject: [PATCH 66/74] clean up MoveInProgressToUnconfirmed --- common/txmgr/address_state.go | 16 +++++++++++++--- common/txmgr/inmemory_store.go | 11 ++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index ab5e41344f3..fd8b11de247 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -471,6 +471,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveCo } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveInProgressToUnconfirmed( + etx txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], txAttempt txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], ) error { as.Lock() @@ -481,10 +482,19 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveIn return fmt.Errorf("move_in_progress_to_unconfirmed: no transaction in progress") } - txAttempt.TxID = tx.ID - txAttempt.State = txmgrtypes.TxAttemptBroadcast tx.State = TxUnconfirmed - tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{txAttempt} + tx.Error = etx.Error + tx.BroadcastAt = etx.BroadcastAt + tx.InitialBroadcastAt = etx.InitialBroadcastAt + txAttempt.State = txmgrtypes.TxAttemptBroadcast + txAttempt.TxID = tx.ID + + for i := 0; i < len(tx.TxAttempts); i++ { + if tx.TxAttempts[i].ID == txAttempt.ID { + tx.TxAttempts[i] = txAttempt + break + } + } as.unconfirmed[tx.ID] = tx as.inprogress = nil diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 41ca2496e47..5ee5dadd071 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -19,11 +19,6 @@ import ( "gopkg.in/guregu/null.v4" ) -// BIG TODO LIST -// TODO: figure out if multiple tx attempts are actually stored in the db for each tx -// TODO: need a way to get id for a tx attempt. since there are some methods where the persistent store creates a tx attempt and doesnt returns it -// TODO: should txAttempt state transitions be handled by the address state manager? - var ( // ErrInvalidChainID is returned when the chain ID is invalid ErrInvalidChainID = errors.New("invalid chain ID") @@ -316,9 +311,9 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat if err := ms.txStore.UpdateTxAttemptInProgressToBroadcast(ctx, tx, attempt, newAttemptState); err != nil { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } - attempt.State = newAttemptState - if err := as.MoveInProgressToUnconfirmed(attempt); err != nil { + // Update in memory store + if err := as.MoveInProgressToUnconfirmed(*tx, attempt); err != nil { return fmt.Errorf("update_tx_attempt_in_progress_to_broadcast: %w", err) } @@ -1207,7 +1202,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE return null.Time{}, fmt.Errorf("find_earliest_unconfirmed_broadcast_time: %w", ErrInvalidChainID) } - // TODO(jtw): this is super niave and might be slow filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return tx.InitialBroadcastAt != nil } @@ -1244,7 +1238,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE return null.Int{}, fmt.Errorf("find_earliest_unconfirmed_broadcast_time: %w", ErrInvalidChainID) } - // TODO(jtw): this is super niave and might be slow filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false From 19fbe957a6ae1732425a3488791bd5ec59ddc1b9 Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 8 Feb 2024 16:04:13 -0500 Subject: [PATCH 67/74] run make generate --- common/txmgr/types/mocks/tx_store.go | 30 ++++++ core/chains/evm/txmgr/mocks/evm_tx_store.go | 104 ++++++-------------- 2 files changed, 60 insertions(+), 74 deletions(-) diff --git a/common/txmgr/types/mocks/tx_store.go b/common/txmgr/types/mocks/tx_store.go index 353f398316d..368ded723b1 100644 --- a/common/txmgr/types/mocks/tx_store.go +++ b/common/txmgr/types/mocks/tx_store.go @@ -43,6 +43,36 @@ func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Abandon(ctx return r0 } +// AllTransactions provides a mock function with given fields: ctx, fromAddress, chainID +func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) AllTransactions(ctx context.Context, fromAddress ADDR, chainID CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error) { + ret := _m.Called(ctx, fromAddress, chainID) + + if len(ret) == 0 { + panic("no return value specified for AllTransactions") + } + + var r0 []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, ADDR, CHAIN_ID) ([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], error)); ok { + return rf(ctx, fromAddress, chainID) + } + if rf, ok := ret.Get(0).(func(context.Context, ADDR, CHAIN_ID) []txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]); ok { + r0 = rf(ctx, fromAddress, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, ADDR, CHAIN_ID) error); ok { + r1 = rf(ctx, fromAddress, chainID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // CheckTxQueueCapacity provides a mock function with given fields: ctx, fromAddress, maxQueuedTransactions, chainID func (_m *TxStore[ADDR, CHAIN_ID, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CheckTxQueueCapacity(ctx context.Context, fromAddress ADDR, maxQueuedTransactions uint64, chainID CHAIN_ID) error { ret := _m.Called(ctx, fromAddress, maxQueuedTransactions, chainID) diff --git a/core/chains/evm/txmgr/mocks/evm_tx_store.go b/core/chains/evm/txmgr/mocks/evm_tx_store.go index 391c58233db..06de8e0d0cd 100644 --- a/core/chains/evm/txmgr/mocks/evm_tx_store.go +++ b/core/chains/evm/txmgr/mocks/evm_tx_store.go @@ -46,6 +46,36 @@ func (_m *EvmTxStore) Abandon(ctx context.Context, id *big.Int, addr common.Addr return r0 } +// AllTransactions provides a mock function with given fields: ctx, fromAddress, chainID +func (_m *EvmTxStore) AllTransactions(ctx context.Context, fromAddress common.Address, chainID *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error) { + ret := _m.Called(ctx, fromAddress, chainID) + + if len(ret) == 0 { + panic("no return value specified for AllTransactions") + } + + var r0 []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], error)); ok { + return rf(ctx, fromAddress, chainID) + } + if rf, ok := ret.Get(0).(func(context.Context, common.Address, *big.Int) []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { + r0 = rf(ctx, fromAddress, chainID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, common.Address, *big.Int) error); ok { + r1 = rf(ctx, fromAddress, chainID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // CheckTxQueueCapacity provides a mock function with given fields: ctx, fromAddress, maxQueuedTransactions, chainID func (_m *EvmTxStore) CheckTxQueueCapacity(ctx context.Context, fromAddress common.Address, maxQueuedTransactions uint64, chainID *big.Int) error { ret := _m.Called(ctx, fromAddress, maxQueuedTransactions, chainID) @@ -1342,80 +1372,6 @@ func (_m *EvmTxStore) TxAttempts(offset int, limit int) ([]types.TxAttempt[*big. return r0, r1, r2 } -// UnconfirmedTransactions provides a mock function with given fields: limit, offset, fromAddress, chainID -func (_m *EvmTxStore) UnconfirmedTransactions(limit int, offset int, fromAddress common.Address, chainID *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error) { - ret := _m.Called(limit, offset, fromAddress, chainID) - - if len(ret) == 0 { - panic("no return value specified for UnconfirmedTransactions") - } - - var r0 []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] - var r1 int - var r2 error - if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error)); ok { - return rf(limit, offset, fromAddress, chainID) - } - if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { - r0 = rf(limit, offset, fromAddress, chainID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) - } - } - - if rf, ok := ret.Get(1).(func(int, int, common.Address, *big.Int) int); ok { - r1 = rf(limit, offset, fromAddress, chainID) - } else { - r1 = ret.Get(1).(int) - } - - if rf, ok := ret.Get(2).(func(int, int, common.Address, *big.Int) error); ok { - r2 = rf(limit, offset, fromAddress, chainID) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// UnstartedTransactions provides a mock function with given fields: limit, offset, fromAddress, chainID -func (_m *EvmTxStore) UnstartedTransactions(limit int, offset int, fromAddress common.Address, chainID *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error) { - ret := _m.Called(limit, offset, fromAddress, chainID) - - if len(ret) == 0 { - panic("no return value specified for UnstartedTransactions") - } - - var r0 []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee] - var r1 int - var r2 error - if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) ([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee], int, error)); ok { - return rf(limit, offset, fromAddress, chainID) - } - if rf, ok := ret.Get(0).(func(int, int, common.Address, *big.Int) []types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]); ok { - r0 = rf(limit, offset, fromAddress, chainID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]types.Tx[*big.Int, common.Address, common.Hash, common.Hash, evmtypes.Nonce, gas.EvmFee]) - } - } - - if rf, ok := ret.Get(1).(func(int, int, common.Address, *big.Int) int); ok { - r1 = rf(limit, offset, fromAddress, chainID) - } else { - r1 = ret.Get(1).(int) - } - - if rf, ok := ret.Get(2).(func(int, int, common.Address, *big.Int) error); ok { - r2 = rf(limit, offset, fromAddress, chainID) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - // UpdateBroadcastAts provides a mock function with given fields: ctx, now, etxIDs func (_m *EvmTxStore) UpdateBroadcastAts(ctx context.Context, now time.Time, etxIDs []int64) error { ret := _m.Called(ctx, now, etxIDs) From dac61147097f08d8637a6370ead4551c27e0314c Mon Sep 17 00:00:00 2001 From: James Walker Date: Thu, 8 Feb 2024 16:04:53 -0500 Subject: [PATCH 68/74] fix bug where used txFromAddress instead of txRequestFromAddress --- common/txmgr/inmemory_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 5ee5dadd071..e1a2a07a495 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -111,7 +111,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Creat ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() - as, ok := ms.addressStates[tx.FromAddress] + as, ok := ms.addressStates[txRequest.FromAddress] if !ok { return tx, fmt.Errorf("create_transaction: %w", ErrAddressNotFound) } From 1c9ec0d88e71e6d5f057bf4c7b737e65ca5bb351 Mon Sep 17 00:00:00 2001 From: James Walker Date: Wed, 14 Feb 2024 17:55:10 -0500 Subject: [PATCH 69/74] address initial comments --- common/txmgr/address_state.go | 44 +++++++++++------ common/txmgr/inmemory_store.go | 90 +++++++++++++++++----------------- 2 files changed, 76 insertions(+), 58 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index fd8b11de247..0c50dba1961 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -63,6 +63,8 @@ func NewAddressState[ counts[tx.State]++ } + // TODO: MAKE BETTER + // nit: probably not a big deal but not all txs have an idempotency key so we're probably initializing this map bigger than it needs to be here. as := AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]{ lggr: lggr, chainID: chainID, @@ -212,7 +214,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) applyT func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchTxAttempts( txStates []txmgrtypes.TxState, - filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txFilter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txAttemptFilter func(*txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, txIDs ...int64, ) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { as.RLock() @@ -220,24 +223,28 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT // if txStates is empty then apply the filter to only the as.allTransactions map if len(txStates) == 0 { - return as.fetchTxAttemptsFromStorage(as.allTransactions, filter, txIDs...) + return as.fetchTxAttemptsFromStorage(as.allTransactions, txFilter, txAttemptFilter, txIDs...) } var txAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] for _, txState := range txStates { switch txState { case TxInProgress: - if as.inprogress != nil && filter(as.inprogress) { - txAttempts = append(txAttempts, as.inprogress.TxAttempts...) + if as.inprogress != nil && txFilter(as.inprogress) { + for _, txAttempt := range as.inprogress.TxAttempts { + if txAttemptFilter(&txAttempt) { + txAttempts = append(txAttempts, txAttempt) + } + } } case TxUnconfirmed: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.unconfirmed, filter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.unconfirmed, txFilter, txAttemptFilter, txIDs...)...) case TxConfirmedMissingReceipt: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmedMissingReceipt, filter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmedMissingReceipt, txFilter, txAttemptFilter, txIDs...)...) case TxConfirmed: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmed, filter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmed, txFilter, txAttemptFilter, txIDs...)...) case TxFatalError: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.fatalErrored, filter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.fatalErrored, txFilter, txAttemptFilter, txIDs...)...) } } @@ -246,7 +253,8 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxAttemptsFromStorage( txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], - filter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txFilter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, + txAttemptFilter func(*txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, txIDs ...int64, ) []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] { as.RLock() @@ -257,8 +265,12 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT if len(txIDs) > 0 { for _, txID := range txIDs { tx := txIDsToTx[txID] - if tx != nil && filter(tx) { - txAttempts = append(txAttempts, tx.TxAttempts...) + if tx != nil && txFilter(tx) { + for _, txAttempt := range tx.TxAttempts { + if txAttemptFilter(&txAttempt) { + txAttempts = append(txAttempts, txAttempt) + } + } } } return txAttempts @@ -266,8 +278,12 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchT // if txIDs is empty then apply the filter to all transactions for _, tx := range txIDsToTx { - if filter(tx) { - txAttempts = append(txAttempts, tx.TxAttempts...) + if txFilter(tx) { + for _, txAttempt := range tx.TxAttempts { + if txAttemptFilter(&txAttempt) { + txAttempts = append(txAttempts, txAttempt) + } + } } } @@ -365,9 +381,9 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) delete delete(as.allTransactions, txID) delete(as.unconfirmed, txID) delete(as.confirmedMissingReceipt, txID) - delete(as.allTransactions, txID) delete(as.confirmed, txID) delete(as.fatalErrored, txID) + as.unstarted.RemoveTxByID(txID) } } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index e1a2a07a495..351a05b83ff 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -332,11 +332,6 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindN return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrAddressNotFound) } - // ensure that the address is not already busy with a transaction in progress - if as.inprogress != nil { - return fmt.Errorf("find_next_unstarted_transaction_from_address: address %s is already busy with a transaction in progress", fromAddress) - } - etx, err := as.PeekNextUnstartedTx() if err != nil || etx == nil { return fmt.Errorf("find_next_unstarted_transaction_from_address: %w", err) @@ -505,18 +500,28 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_next_unstarted_transaction_from_address: %w", ErrInvalidChainID) } - filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + txFilter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return tx.TxAttempts != nil && len(tx.TxAttempts) > 0 } + txAttemptFilter := func(attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return true + } states := []txmgrtypes.TxState{TxConfirmedMissingReceipt} attempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { - attempts = append(attempts, as.FetchTxAttempts(states, filter)...) + attempts = append(attempts, as.FetchTxAttempts(states, txFilter, txAttemptFilter)...) } + // TODO: FINISH THIS // sort by tx_id ASC, gas_price DESC, gas_tip_cap DESC sort.SliceStable(attempts, func(i, j int) bool { + /* + if attempts[i].TxID == attempts[j].TxID { + // sort by gas_price DESC + } + */ + return attempts[i].TxID < attempts[j].TxID }) @@ -588,20 +593,18 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return attempts, fmt.Errorf("find_tx_attempts_requiring_receipt_fetch: %w", ErrInvalidChainID) } - filterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { - if tx.TxAttempts != nil && len(tx.TxAttempts) > 0 { - attempt := tx.TxAttempts[0] - return attempt.State != txmgrtypes.TxAttemptInsufficientFunds - } - - return false + txFilterFn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.TxAttempts != nil && len(tx.TxAttempts) > 0 + } + txAttemptFilterFn := func(attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return attempt.State != txmgrtypes.TxAttemptInsufficientFunds } states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} attempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { - attempts = append(attempts, as.FetchTxAttempts(states, filterFn)...) + attempts = append(attempts, as.FetchTxAttempts(states, txFilterFn, txAttemptFilterFn)...) } // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC sort.Slice(attempts, func(i, j int) bool { @@ -1088,22 +1091,17 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return nil, fmt.Errorf("find_tx_attempts_requiring_resend: %w", ErrAddressNotFound) } - filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + txFilter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false } - attempt := tx.TxAttempts[0] - if attempt.State == txmgrtypes.TxAttemptInProgress { - return false - } - if tx.BroadcastAt.After(olderThan) { - return false - } - - return false + return tx.BroadcastAt.Before(olderThan) || tx.BroadcastAt.Equal(olderThan) + } + txAttemptFilter := func(attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return attempt.State != txmgrtypes.TxAttemptInProgress } states := []txmgrtypes.TxState{TxUnconfirmed, TxConfirmedMissingReceipt} - attempts := as.FetchTxAttempts(states, filter) + attempts := as.FetchTxAttempts(states, txFilter, txAttemptFilter) // sort by sequence ASC, gas_price DESC, gas_tip_cap DESC sort.Slice(attempts, func(i, j int) bool { return (*attempts[i].Tx.Sequence).Int64() < (*attempts[j].Tx.Sequence).Int64() @@ -1285,15 +1283,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) GetIn return nil, fmt.Errorf("get_in_progress_tx_attempts: %w", ErrAddressNotFound) } - filter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { - if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { - return false - } - attempt := tx.TxAttempts[0] + txFilter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return tx.TxAttempts != nil && len(tx.TxAttempts) > 0 + } + txAttemptFilter := func(attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { return attempt.State == txmgrtypes.TxAttemptInProgress } states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt, TxUnconfirmed} - attempts := as.FetchTxAttempts(states, filter) + attempts := as.FetchTxAttempts(states, txFilter, txAttemptFilter) // deep copy the attempts var eAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] @@ -1567,27 +1564,32 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) IsTxF return false, fmt.Errorf("is_tx_finalized: %w", ErrInvalidChainID) } - fn := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + txFilter := func(tx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { if tx.ID != txID { return false } - if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { - return false - } - attempt := tx.TxAttempts[0] - if attempt.Receipts == nil || len(attempt.Receipts) == 0 { - return false - } - if attempt.Receipts[0].GetBlockNumber() == nil { - return false + + for _, attempt := range tx.TxAttempts { + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + continue + } + // there can only be one receipt per attempt + if attempt.Receipts[0].GetBlockNumber() == nil { + continue + } + + return attempt.Receipts[0].GetBlockNumber().Int64() <= (blockHeight - int64(tx.MinConfirmations.Uint32)) } - return attempt.Receipts[0].GetBlockNumber().Int64() <= (blockHeight - int64(tx.MinConfirmations.Uint32)) + return false + } + txAttemptFilter := func(attempt *txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool { + return attempt.Receipts != nil && len(attempt.Receipts) > 0 } ms.addressStatesLock.RLock() defer ms.addressStatesLock.RUnlock() for _, as := range ms.addressStates { - txas := as.FetchTxAttempts(nil, fn, txID) + txas := as.FetchTxAttempts(nil, txFilter, txAttemptFilter, txID) if len(txas) > 0 { return true, nil } From fb8be7c3ac31405fab339b755532aeb63c57212b Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 16 Feb 2024 12:33:38 -0500 Subject: [PATCH 70/74] fix some txAttempt loops --- common/txmgr/inmemory_store.go | 102 ++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 351a05b83ff..74c54ec1e50 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -633,6 +633,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT return false } + // TODO: loop through all attempts since any of them can have a receipt if tx.TxAttempts[0].Receipts == nil || len(tx.TxAttempts[0].Receipts) == 0 { return false } @@ -845,15 +846,18 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false } - attempt := tx.TxAttempts[0] - if attempt.Receipts == nil || len(attempt.Receipts) == 0 { - return false - } - if attempt.Receipts[0].GetBlockNumber() == nil { - return false + + for _, attempt := range tx.TxAttempts { + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + continue + } + if attempt.Receipts[0].GetBlockNumber() == nil { + continue + } + return attempt.Receipts[0].GetBlockNumber().Int64() >= blockNum } - return attempt.Receipts[0].GetBlockNumber().Int64() >= blockNum + return false } txsLock := sync.Mutex{} @@ -947,23 +951,22 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) ReapT if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false } - attempt := tx.TxAttempts[0] - if attempt.Receipts == nil || len(attempt.Receipts) == 0 { - return false - } - if attempt.Receipts[0].GetBlockNumber() == nil { - return false - } - if attempt.Receipts[0].GetBlockNumber().Int64() >= minBlockNumberToKeep { - return false - } - if tx.CreatedAt.After(timeThreshold) { - return false - } - if tx.State != TxConfirmed { - return false + for _, attempt := range tx.TxAttempts { + if attempt.Receipts == nil || len(attempt.Receipts) == 0 { + continue + } + if attempt.Receipts[0].GetBlockNumber() == nil { + continue + } + if attempt.Receipts[0].GetBlockNumber().Int64() >= minBlockNumberToKeep { + continue + } + if tx.CreatedAt.After(timeThreshold) { + continue + } + return tx.State == TxConfirmed } - return true + return false } wg := sync.WaitGroup{} @@ -1037,7 +1040,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Delet return false } - return tx.TxAttempts[0].ID == attempt.ID + for _, a := range tx.TxAttempts { + if a.ID == attempt.ID { + return true + } + } + return false } as.DeleteTxs(as.FetchTxs(nil, filter)...) @@ -1060,9 +1068,12 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false } - attempt := tx.TxAttempts[0] - - return attempt.State == txmgrtypes.TxAttemptInsufficientFunds + for _, attempt := range tx.TxAttempts { + if attempt.State == txmgrtypes.TxAttemptInsufficientFunds { + return true + } + } + return false } states := []txmgrtypes.TxState{TxUnconfirmed} txs := as.FetchTxs(states, filter) @@ -1153,18 +1164,23 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindT if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false } - attempt := tx.TxAttempts[0] - if attempt.State != txmgrtypes.TxAttemptBroadcast { - return false - } - if len(attempt.Receipts) == 0 { - return false - } - if attempt.Receipts[0].GetBlockNumber() == nil { - return false + for _, attempt := range tx.TxAttempts { + if attempt.State != txmgrtypes.TxAttemptBroadcast { + continue + } + if len(attempt.Receipts) == 0 { + continue + } + if attempt.Receipts[0].GetBlockNumber() == nil { + continue + } + blockNum := attempt.Receipts[0].GetBlockNumber().Int64() + if blockNum >= lowBlockNumber && blockNum <= highBlockNumber { + return true + } } - blockNum := attempt.Receipts[0].GetBlockNumber().Int64() - return blockNum >= lowBlockNumber && blockNum <= highBlockNumber + + return false } states := []txmgrtypes.TxState{TxConfirmed, TxConfirmedMissingReceipt} txsLock := sync.Mutex{} @@ -1240,8 +1256,14 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FindE if tx.TxAttempts == nil || len(tx.TxAttempts) == 0 { return false } - attempt := tx.TxAttempts[0] - return attempt.BroadcastBeforeBlockNum != nil + + for _, attempt := range tx.TxAttempts { + if attempt.BroadcastBeforeBlockNum != nil { + return true + } + } + + return false } states := []txmgrtypes.TxState{TxUnconfirmed} txsLock := sync.Mutex{} From 526981dd3259274f2e752b5d8fa6e56017efb75a Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 16 Feb 2024 15:30:24 -0500 Subject: [PATCH 71/74] address some comments --- common/txmgr/address_state.go | 94 +++++++++++++++++-------------- common/txmgr/broadcaster.go | 2 +- common/txmgr/inmemory_store.go | 1 + common/txmgr/tx_priority_queue.go | 2 + common/txmgr/txmgr.go | 3 +- 5 files changed, 59 insertions(+), 43 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index 0c50dba1961..c39bd2ac80d 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -34,6 +34,8 @@ type AddressState[ confirmed map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] allTransactions map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] fatalErrored map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] + // TODO: FINISH populate attemptHashToTxAttempt + attemptHashToTxAttempt map[TX_HASH]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } // NewAddressState returns a new AddressState instance @@ -59,8 +61,12 @@ func NewAddressState[ TxConfirmed: 0, TxFatalError: 0, } + idempotencyKeysCount := 0 for _, tx := range txs { counts[tx.State]++ + if tx.IdempotencyKey != nil { + idempotencyKeysCount++ + } } // TODO: MAKE BETTER @@ -70,7 +76,7 @@ func NewAddressState[ chainID: chainID, fromAddress: fromAddress, - idempotencyKeyToTx: make(map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)), + idempotencyKeyToTx: make(map[string]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], idempotencyKeysCount), unstarted: NewTxPriorityQueue[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE](maxUnstarted), inprogress: nil, unconfirmed: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxUnconfirmed]), @@ -223,7 +229,7 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT // if txStates is empty then apply the filter to only the as.allTransactions map if len(txStates) == 0 { - return as.fetchTxAttemptsFromStorage(as.allTransactions, txFilter, txAttemptFilter, txIDs...) + return as.fetchTxAttempts(as.allTransactions, txFilter, txAttemptFilter, txIDs...) } var txAttempts []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] @@ -238,20 +244,20 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) FetchT } } case TxUnconfirmed: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.unconfirmed, txFilter, txAttemptFilter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttempts(as.unconfirmed, txFilter, txAttemptFilter, txIDs...)...) case TxConfirmedMissingReceipt: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmedMissingReceipt, txFilter, txAttemptFilter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttempts(as.confirmedMissingReceipt, txFilter, txAttemptFilter, txIDs...)...) case TxConfirmed: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.confirmed, txFilter, txAttemptFilter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttempts(as.confirmed, txFilter, txAttemptFilter, txIDs...)...) case TxFatalError: - txAttempts = append(txAttempts, as.fetchTxAttemptsFromStorage(as.fatalErrored, txFilter, txAttemptFilter, txIDs...)...) + txAttempts = append(txAttempts, as.fetchTxAttempts(as.fatalErrored, txFilter, txAttemptFilter, txIDs...)...) } } return txAttempts } -func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxAttemptsFromStorage( +func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) fetchTxAttempts( txIDsToTx map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], txFilter func(*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, txAttemptFilter func(*txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) bool, @@ -444,27 +450,13 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUn return fmt.Errorf("move_unstarted_to_in_progress: no unstarted transaction to move to in_progress") } tx.State = TxInProgress - as.inprogress = tx - - newTxAttempts := []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} - affectedTxAttempts := 0 - for i := 0; i < len(tx.TxAttempts); i++ { - // Remove any previous attempts that are in a fatal error state which share the same hash - if tx.TxAttempts[i].Hash == txAttempt.Hash && - tx.State == TxFatalError && tx.Error == null.NewString("abandoned", true) { - affectedTxAttempts++ - continue - } - newTxAttempts = append(newTxAttempts, tx.TxAttempts[i]) - } - if affectedTxAttempts > 0 { - as.lggr.Debugf("Replacing abandoned tx with tx hash %s with tx_id=%d with identical tx hash", txAttempt.Hash, txAttempt.TxID) - } - tx.TxAttempts = append(newTxAttempts, *txAttempt) + tx.TxAttempts = []txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{*txAttempt} tx.Sequence = etx.Sequence tx.BroadcastAt = etx.BroadcastAt tx.InitialBroadcastAt = etx.InitialBroadcastAt + as.inprogress = tx + return nil } @@ -524,28 +516,48 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUn as.Lock() defer as.Unlock() - for _, tx := range as.unconfirmed { - if tx.TxAttempts == nil { - continue - } - for i := 0; i < len(tx.TxAttempts); i++ { - txAttempt := tx.TxAttempts[i] - if receipt.GetTxHash() == txAttempt.Hash { - // TODO(jtw): not sure how to set blocknumber, transactionindex, and receipt on conflict - txAttempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} - txAttempt.State = txmgrtypes.TxAttemptBroadcast - if txAttempt.BroadcastBeforeBlockNum == nil { - blockNum := receipt.GetBlockNumber().Int64() - txAttempt.BroadcastBeforeBlockNum = &blockNum - } + /* + for _, tx := range as.unconfirmed { + if tx.TxAttempts == nil { + continue + } + for i := 0; i < len(tx.TxAttempts); i++ { + txAttempt := tx.TxAttempts[i] + if receipt.GetTxHash() == txAttempt.Hash { + // TODO(jtw): not sure how to set blocknumber, transactionindex, and receipt on conflict + txAttempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} + txAttempt.State = txmgrtypes.TxAttemptBroadcast + if txAttempt.BroadcastBeforeBlockNum == nil { + blockNum := receipt.GetBlockNumber().Int64() + txAttempt.BroadcastBeforeBlockNum = &blockNum + } - tx.State = TxConfirmed - return nil + tx.State = TxConfirmed + return nil + } } } + */ + txAttempt, ok := as.attemptHashToTxAttempt[receipt.GetTxHash()] + if !ok { + return fmt.Errorf("move_unconfirmed_to_confirmed: no unconfirmed transaction with receipt %v: %w", receipt, ErrTxnNotFound) + } + // TODO(jtw): not sure how to set blocknumber, transactionindex, and receipt on conflict + txAttempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} + txAttempt.State = txmgrtypes.TxAttemptBroadcast + if txAttempt.BroadcastBeforeBlockNum == nil { + blockNum := receipt.GetBlockNumber().Int64() + txAttempt.BroadcastBeforeBlockNum = &blockNum } + tx, ok := as.unconfirmed[txAttempt.TxID] + if !ok { + // TODO: WHAT SHOULD WE DO HERE? + // THIS WOULD BE A BIG BUG + return fmt.Errorf("move_unconfirmed_to_confirmed: no unconfirmed transaction with ID %d: %w", txAttempt.TxID, ErrTxnNotFound) + } + tx.State = TxConfirmed - return fmt.Errorf("move_unconfirmed_to_confirmed: no unconfirmed transaction with receipt %v: %w", receipt, ErrTxnNotFound) + return nil } func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUnstartedToFatalError( diff --git a/common/txmgr/broadcaster.go b/common/txmgr/broadcaster.go index 0eb45e3b37c..9f2204f37e2 100644 --- a/common/txmgr/broadcaster.go +++ b/common/txmgr/broadcaster.go @@ -709,7 +709,7 @@ func (eb *Broadcaster[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]) next defer cancel() etx := &txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE]{} if err := eb.txStore.FindNextUnstartedTransactionFromAddress(ctx, etx, fromAddress, eb.chainID); err != nil { - if errors.Is(err, ErrTxnNotFound) { + if errors.Is(err, sql.ErrNoRows) { // Finish. No more transactions left to process. Hoorah! return nil, nil } diff --git a/common/txmgr/inmemory_store.go b/common/txmgr/inmemory_store.go index 74c54ec1e50..c62fb3070ff 100644 --- a/common/txmgr/inmemory_store.go +++ b/common/txmgr/inmemory_store.go @@ -246,6 +246,7 @@ func (ms *InMemoryStore[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) Updat return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) } + // TODO: REDO THIS AND TAKE SOME OF THE LOGIC OUT OF MOVE UNSTARTED TO IN PROGRESS // Update in address state in memory if err := as.MoveUnstartedToInProgress(tx, attempt); err != nil { return fmt.Errorf("update_tx_unstarted_to_in_progress: %w", err) diff --git a/common/txmgr/tx_priority_queue.go b/common/txmgr/tx_priority_queue.go index a72d7422c62..aeb07969a70 100644 --- a/common/txmgr/tx_priority_queue.go +++ b/common/txmgr/tx_priority_queue.go @@ -9,6 +9,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/common/types" ) +// TODO: HIDE THE HEAP INTERFACE FROM THE USER + // TxPriorityQueue is a priority queue of transactions prioritized by creation time. The oldest transaction is at the front of the queue. type TxPriorityQueue[ CHAIN_ID types.ID, diff --git a/common/txmgr/txmgr.go b/common/txmgr/txmgr.go index c062931883e..3e3fa9a20db 100644 --- a/common/txmgr/txmgr.go +++ b/common/txmgr/txmgr.go @@ -2,6 +2,7 @@ package txmgr import ( "context" + "database/sql" "errors" "fmt" "math/big" @@ -486,7 +487,7 @@ func (b *Txm[CHAIN_ID, HEAD, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) CreateTran if txRequest.IdempotencyKey != nil { var existingTx *txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] existingTx, err = b.txStore.FindTxWithIdempotencyKey(ctx, *txRequest.IdempotencyKey, b.chainID) - if err != nil && !errors.Is(err, ErrTxnNotFound) { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return tx, fmt.Errorf("Failed to search for transaction with IdempotencyKey: %w", err) } if existingTx != nil { From a6c68c522940930116493e6e39bae6480e6732f9 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 16 Feb 2024 17:09:53 -0500 Subject: [PATCH 72/74] update address_state --- common/txmgr/address_state.go | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/common/txmgr/address_state.go b/common/txmgr/address_state.go index c39bd2ac80d..9c2a4a52ece 100644 --- a/common/txmgr/address_state.go +++ b/common/txmgr/address_state.go @@ -35,6 +35,7 @@ type AddressState[ allTransactions map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] fatalErrored map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] // TODO: FINISH populate attemptHashToTxAttempt + // TODO: ANY NEW ATTEMPTS NEED TO BE ADDED TO THIS MAP attemptHashToTxAttempt map[TX_HASH]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE] } @@ -61,12 +62,16 @@ func NewAddressState[ TxConfirmed: 0, TxFatalError: 0, } - idempotencyKeysCount := 0 + var idempotencyKeysCount int + var txAttemptCount int for _, tx := range txs { counts[tx.State]++ if tx.IdempotencyKey != nil { idempotencyKeysCount++ } + if tx.State == TxUnconfirmed { + txAttemptCount += len(tx.TxAttempts) + } } // TODO: MAKE BETTER @@ -84,6 +89,7 @@ func NewAddressState[ confirmed: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxConfirmed]), allTransactions: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], len(txs)), fatalErrored: make(map[int64]*txmgrtypes.Tx[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], counts[TxFatalError]), + attemptHashToTxAttempt: make(map[TX_HASH]txmgrtypes.TxAttempt[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, SEQ, FEE], txAttemptCount), } // Load all transactions supplied @@ -107,6 +113,9 @@ func NewAddressState[ if tx.IdempotencyKey != nil { as.idempotencyKeyToTx[*tx.IdempotencyKey] = &tx } + for _, txAttempt := range tx.TxAttempts { + as.attemptHashToTxAttempt[txAttempt.Hash] = txAttempt + } } return &as, nil @@ -516,28 +525,6 @@ func (as *AddressState[CHAIN_ID, ADDR, TX_HASH, BLOCK_HASH, R, SEQ, FEE]) MoveUn as.Lock() defer as.Unlock() - /* - for _, tx := range as.unconfirmed { - if tx.TxAttempts == nil { - continue - } - for i := 0; i < len(tx.TxAttempts); i++ { - txAttempt := tx.TxAttempts[i] - if receipt.GetTxHash() == txAttempt.Hash { - // TODO(jtw): not sure how to set blocknumber, transactionindex, and receipt on conflict - txAttempt.Receipts = []txmgrtypes.ChainReceipt[TX_HASH, BLOCK_HASH]{receipt} - txAttempt.State = txmgrtypes.TxAttemptBroadcast - if txAttempt.BroadcastBeforeBlockNum == nil { - blockNum := receipt.GetBlockNumber().Int64() - txAttempt.BroadcastBeforeBlockNum = &blockNum - } - - tx.State = TxConfirmed - return nil - } - } - } - */ txAttempt, ok := as.attemptHashToTxAttempt[receipt.GetTxHash()] if !ok { return fmt.Errorf("move_unconfirmed_to_confirmed: no unconfirmed transaction with receipt %v: %w", receipt, ErrTxnNotFound) From aae832fd16cd399151eb29979b3a3b3cb11f0eb9 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 16 Feb 2024 17:12:25 -0500 Subject: [PATCH 73/74] add back sql ErrNoRows --- core/chains/evm/txmgr/evm_tx_store_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/txmgr/evm_tx_store_test.go b/core/chains/evm/txmgr/evm_tx_store_test.go index c5553c19d90..35d684727d1 100644 --- a/core/chains/evm/txmgr/evm_tx_store_test.go +++ b/core/chains/evm/txmgr/evm_tx_store_test.go @@ -1,6 +1,7 @@ package txmgr_test import ( + "database/sql" "fmt" "math/big" "testing" @@ -1262,7 +1263,7 @@ func TestORM_FindNextUnstartedTransactionFromAddress(t *testing.T) { resultEtx := new(txmgr.Tx) err := txStore.FindNextUnstartedTransactionFromAddress(testutils.Context(t), resultEtx, fromAddress, ethClient.ConfiguredChainID()) - assert.ErrorIs(t, err, txmgrcommon.ErrTxnNotFound) + assert.ErrorIs(t, err, sql.ErrNoRows) }) t.Run("finds unstarted tx", func(t *testing.T) { From 43e821190e8ff321064ecab07f9736c055b5f2d5 Mon Sep 17 00:00:00 2001 From: James Walker Date: Fri, 16 Feb 2024 17:14:23 -0500 Subject: [PATCH 74/74] add back sql ErrNoRows to evm_tx_store --- core/chains/evm/txmgr/evm_tx_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/txmgr/evm_tx_store.go b/core/chains/evm/txmgr/evm_tx_store.go index a3aad5480dd..7dadc45e840 100644 --- a/core/chains/evm/txmgr/evm_tx_store.go +++ b/core/chains/evm/txmgr/evm_tx_store.go @@ -1579,7 +1579,7 @@ func (o *evmTxStore) FindNextUnstartedTransactionFromAddress(ctx context.Context err := qq.Get(&dbEtx, `SELECT * FROM evm.txes WHERE from_address = $1 AND state = 'unstarted' AND evm_chain_id = $2 ORDER BY value ASC, created_at ASC, id ASC`, fromAddress, chainID.String()) if err != nil { if errors.Is(err, sql.ErrNoRows) { - return txmgr.ErrTxnNotFound + return sql.ErrNoRows } return pkgerrors.Wrap(err, "failed to FindNextUnstartedTransactionFromAddress") }