Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Classify Arbitrum rpc server errors #15488

Merged
merged 8 commits into from
Dec 12, 2024
Merged

Classify Arbitrum rpc server errors #15488

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0ec3e83
Add service errors
DylanTinianov Dec 3, 2024
5f70f49
Merge branch 'develop' of https://github.com/smartcontractkit/chainli…
DylanTinianov Dec 5, 2024
f67bf9d
Add default service errors
DylanTinianov Dec 11, 2024
029654e
Fix tests
DylanTinianov Dec 11, 2024
8e1cfe2
Update giant-eels-jump.md
DylanTinianov Dec 11, 2024
af35fe1
Merge branch 'develop' of https://github.com/smartcontractkit/chainli…
DylanTinianov Dec 11, 2024
96f035e
Merge branch 'BCFR-1079-rpc-internal-timeouts' of https://github.com/…
DylanTinianov Dec 11, 2024
f2d65b1
Update errors.go
DylanTinianov Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/giant-eels-jump.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink": patch
---

Add error handling for Arbitrum RPC server timeouts. #added
18 changes: 15 additions & 3 deletions core/chains/evm/client/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ const (
ServiceUnavailable
TerminallyStuck
TooManyResults
ServiceTimeout
)

type ClientErrors map[int]*regexp.Regexp
Expand Down Expand Up @@ -160,7 +161,7 @@ var arbitrum = ClientErrors{
Fatal: arbitrumFatal,
L2FeeTooLow: regexp.MustCompile(`(: |^)max fee per gas less than block base fee(:|$)`),
L2Full: regexp.MustCompile(`(: |^)(queue full|sequencer pending tx pool full, please try again)(:|$)`),
ServiceUnavailable: regexp.MustCompile(`(: |^)502 Bad Gateway: [\s\S]*$|network is unreachable|i/o timeout`),
ServiceUnavailable: regexp.MustCompile(`(: |^)502 Bad Gateway: [\s\S]*$|network is unreachable|i/o timeout|(: |^)503 Service Temporarily Unavailable(:|$)`),
}

// Treasure
Expand Down Expand Up @@ -289,7 +290,7 @@ var internal = ClientErrors{
TerminallyStuck: regexp.MustCompile(TerminallyStuckMsg),
}

var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm, treasure, mantle, aStar, hedera, gnosis, internal}
DylanTinianov marked this conversation as resolved.
Show resolved Hide resolved
var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm, treasure, mantle, aStar, hedera, gnosis, internal, defaultClient}

// ClientErrorRegexes returns a map of compiled regexes for each error type
func ClientErrorRegexes(errsRegex config.ClientErrors) *ClientErrors {
Expand Down Expand Up @@ -398,6 +399,11 @@ func (s *SendError) IsServiceUnavailable(configErrors *ClientErrors) bool {
return s.is(ServiceUnavailable, configErrors) || pkgerrors.Is(s.err, commonclient.ErroringNodeError)
}

// IsServiceTimeout indicates if the error was caused by a service timeout
func (s *SendError) IsServiceTimeout(configErrors *ClientErrors) bool {
return s.is(ServiceTimeout, configErrors)
}

// IsTerminallyStuck indicates if a transaction was stuck without any chance of inclusion
func (s *SendError) IsTerminallyStuckConfigError(configErrors *ClientErrors) bool {
return s.is(TerminallyStuck, configErrors)
Expand Down Expand Up @@ -619,6 +625,10 @@ func ClassifySendError(err error, clientErrors config.ClientErrors, lggr logger.
lggr.Errorw(fmt.Sprintf("service unavailable while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx)
return commonclient.Retryable
}
if sendError.IsServiceTimeout(configErrors) {
lggr.Errorw(fmt.Sprintf("service timed out while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx)
return commonclient.Retryable
}
if sendError.IsTimeout() {
lggr.Errorw(fmt.Sprintf("timeout while sending transaction %x", tx.Hash()), "err", sendError, "etx", tx)
return commonclient.Retryable
Expand Down Expand Up @@ -666,7 +676,9 @@ var drpc = ClientErrors{

// Linkpool, Blockdaemon, and Chainstack all return "request timed out" if the log results are too large for them to process
var defaultClient = ClientErrors{
TooManyResults: regexp.MustCompile(`request timed out`),
TooManyResults: regexp.MustCompile(`request timed out`),
DylanTinianov marked this conversation as resolved.
Show resolved Hide resolved
ServiceTimeout: regexp.MustCompile(`(: |^)408 Request Timeout(:|$)`),
ServiceUnavailable: regexp.MustCompile(`(: |^)503 Service Temporarily Unavailable(:|$)`),
}

// JSON-RPC error codes which can indicate a refusal of the server to process an eth_getLogs request because the result set is too large
Expand Down
15 changes: 15 additions & 0 deletions core/chains/evm/client/errors_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ func Test_Eth_Errors(t *testing.T) {
{"network is unreachable", true, "Arbitrum"},
{"client error service unavailable", true, "tomlConfig"},
{"[Request ID: 825608a8-fd8a-4b5b-aea7-92999509306d] Error invoking RPC: [Request ID: 825608a8-fd8a-4b5b-aea7-92999509306d] Transaction execution returns a null value for transaction", true, "hedera"},
{"call failed: 503 Service Temporarily Unavailable: <html>\r\n<head><title>503 Service Temporarily Unavailable</title></head>\r\n<body>\r\n<center><h1>503 Service Temporarily Unavailable</h1></center>\r\n</body>\r\n</html>\r\n", true, "Arbitrum"},
}
for _, test := range tests {
err = evmclient.NewSendErrorS(test.message)
Expand All @@ -260,6 +261,20 @@ func Test_Eth_Errors(t *testing.T) {
}
})

t.Run("IsServiceTimeout", func(t *testing.T) {
tests := []errorCase{
{"call failed: 408 Request Timeout: {", true, "Arbitrum"},
{"408 Request Timeout: {\"id\":303,\"jsonrpc\":\"2.0\",\"error\":{\"code\\\":-32009,\\\"message\\\":\\\"request timeout\\\"}}\",\"errVerbose\":\"408 Request Timeout:\n", true, "Arbitrum"},
{"request timeout", false, "tomlConfig"},
}
for _, test := range tests {
err = evmclient.NewSendErrorS(test.message)
assert.Equal(t, err.IsServiceTimeout(clientErrors), test.expect)
err = newSendErrorWrapped(test.message)
assert.Equal(t, err.IsServiceTimeout(clientErrors), test.expect)
}
})

t.Run("IsTxFeeExceedsCap", func(t *testing.T) {
tests := []errorCase{
{"tx fee (1.10 ether) exceeds the configured cap (1.00 ether)", true, "geth"},
Expand Down
Loading