From 1f33a3c6b87c1726138e82f1f1e68ce267e856f9 Mon Sep 17 00:00:00 2001 From: Manu NALEPA Date: Wed, 12 Jul 2023 17:22:28 +0200 Subject: [PATCH] Add some retries quite specific to Chainstack errors --- eth_validator_watcher/beacon.py | 54 +++++++++++++++----- eth_validator_watcher/missed_attestations.py | 4 +- poetry.lock | 17 +++++- pyproject.toml | 1 + tests/beacon/test_get_block.py | 2 +- 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/eth_validator_watcher/beacon.py b/eth_validator_watcher/beacon.py index febecd0..4b1579f 100644 --- a/eth_validator_watcher/beacon.py +++ b/eth_validator_watcher/beacon.py @@ -3,11 +3,13 @@ from collections import defaultdict from functools import lru_cache -from typing import Optional +from typing import Any, Optional from requests import Response, Session, codes from requests.adapters import HTTPAdapter, Retry from requests.exceptions import RetryError +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed +from requests.exceptions import ChunkedEncodingError from .models import ( BeaconType, @@ -45,16 +47,38 @@ def __init__(self, url: str) -> None: max_retries=Retry( backoff_factor=0.5, total=3, - status_forcelist=[codes.not_found], + status_forcelist=[ + codes.not_found, + codes.bad_gateway, + codes.service_unavailable, + ], ) ) self.__http.mount("http://", adapter) self.__http.mount("https://", adapter) + @retry( + stop=stop_after_attempt(5), + wait=wait_fixed(3), + retry=retry_if_exception_type(ChunkedEncodingError), + ) + def __get(self, *args: Any, **kwargs: Any) -> Response: + """Wrapper around requests.get()""" + return self.__http.get(*args, **kwargs) + + @retry( + stop=stop_after_attempt(5), + wait=wait_fixed(3), + retry=retry_if_exception_type(ChunkedEncodingError), + ) + def __post(self, *args: Any, **kwargs: Any) -> Response: + """Wrapper around requests.get()""" + return self.__http.post(*args, **kwargs) + def get_genesis(self) -> Genesis: """Get genesis data.""" - response = self.__http.get(f"{self.__url}/eth/v1/beacon/genesis") + response = self.__get(f"{self.__url}/eth/v1/beacon/genesis", timeout=5) response.raise_for_status() genesis_dict = response.json() return Genesis(**genesis_dict) @@ -66,7 +90,9 @@ def get_block(self, slot: int) -> Block: slot: Slot corresponding to the block to retrieve """ try: - response = self.__http.get(f"{self.__url}/eth/v2/beacon/blocks/{slot}") + response = self.__get( + f"{self.__url}/eth/v2/beacon/blocks/{slot}", timeout=5 + ) except RetryError as e: # If we are here, it means the block does not exist @@ -83,8 +109,8 @@ def get_proposer_duties(self, epoch: int) -> ProposerDuties: epoch: Epoch corresponding to the proposer duties to retrieve """ - response = self.__http.get( - f"{self.__url}/eth/v1/validator/duties/proposer/{epoch}" + response = self.__get( + f"{self.__url}/eth/v1/validator/duties/proposer/{epoch}", timeout=10 ) response.raise_for_status() @@ -100,8 +126,8 @@ def get_status_to_index_to_validator( outer value (=inner key): Index of validator inner value : Validator """ - response = self.__http.get( - f"{self.__url}/eth/v1/beacon/states/head/validators", + response = self.__get( + f"{self.__url}/eth/v1/beacon/states/head/validators", timeout=25 ) response.raise_for_status() @@ -131,9 +157,10 @@ def get_duty_slot_to_committee_index_to_validators_index( Parameters: epoch: Epoch """ - response = self.__http.get( + response = self.__get( f"{self.__url}/eth/v1/beacon/states/head/committees", params=dict(epoch=epoch), + timeout=10, ) response.raise_for_status() @@ -219,11 +246,12 @@ def __get_validators_liveness_lighthouse( validators_index: Set of validator indexs corresponding to the liveness to retrieve """ - return self.__http.post( + return self.__post( f"{self.__url}/lighthouse/liveness", json=ValidatorsLivenessRequestLighthouse( epoch=epoch, indices=sorted(list(validators_index)) ).model_dump(), + timeout=10, ) def __get_validators_liveness_teku( @@ -238,11 +266,12 @@ def __get_validators_liveness_teku( validators_index: Set of validator indexs corresponding to the liveness to retrieve """ - return self.__http.post( + return self.__post( f"{self.__url}/eth/v1/validator/liveness/{epoch}", json=ValidatorsLivenessRequestTeku( indices=sorted(list(validators_index)) ).model_dump(), + timeout=10, ) def __get_validators_liveness_beacon_api( @@ -257,10 +286,11 @@ def __get_validators_liveness_beacon_api( validators_index: Set of validator indexs corresponding to the liveness to retrieve """ - return self.__http.post( + return self.__post( f"{self.__url}/eth/v1/validator/liveness/{epoch}", json=[ str(validator_index) for validator_index in sorted(list(validators_index)) ], + timeout=10, ) diff --git a/eth_validator_watcher/missed_attestations.py b/eth_validator_watcher/missed_attestations.py index a82717e..b5109b2 100644 --- a/eth_validator_watcher/missed_attestations.py +++ b/eth_validator_watcher/missed_attestations.py @@ -111,7 +111,7 @@ def process_double_missed_attestations( short_first_pubkeys_str = ", ".join(short_first_pubkeys) message_console = ( - f"😱 Our validator {short_first_pubkeys_str} and " + f"😱 Our validator {short_first_pubkeys_str} and " f"{len(double_dead_indexes) - len(short_first_pubkeys)} more " f"missed 2 attestations in a raw from epoch {epoch - 2}" ) @@ -120,7 +120,7 @@ def process_double_missed_attestations( if slack is not None: message_slack = ( - f"😱 Our validator `{short_first_pubkeys_str}` and " + f"😱 Our validator `{short_first_pubkeys_str}` and " f"`{len(double_dead_indexes) - len(short_first_pubkeys)}` more " f"missed 2 attestations in a raw from epoch `{epoch - 2}`" ) diff --git a/poetry.lock b/poetry.lock index ac7a999..dac393a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -702,6 +702,21 @@ files = [ optional = ["SQLAlchemy (>=1.4,<3)", "aiodns (>1.0)", "aiohttp (>=3.7.3,<4)", "boto3 (<=2)", "websocket-client (>=1,<2)", "websockets (>=10,<11)"] testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werkzeug (<2)", "black (==22.8.0)", "boto3 (<=2)", "click (==8.0.4)", "databases (>=0.5)", "flake8 (>=5,<6)", "itsdangerous (==1.1.0)", "moto (>=3,<4)", "psutil (>=5,<6)", "pytest (>=6.2.5,<7)", "pytest-asyncio (<1)", "pytest-cov (>=2,<3)"] +[[package]] +name = "tenacity" +version = "8.2.2" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "tenacity-8.2.2-py3-none-any.whl", hash = "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0"}, + {file = "tenacity-8.2.2.tar.gz", hash = "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0"}, +] + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "tomli" version = "2.0.1" @@ -781,4 +796,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c1e1fd603a8e7d6ef81b395d447ec3c621a2d06f709fd85d484b86ad762a46f9" +content-hash = "eedc6a0bd723fec2a385e02cad19e1c9b340e07b0a62d6d905cb9ee427fd86bb" diff --git a/pyproject.toml b/pyproject.toml index cf5c682..7311824 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ pydantic = "^2.0" requests = "^2.31.0" typer = "^0.9.0" slack-sdk = "^3.21.3" +tenacity = "^8.2.2" [tool.poetry.group.dev.dependencies] mypy = "^1.2.0" diff --git a/tests/beacon/test_get_block.py b/tests/beacon/test_get_block.py index b3bb1dc..d2d635b 100644 --- a/tests/beacon/test_get_block.py +++ b/tests/beacon/test_get_block.py @@ -27,7 +27,7 @@ def test_get_block_exists() -> None: def test_get_block_does_not_exist() -> None: - def get(url: str) -> Response: + def get(url: str, **_) -> Response: assert url == "http://beacon-node:5052/eth/v2/beacon/blocks/42" raise RetryError