Merge master into restaking

Signed-off-by: cyc60 <avsysoev60@gmail.com>

README.md

@@ -154,14 +154,14 @@ Head to [Usage](#usage) to launch your operator service.
 Pull the latest docker operator docker image:
 
 ```bash
-docker pull europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.3.restaking
+docker pull europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.5.restaking
 ```
 
 You can also build the docker image from source by cloning this repo and executing the following command from within
 the `v3-operator` folder:
 
 ```bash
-docker build --pull -t europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.3.restaking .
+docker build --pull -t europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.5.restaking .
 ```
 
 You will execute Operator Service commands using the format below (note the use of flags are optional):
@@ -170,7 +170,7 @@ You will execute Operator Service commands using the format below (note the use
 docker run --rm -ti \
 -u $(id -u):$(id -g) \
 -v ~/.stakewise/:/data \
-europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.3.restaking \
+europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.5.restaking \
 src/main.py COMMAND \
 --flagA=123 \
 --flagB=xyz
@@ -393,7 +393,7 @@ below:
 docker run --restart on-failure:10 \
 -u $(id -u):$(id -g) \
 -v ~/.stakewise/:/data \
-europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.3.restaking \
+europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.0.5.restaking \
 src/main.py start \
 --vault=0x3320ad928c20187602a2b2c04eeaa813fa899468 \
 --data-dir=/data \

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "v3-operator"
-version = "v2.0.3"
+version = "v2.0.5"
 description = "StakeWise operator service for registering vault validators"
 authors = ["StakeWise Labs <info@stakewise.io>"]
 package-mode = false
@@ -77,7 +77,11 @@ exclude = ["conftest.py"]
 exclude = ["test"]
 ignore_missing_imports = true
 python_version = "3.10"
+disallow_untyped_defs = true
 disallow_incomplete_defs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_unused_configs = true
 
 [[tool.mypy.overrides]]
 module = 'web3.*'
@@ -107,3 +111,6 @@ exclude = '''
 
 [tool.coverage.report]
 fail_under = 70
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

scripts/install.sh

@@ -281,7 +281,7 @@ http_copy() {
 github_release() {
   owner_repo=$1
   version=$2
-  test -z "$version" && version="v2.0.3.restaking"
+  test -z "$version" && version="v2.0.5.restaking"
   giturl="https://github.com/${owner_repo}/releases/${version}"
   json=$(http_copy "$giturl" "Accept:application/json")
   test -z "$json" && return 1

src/commands/create_keys.py

@@ -107,6 +107,7 @@ def create_keys(
         vault_config.increment_mnemonic_index(count)
 
         # move files from tmp dir
+        keystores_dir.mkdir(exist_ok=True)
         tmp_deposit_data_file.replace(deposit_data_file)
         for src_file in tmp_keystores_dir.glob('*'):
             src_file.rename(keystores_dir.joinpath(src_file.name))

src/commands/remote_signer_setup.py

@@ -174,7 +174,7 @@ async def main() -> None:
         ):
             data = {
                 'keystores': keystores_json_chunk,
-                'passwords': [kf.password for kf in keystore_files_chunk],
+                'passwords': [kf.password for kf in keystore_files_chunk],  # type: ignore
             }
 
             # Only add tags and fee_recipient if --dappnode is set

src/commands/start.py

@@ -13,6 +13,7 @@
 from src.common.vault_config import VaultConfig
 from src.config.networks import AVAILABLE_NETWORKS
 from src.config.settings import (
+    DEFAULT_HASHI_VAULT_PARALLELISM,
     DEFAULT_MAX_FEE_PER_GAS_GWEI,
     DEFAULT_METRICS_HOST,
     DEFAULT_METRICS_PORT,
@@ -179,7 +180,14 @@
 @click.option(
     '--hashi-vault-key-path',
     envvar='HASHI_VAULT_KEY_PATH',
-    help='Key path in the K/V secret engine where validator signing keys are stored.',
+    multiple=True,
+    help='Key path(s) in the K/V secret engine where validator signing keys are stored.',
+)
+@click.option(
+    '--hashi-vault-parallelism',
+    envvar='HASHI_VAULT_PARALLELISM',
+    help='How much requests to K/V secrets engine to do in parallel.',
+    default=DEFAULT_HASHI_VAULT_PARALLELISM,
 )
 @click.option(
     '--log-format',
@@ -235,9 +243,10 @@ def start(
     keystores_dir: str | None,
     keystores_password_file: str | None,
     remote_signer_url: str | None,
-    hashi_vault_key_path: str | None,
+    hashi_vault_key_path: list[str] | None,
     hashi_vault_token: str | None,
     hashi_vault_url: str | None,
+    hashi_vault_parallelism: int,
     hot_wallet_file: str | None,
     hot_wallet_password_file: str | None,
     max_fee_per_gas_gwei: int,
@@ -268,7 +277,8 @@ def start(
         keystores_password_file=keystores_password_file,
         remote_signer_url=remote_signer_url,
         hashi_vault_token=hashi_vault_token,
-        hashi_vault_key_path=hashi_vault_key_path,
+        hashi_vault_key_paths=hashi_vault_key_path,
+        hashi_vault_parallelism=hashi_vault_parallelism,
         hashi_vault_url=hashi_vault_url,
         hot_wallet_file=hot_wallet_file,
         hot_wallet_password_file=hot_wallet_password_file,

src/commands/start_base.py

@@ -110,7 +110,7 @@ def log_start() -> None:
         logger.info('%s, version %s', start_str, src.__version__)
 
 
-def setup_sentry():
+def setup_sentry() -> None:
     if settings.sentry_dsn:
         # pylint: disable-next=import-outside-toplevel
         import sentry_sdk

src/commands/validators_exit.py

@@ -17,7 +17,7 @@
 from src.common.validators import validate_eth_address
 from src.common.vault_config import VaultConfig
 from src.config.networks import AVAILABLE_NETWORKS
-from src.config.settings import settings
+from src.config.settings import DEFAULT_HASHI_VAULT_PARALLELISM, settings
 from src.validators.keystores.base import BaseKeystore
 from src.validators.keystores.load import load_keystore
 
@@ -84,9 +84,16 @@ class ValidatorExit:
 )
 @click.option(
     '--hashi-vault-key-path',
+    multiple=True,
     envvar='HASHI_VAULT_KEY_PATH',
     help='Key path in the K/V secret engine where validator signing keys are stored.',
 )
+@click.option(
+    '--hashi-vault-parallelism',
+    envvar='HASHI_VAULT_PARALLELISM',
+    help='How much requests to K/V secrets engine to do in parallel.',
+    default=DEFAULT_HASHI_VAULT_PARALLELISM,
+)
 @click.option(
     '-v',
     '--verbose',
@@ -111,16 +118,17 @@ class ValidatorExit:
     type=int,
 )
 @click.command(help='Performs a voluntary exit for active vault validators.')
-# pylint: disable-next=too-many-arguments
+# pylint: disable-next=too-many-arguments,too-many-locals
 def validators_exit(
     network: str,
     vault: HexAddress,
     count: int | None,
     consensus_endpoints: str,
     remote_signer_url: str,
-    hashi_vault_key_path: str | None,
+    hashi_vault_key_path: list[str] | None,
     hashi_vault_token: str | None,
     hashi_vault_url: str | None,
+    hashi_vault_parallelism: int,
     data_dir: str,
     verbose: bool,
     log_level: str,
@@ -139,8 +147,9 @@ def validators_exit(
         consensus_endpoints=consensus_endpoints,
         remote_signer_url=remote_signer_url,
         hashi_vault_token=hashi_vault_token,
-        hashi_vault_key_path=hashi_vault_key_path,
+        hashi_vault_key_paths=hashi_vault_key_path,
         hashi_vault_url=hashi_vault_url,
+        hashi_vault_parallelism=hashi_vault_parallelism,
         verbose=verbose,
         log_level=log_level,
         pool_size=pool_size,

src/common/app_state.py

@@ -21,7 +21,7 @@ class ExitSignatureUpdateCache:
 
 # pylint: disable-next=too-few-public-methods
 class AppState(metaclass=Singleton):
-    def __init__(self):
+    def __init__(self) -> None:
         self.exit_signature_update_cache = ExitSignatureUpdateCache()
         self.oracles_cache: OraclesCache | None = None
         self.last_withdrawals_update_timestamp: int | None = None

src/common/clients.py

@@ -1,6 +1,7 @@
 import logging
 import sqlite3
 from functools import cached_property
+from sqlite3 import Connection
 from typing import cast
 
 from sw_utils import (
@@ -19,7 +20,7 @@
 
 
 class Database:
-    def get_db_connection(self) -> sqlite3.Connection:
+    def get_db_connection(self) -> Connection:
         return sqlite3.connect(settings.database)
 
     def create_db_dir(self) -> None:
@@ -45,7 +46,7 @@ def client(self) -> AsyncWeb3:
 
         return w3
 
-    def __getattr__(self, item):
+    def __getattr__(self, item):  # type: ignore
         return getattr(self.client, item)
 
 
@@ -58,7 +59,7 @@ def client(self) -> ExtendedAsyncBeacon:
             retry_timeout=settings.consensus_retry_timeout,
         )
 
-    def __getattr__(self, item):
+    def __getattr__(self, item):  # type: ignore
         return getattr(self.client, item)
 
 

src/common/contracts.py

@@ -155,7 +155,7 @@ async def mev_escrow(self) -> ChecksumAddress:
     async def version(self) -> int:
         return await self.contract.functions.version().call()
 
-    async def validators_manager(self):
+    async def validators_manager(self) -> ChecksumAddress:
         return await self.contract.functions.validatorsManager().call()
 
 

src/common/credentials.py

@@ -138,7 +138,7 @@ def generate_credentials(
             show_pos=True,
         ) as progress_bar, Pool(processes=pool_size) as pool:
 
-            def bar_updated(result):
+            def bar_updated(result: list) -> None:
                 progress_bar.update(len(result))
 
             results = []

src/common/exceptions.py

@@ -3,7 +3,7 @@
 
 
 class InvalidOraclesRequestError(ValueError):
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__(INVALID_ORACLES_REQUEST)
 
 

src/common/execution.py

@@ -110,7 +110,6 @@ async def get_protocol_config() -> ProtocolConfig:
         rewards_threshold=oracles_cache.rewards_threshold,
         validators_threshold=oracles_cache.validators_threshold,
     )
-    pc.rewards_threshold = cast(int, pc.rewards_threshold)
     return pc
 
 

src/common/graph.py

@@ -13,7 +13,7 @@
 class GraphClient:
     gql_client: Client
 
-    def __init__(self):
+    def __init__(self) -> None:
         transport = AIOHTTPTransport(
             url=settings.network_config.STAKEWISE_API_URL,
             timeout=GRAPH_API_TIMEOUT,

src/common/logging.py

@@ -13,7 +13,7 @@
 ]
 
 
-def setup_logging():
+def setup_logging() -> None:
     if settings.log_format == LOG_JSON:
         formatter = JsonFormatter('%(timestamp)s %(level)s %(name)s %(message)s')
         logHandler = logging.StreamHandler()

src/common/metrics.py

@@ -14,7 +14,7 @@
 # pylint: disable=too-few-public-methods
 # pylint: disable-next=too-many-instance-attributes
 class Metrics:
-    def __init__(self):
+    def __init__(self) -> None:
         self.app_version = Info(
             'app_version', 'V3 Operator version', namespace=settings.metrics_prefix
         )
@@ -46,16 +46,16 @@ def __init__(self):
             namespace=settings.metrics_prefix,
         )
 
-    def set_app_version(self):
+    def set_app_version(self) -> None:
         self.app_version.info({'version': src.__version__})
 
 
 # pylint: disable-next=too-few-public-methods
 class LazyMetrics:
-    def __init__(self):
+    def __init__(self) -> None:
         self._metrics: Metrics | None = None
 
-    def __getattr__(self, item):
+    def __getattr__(self, item):  # type: ignore
         if self._metrics is None:
             self._metrics = Metrics()
         return getattr(self._metrics, item)

src/common/startup_check.py

@@ -30,7 +30,7 @@
 IPFS_HASH_EXAMPLE = 'QmawUdo17Fvo7xa6ARCUSMV1eoVwPtVuzx8L8Crj2xozWm'
 
 
-def validate_settings():
+def validate_settings() -> None:
     if not settings.execution_endpoints:
         raise ValueError('EXECUTION_ENDPOINTS is missing')
 
@@ -209,7 +209,7 @@ async def check_withdrawal_address() -> None:
             raise ValueError('Invalid withdrawal address in deposit data')
 
 
-async def startup_checks():
+async def startup_checks() -> None:
     validate_settings()
 
     logger.info('Checking connection to database...')

src/common/typings.py

@@ -34,7 +34,7 @@ class OraclesApproval:
 class Singleton(type):
     _instances: dict = {}
 
-    def __call__(cls, *args, **kwargs):
+    def __call__(cls, *args, **kwargs):  # type: ignore
         if cls not in cls._instances:
             cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
         return cls._instances[cls]

src/common/utils.py

@@ -3,6 +3,7 @@
 from collections import defaultdict
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import Any, Iterator
 
 import click
 import tenacity
@@ -94,12 +95,12 @@ def process_oracles_approvals(
     return OraclesApproval(ipfs_hash=winner[0], signatures=signatures, deadline=winner[1])
 
 
-def chunkify(items, size):
+def chunkify(items: list | range, size: int) -> Iterator[list | range]:
     for i in range(0, len(items), size):
         yield items[i : i + size]
 
 
-def greenify(value):
+def greenify(value: Any) -> str:
     return click.style(value, bold=True, fg='green')
 
 
@@ -115,7 +116,7 @@ async def calc_slot_by_block_number(block_number: BlockNumber) -> int:
 
 
 class JsonFormatter(jsonlogger.JsonFormatter):
-    def add_fields(self, log_record, record, message_dict):
+    def add_fields(self, log_record, record, message_dict):  # type: ignore
         super().add_fields(log_record, record, message_dict)
         if not log_record.get('timestamp'):
             date = datetime.fromtimestamp(record.created, tz=timezone.utc)