From dd2368afad322128f9b0f08d1af28a18feb425ff Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Wed, 23 Oct 2024 09:10:28 -0300 Subject: [PATCH] Fix CallbackError wrapping, type registries in file --- docs/7.references/2.config.md | 3 +- schemas/dipdup-3.0.json | 13 +++++++ src/demo_substrate_events/dipdup.yaml | 1 + src/dipdup/cli.py | 14 +++++++- src/dipdup/codegen/substrate.py | 2 +- src/dipdup/config/substrate.py | 2 ++ src/dipdup/context.py | 6 ++-- src/dipdup/indexes/substrate.py | 2 +- .../demo_substrate_events/dipdup.yaml.j2 | 1 + src/dipdup/runtimes.py | 34 ++++++++++++------- 10 files changed, 58 insertions(+), 20 deletions(-) diff --git a/docs/7.references/2.config.md b/docs/7.references/2.config.md index 52d32c9df..b4db5e426 100644 --- a/docs/7.references/2.config.md +++ b/docs/7.references/2.config.md @@ -1110,12 +1110,13 @@ description: "Config file reference" ## dipdup.config.substrate.SubstrateRuntimeConfig -class dipdup.config.substrate.SubstrateRuntimeConfig(kind='substrate') +class dipdup.config.substrate.SubstrateRuntimeConfig(kind='substrate', type_registry=None)

Substrate runtime config

Parameters:

  • kind (Literal['substrate']) – Always ‘substrate’

    • +

  • type_registry (str | None) – Path to type registry or its alias

diff --git a/schemas/dipdup-3.0.json b/schemas/dipdup-3.0.json index 0cff0afb7..d184fccfe 100644 --- a/schemas/dipdup-3.0.json +++ b/schemas/dipdup-3.0.json @@ -1717,6 +1717,19 @@ "title": "kind", "type": "string", "description": "Always 'substrate'" + }, + "type_registry": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "type_registry", + "description": "Path to type registry or its alias" } }, "title": "SubstrateRuntimeConfig", diff --git a/src/demo_substrate_events/dipdup.yaml b/src/demo_substrate_events/dipdup.yaml index dc720b148..33995680c 100644 --- a/src/demo_substrate_events/dipdup.yaml +++ b/src/demo_substrate_events/dipdup.yaml @@ -4,6 +4,7 @@ package: demo_substrate_events runtimes: assethub: kind: substrate + type_registry: statemint datasources: subsquid: diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index ca5f0d05b..81c6a4bdd 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -3,6 +3,7 @@ import atexit import logging import sys +import traceback from collections.abc import Callable from collections.abc import Coroutine from contextlib import AsyncExitStack @@ -21,6 +22,7 @@ from dipdup import __version__ from dipdup import env from dipdup._version import check_version +from dipdup.exceptions import CallbackError from dipdup.install import EPILOG from dipdup.install import WELCOME_ASCII from dipdup.sys import set_up_process @@ -135,10 +137,20 @@ def _print_help_atexit(error: Exception, report_id: str) -> None: from dipdup.exceptions import Error def _print() -> None: + nonlocal error + if isinstance(error, Error): echo(error.help(), err=True) else: - echo(Error.default_help(), err=True) + # NOTE: check the traceback to find out if it's from a callback + tb = traceback.extract_tb(error.__traceback__) + for frame in tb: + if frame.name == 'fire_handler': + module = next(f.filename for f in tb if '/handlers/' in f.filename or '/hooks/' in f.filename) + echo(CallbackError(module=Path(module).stem, exc=error).help(), err=True) + break + else: + echo(Error.default_help(), err=True) echo(f'Report saved; run `dipdup report show {report_id}` to view it', err=True) diff --git a/src/dipdup/codegen/substrate.py b/src/dipdup/codegen/substrate.py index 566753d28..0a12a3aa9 100644 --- a/src/dipdup/codegen/substrate.py +++ b/src/dipdup/codegen/substrate.py @@ -249,7 +249,7 @@ async def _generate_type(self, schema_path: Path, force: bool) -> None: def _get_runtime(self, name: str) -> SubstrateRuntime: if name not in self._runtimes: self._runtimes[name] = SubstrateRuntime( - name=name, + config=self._config.runtimes[name], package=self._package, ) return self._runtimes[name] diff --git a/src/dipdup/config/substrate.py b/src/dipdup/config/substrate.py index 5286636e9..3771ee75e 100644 --- a/src/dipdup/config/substrate.py +++ b/src/dipdup/config/substrate.py @@ -21,9 +21,11 @@ class SubstrateRuntimeConfig(RuntimeConfig): """Substrate runtime config :param kind: Always 'substrate' + :param type_registry: Path to type registry or its alias """ kind: Literal['substrate'] = 'substrate' + type_registry: str | None = None @dataclass(config=ConfigDict(extra='forbid'), kw_only=True) diff --git a/src/dipdup/context.py b/src/dipdup/context.py index 720aee215..4f3906df5 100644 --- a/src/dipdup/context.py +++ b/src/dipdup/context.py @@ -56,7 +56,6 @@ from dipdup.datasources.substrate_subsquid import SubstrateSubsquidDatasource from dipdup.datasources.tezos_tzkt import TezosTzktDatasource from dipdup.datasources.tzip_metadata import TzipMetadataDatasource -from dipdup.exceptions import CallbackError from dipdup.exceptions import ConfigurationError from dipdup.exceptions import ContractAlreadyExistsError from dipdup.exceptions import FrameworkException @@ -725,6 +724,7 @@ async def execute_sql_query( conn=None, ) + # NOTE: Aliases, remove later execute_sql = execute_sql_script execute_script = execute_sql_script execute_query = execute_sql_query @@ -736,8 +736,8 @@ def _callback_wrapper(self, module: str) -> Iterator[None]: # NOTE: Do not wrap known errors like ProjectImportError except FrameworkException: raise - except Exception as e: - raise CallbackError(module, e) from e + # except Exception as e: + # raise CallbackError(module, e) from e def _get_handler(self, name: str, index: str) -> HandlerConfig: try: diff --git a/src/dipdup/indexes/substrate.py b/src/dipdup/indexes/substrate.py index 4dbf6581f..78e7a18a2 100644 --- a/src/dipdup/indexes/substrate.py +++ b/src/dipdup/indexes/substrate.py @@ -34,6 +34,6 @@ def __init__( ) -> None: super().__init__(ctx, config, datasources) self.runtime = SubstrateRuntime( - name=config.runtime.name, + config=config.runtime, package=ctx.package, ) diff --git a/src/dipdup/projects/demo_substrate_events/dipdup.yaml.j2 b/src/dipdup/projects/demo_substrate_events/dipdup.yaml.j2 index d83f8ff14..35e2bb728 100644 --- a/src/dipdup/projects/demo_substrate_events/dipdup.yaml.j2 +++ b/src/dipdup/projects/demo_substrate_events/dipdup.yaml.j2 @@ -4,6 +4,7 @@ package: {{ project.package }} runtimes: assethub: kind: substrate + type_registry: statemint datasources: subsquid: diff --git a/src/dipdup/runtimes.py b/src/dipdup/runtimes.py index d491d2f3d..322c442f8 100644 --- a/src/dipdup/runtimes.py +++ b/src/dipdup/runtimes.py @@ -1,11 +1,14 @@ import logging import re +from functools import cache from functools import cached_property +from pathlib import Path from typing import TYPE_CHECKING from typing import Any import orjson +from dipdup.config.substrate import SubstrateRuntimeConfig from dipdup.exceptions import FrameworkException from dipdup.package import DipDupPackage @@ -14,10 +17,6 @@ _logger = logging.getLogger(__name__) -ALIASES = { - 'assethub': 'statemint', -} - def extract_args_name(description: str) -> list[str]: pattern = r'\((.*?)\)|\[(.*?)\]' @@ -30,6 +29,17 @@ def extract_args_name(description: str) -> list[str]: return [arg.strip('\\') for arg in args_str.split(', ')] +@cache +def get_type_registry(name_or_path: str | Path) -> 'RuntimeConfigurationObject': + from scalecodec.type_registry import load_type_registry_preset # type: ignore[import-untyped] + + if isinstance(name_or_path, str) and Path(name_or_path).is_file(): + name_or_path = Path(name_or_path) + if isinstance(name_or_path, Path): + return orjson.loads(name_or_path.read_bytes()) + return load_type_registry_preset(name_or_path) + + class SubstrateSpecVersion: def __init__(self, name: str, metadata: list[dict[str, Any]]) -> None: self._name = name @@ -60,10 +70,10 @@ def get_event_abi(self, qualname: str) -> dict[str, Any]: class SubstrateRuntime: def __init__( self, - name: str, + config: SubstrateRuntimeConfig, package: DipDupPackage, ) -> None: - self._name = name + self._config = config self._package = package # TODO: unload by LRU? self._spec_versions: dict[str, SubstrateSpecVersion] = {} @@ -71,14 +81,12 @@ def __init__( @cached_property def runtime_config(self) -> 'RuntimeConfigurationObject': from scalecodec.base import RuntimeConfigurationObject - from scalecodec.type_registry import load_type_registry_preset # type: ignore[import-untyped] # FIXME: ss58_format runtime_config = RuntimeConfigurationObject(ss58_format=99) - for name in ('core', ALIASES.get(self._name, self._name)): - preset = load_type_registry_preset(name) - assert preset - runtime_config.update_type_registry(preset) + # FIXME: When to use 'legacy' instead? + runtime_config.update_type_registry(get_type_registry('core')) + runtime_config.update_type_registry(get_type_registry(self._config.type_registry)) return runtime_config @@ -86,14 +94,14 @@ def get_spec_version(self, name: str) -> SubstrateSpecVersion: if name not in self._spec_versions: _logger.info('loading spec version `%s`', name) try: - metadata = orjson.loads(self._package.abi.joinpath(self._name, f'v{name}.json').read_bytes()) + metadata = orjson.loads(self._package.abi.joinpath(self._config.name, f'v{name}.json').read_bytes()) self._spec_versions[name] = SubstrateSpecVersion( name=f'v{name}', metadata=metadata, ) except FileNotFoundError: # FIXME: Using last known version to help with missing abis - last_known = tuple(self._package.abi.joinpath(self._name).glob('v*.json'))[-1].stem + last_known = tuple(self._package.abi.joinpath(self._config.name).glob('v*.json'))[-1].stem _logger.info('using last known version `%s`', last_known) self._spec_versions[name] = self.get_spec_version(last_known[1:])