Fix CallbackError wrapping, type registries in file

docs/7.references/2.config.md

@@ -1110,12 +1110,13 @@ description: "Config file reference"
 
 ## dipdup.config.substrate.SubstrateRuntimeConfig
 
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.substrate.</span></span><span class="sig-name descname"><span class="pre">SubstrateRuntimeConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">kind</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'substrate'</span></span></em><span class="sig-paren">)</span></dt>
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.config.substrate.</span></span><span class="sig-name descname"><span class="pre">SubstrateRuntimeConfig</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">kind</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'substrate'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">type_registry</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span></dt>
 <dd><p>Substrate runtime config</p>
 <dl class="field-list simple">
 <dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
 <li><p><strong>kind</strong> (<em>Literal</em><em>[</em><em>'substrate'</em><em>]</em>) – Always ‘substrate’</p></li>
+<li><p><strong>type_registry</strong> (<em>str</em><em> | </em><em>None</em>) – Path to type registry or its alias</p></li>
 
 </ul>
 </dd>

schemas/dipdup-3.0.json

@@ -1717,6 +1717,19 @@
           "title": "kind",
           "type": "string",
           "description": "Always 'substrate'"
+        },
+        "type_registry": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "type_registry",
+          "description": "Path to type registry or its alias"
         }
       },
       "title": "SubstrateRuntimeConfig",

src/demo_substrate_events/dipdup.yaml

@@ -4,6 +4,7 @@ package: demo_substrate_events
 runtimes:
   assethub:
     kind: substrate
+    type_registry: statemint
 
 datasources:
   subsquid:

src/dipdup/cli.py

@@ -3,6 +3,7 @@
 import atexit
 import logging
 import sys
+import traceback
 from collections.abc import Callable
 from collections.abc import Coroutine
 from contextlib import AsyncExitStack
@@ -21,6 +22,7 @@
 from dipdup import __version__
 from dipdup import env
 from dipdup._version import check_version
+from dipdup.exceptions import CallbackError
 from dipdup.install import EPILOG
 from dipdup.install import WELCOME_ASCII
 from dipdup.sys import set_up_process
@@ -135,10 +137,20 @@ def _print_help_atexit(error: Exception, report_id: str) -> None:
     from dipdup.exceptions import Error
 
     def _print() -> None:
+        nonlocal error
+
         if isinstance(error, Error):
             echo(error.help(), err=True)
         else:
-            echo(Error.default_help(), err=True)
+            # NOTE: check the traceback  to find out if it's from a callback
+            tb = traceback.extract_tb(error.__traceback__)
+            for frame in tb:
+                if frame.name == 'fire_handler':
+                    module = next(f.filename for f in tb if '/handlers/' in f.filename or '/hooks/' in f.filename)
+                    echo(CallbackError(module=Path(module).stem, exc=error).help(), err=True)
+                    break
+            else:
+                echo(Error.default_help(), err=True)
 
         echo(f'Report saved; run `dipdup report show {report_id}` to view it', err=True)
 

src/dipdup/codegen/substrate.py

@@ -249,7 +249,7 @@ async def _generate_type(self, schema_path: Path, force: bool) -> None:
     def _get_runtime(self, name: str) -> SubstrateRuntime:
         if name not in self._runtimes:
             self._runtimes[name] = SubstrateRuntime(
-                name=name,
+                config=self._config.runtimes[name],
                 package=self._package,
             )
         return self._runtimes[name]

src/dipdup/config/substrate.py

@@ -21,9 +21,11 @@ class SubstrateRuntimeConfig(RuntimeConfig):
     """Substrate runtime config
 
     :param kind: Always 'substrate'
+    :param type_registry: Path to type registry or its alias
     """
 
     kind: Literal['substrate'] = 'substrate'
+    type_registry: str | None = None
 
 
 @dataclass(config=ConfigDict(extra='forbid'), kw_only=True)

src/dipdup/context.py

@@ -56,7 +56,6 @@
 from dipdup.datasources.substrate_subsquid import SubstrateSubsquidDatasource
 from dipdup.datasources.tezos_tzkt import TezosTzktDatasource
 from dipdup.datasources.tzip_metadata import TzipMetadataDatasource
-from dipdup.exceptions import CallbackError
 from dipdup.exceptions import ConfigurationError
 from dipdup.exceptions import ContractAlreadyExistsError
 from dipdup.exceptions import FrameworkException
@@ -725,6 +724,7 @@ async def execute_sql_query(
             conn=None,
         )
 
+    # NOTE: Aliases, remove later
     execute_sql = execute_sql_script
     execute_script = execute_sql_script
     execute_query = execute_sql_query
@@ -736,8 +736,8 @@ def _callback_wrapper(self, module: str) -> Iterator[None]:
         # NOTE: Do not wrap known errors like ProjectImportError
         except FrameworkException:
             raise
-        except Exception as e:
-            raise CallbackError(module, e) from e
+        # except Exception as e:
+        #     raise CallbackError(module, e) from e
 
     def _get_handler(self, name: str, index: str) -> HandlerConfig:
         try:

src/dipdup/indexes/substrate.py

@@ -34,6 +34,6 @@ def __init__(
     ) -> None:
         super().__init__(ctx, config, datasources)
         self.runtime = SubstrateRuntime(
-            name=config.runtime.name,
+            config=config.runtime,
             package=ctx.package,
         )

src/dipdup/projects/demo_substrate_events/dipdup.yaml.j2

@@ -4,6 +4,7 @@ package: {{ project.package }}
 runtimes:
   assethub:
     kind: substrate
+    type_registry: statemint
 
 datasources:
   subsquid:

src/dipdup/runtimes.py

@@ -1,11 +1,14 @@
 import logging
 import re
+from functools import cache
 from functools import cached_property
+from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
 
 import orjson
 
+from dipdup.config.substrate import SubstrateRuntimeConfig
 from dipdup.exceptions import FrameworkException
 from dipdup.package import DipDupPackage
 
@@ -14,10 +17,6 @@
 
 _logger = logging.getLogger(__name__)
 
-ALIASES = {
-    'assethub': 'statemint',
-}
-
 
 def extract_args_name(description: str) -> list[str]:
     pattern = r'\((.*?)\)|\[(.*?)\]'
@@ -30,6 +29,17 @@ def extract_args_name(description: str) -> list[str]:
     return [arg.strip('\\') for arg in args_str.split(', ')]
 
 
+@cache
+def get_type_registry(name_or_path: str | Path) -> 'RuntimeConfigurationObject':
+    from scalecodec.type_registry import load_type_registry_preset  # type: ignore[import-untyped]
+
+    if isinstance(name_or_path, str) and Path(name_or_path).is_file():
+        name_or_path = Path(name_or_path)
+    if isinstance(name_or_path, Path):
+        return orjson.loads(name_or_path.read_bytes())
+    return load_type_registry_preset(name_or_path)
+
+
 class SubstrateSpecVersion:
     def __init__(self, name: str, metadata: list[dict[str, Any]]) -> None:
         self._name = name
@@ -60,40 +70,38 @@ def get_event_abi(self, qualname: str) -> dict[str, Any]:
 class SubstrateRuntime:
     def __init__(
         self,
-        name: str,
+        config: SubstrateRuntimeConfig,
         package: DipDupPackage,
     ) -> None:
-        self._name = name
+        self._config = config
         self._package = package
         # TODO: unload by LRU?
         self._spec_versions: dict[str, SubstrateSpecVersion] = {}
 
     @cached_property
     def runtime_config(self) -> 'RuntimeConfigurationObject':
         from scalecodec.base import RuntimeConfigurationObject
-        from scalecodec.type_registry import load_type_registry_preset  # type: ignore[import-untyped]
 
         # FIXME: ss58_format
         runtime_config = RuntimeConfigurationObject(ss58_format=99)
-        for name in ('core', ALIASES.get(self._name, self._name)):
-            preset = load_type_registry_preset(name)
-            assert preset
-            runtime_config.update_type_registry(preset)
+        # FIXME: When to use 'legacy' instead?
+        runtime_config.update_type_registry(get_type_registry('core'))
+        runtime_config.update_type_registry(get_type_registry(self._config.type_registry))
 
         return runtime_config
 
     def get_spec_version(self, name: str) -> SubstrateSpecVersion:
         if name not in self._spec_versions:
             _logger.info('loading spec version `%s`', name)
             try:
-                metadata = orjson.loads(self._package.abi.joinpath(self._name, f'v{name}.json').read_bytes())
+                metadata = orjson.loads(self._package.abi.joinpath(self._config.name, f'v{name}.json').read_bytes())
                 self._spec_versions[name] = SubstrateSpecVersion(
                     name=f'v{name}',
                     metadata=metadata,
                 )
             except FileNotFoundError:
                 # FIXME: Using last known version to help with missing abis
-                last_known = tuple(self._package.abi.joinpath(self._name).glob('v*.json'))[-1].stem
+                last_known = tuple(self._package.abi.joinpath(self._config.name).glob('v*.json'))[-1].stem
                 _logger.info('using last known version `%s`', last_known)
                 self._spec_versions[name] = self.get_spec_version(last_known[1:])