From 73a915fffdb1dc16a583576de11448f26b710b8d Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 13 Nov 2024 10:45:46 +0100 Subject: [PATCH] Static `__all__`, enhanced find_missing and hooks (#4) Changes: - expand find_missing - add sorted_exports - bump version - add hooks - handle sets properly --- docs/helpers.md | 2 +- docs/release-notes.md | 12 ++ docs/specials.md | 90 +++++++++ docs/testing.md | 24 ++- docs/tutorial.md | 87 ++++++++- monkay/__about__.py | 2 +- monkay/__init__.py | 11 +- monkay/base.py | 275 +++++++++++++++++++++------- tests/targets/module_broken_all.py | 1 + tests/targets/module_full.py | 11 ++ tests/targets/module_missing_all.py | 9 + tests/targets/module_prefixed.py | 25 +++ tests/targets/module_shadowed.py | 13 ++ tests/test_basic.py | 44 ++--- tests/test_hooks.py | 89 +++++++++ tests/test_missing.py | 62 +++++++ 16 files changed, 649 insertions(+), 108 deletions(-) create mode 100644 tests/targets/module_broken_all.py create mode 100644 tests/targets/module_missing_all.py create mode 100644 tests/targets/module_prefixed.py create mode 100644 tests/targets/module_shadowed.py create mode 100644 tests/test_hooks.py create mode 100644 tests/test_missing.py diff --git a/docs/helpers.md b/docs/helpers.md index ca457eb..ea0139c 100644 --- a/docs/helpers.md +++ b/docs/helpers.md @@ -3,7 +3,7 @@ Monkay comes with some helpers -- `load(path, *, allow_splits=":.", package=None)`: Load a path like Monkay. `allow_splits` allows to configure if attributes are seperated via . or :. +- `load(path, *, allow_splits=":.", package=None)`: Load a path like Monkay. `allow_splits` allows to configure if attributes are separated via . or :. When both are specified, both split ways are possible (Default). - `load_any(module_path, potential_attrs, *, non_first_deprecated=False, package=None)`: Checks for a module if any attribute name matches. Return attribute value or raises ImportError when non matches. When `non_first_deprecated` is `True`, a DeprecationMessage is issued for the non-first attribute which matches. This can be handy for deprecating module interfaces. diff --git a/docs/release-notes.md b/docs/release-notes.md index e722e0c..9c2e8cf 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,17 @@ # Release notes +## Version 0.0.5 + +### Added + +- `sorted_exports` for sorted `__all__` exports. +- Hooks for add_lazy_import, add_deprecated_lazy_import. + +### Changed + +- `find_missing` test method has some different error names. +- `find_missing` doesn't require the all_var anymore. + ## Version 0.0.4 ### Added diff --git a/docs/specials.md b/docs/specials.md index 9343de0..612d8c6 100644 --- a/docs/specials.md +++ b/docs/specials.md @@ -4,3 +4,93 @@ Provide the `package` parameter to Monkay. By default it is set to the `__spec__.parent` of the module. For a toplevel module it is the same name like the module. + +## Adding dynamically lazy imports + +For adding lazy imports there are two methods: + +- `add_lazy_import(export_name, path_or_fn, *, no_hooks=False)`: Adding new lazy import or fail if already exist. +- `add_deprecated_lazy_import(export_name, DeprecatedImport, *, no_hoosk=False)`: Adding new deprecated lazy import or fail if already exist. + +By default the `__all__` variable is not modified but sometimes this is desirable. + +For this cases hooks exist: + +- `pre_add_lazy_import_hook(key, value, type_: Literal["lazy_import" | "deprecated_lazy_import"])`: Wrap around key and value and takes as third parameter the type. +- `post_add_lazy_import_hook(key)`: The way to go to update the `__all__` variable dynamically. + +The hooks are only executed when manually adding a lazy import and not during the setup of Monkay. + +### Example: Automatically update `__all__` + +``` python + +from monkay import Monkay + +# we use a set +__all__ = {"bar"} + +monkay = Monkay( + # required for autohooking + globals(), + lazy_imports={ + "bar": "tests.targets.fn_module:bar", + }, + settings_path="settings_path:Settings", + post_add_lazy_import_hook=__all__.add +) + +if monkay.settings.with_deprecated: + monkay.add_deprecated_lazy_import( + "deprecated", + { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + } + ) + # __all__ has now also deprecated when with_deprecated is true +``` + +### Example: prefix lazy imports + +``` python + +from monkay import Monkay + +# we use a set +__all__ = {"bar"} + +def prefix_fn(name: str, value: Any, type_: str) -> tuple[str, Any]: + return f"{type_}_prefix_{name}", value + +monkay = Monkay( + # required for autohooking + globals(), + lazy_imports={ + "bar": "tests.targets.fn_module:bar", + }, + pre_add_lazy_import_hook=prefix_fn, + post_add_lazy_import_hook=__all__.add +) +monkay.add_deprecated_lazy_import( + "deprecated", + { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + } +) +# __all__, lazy_imports has now also deprecated under a type prefix name +# but we can skip the hooks with no_hooks=True + +monkay.add_deprecated_lazy_import( + "deprecated", + { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + }, + no_hooks=True +) +``` diff --git a/docs/testing.md b/docs/testing.md index 5248995..f1cde0f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -10,15 +10,15 @@ For tests, but not limited to, Monkay provides three methods returning a context - `with_instance(instance, * apply_extensions=False,use_extensions_overwrite=True)` -## Check lazy imports +## Check imports and exports -Monkay provides the debug method `find_missing(*, all_var=None, search_pathes=None, ignore_deprecated_import_errors=False, require_search_path_all_var=True)`. +Monkay provides the debug method `find_missing(*, all_var=True, search_pathes=None, ignore_deprecated_import_errors=False, require_search_path_all_var=True)`. It is quite expensive so it should be only called for debugging, testing and in error cases. It returns a dictionary containing items which had issues, e.g. imports failed or not in `__all__` variable. When providing `search_pathes` (module pathes as string), all exports are checked if they are in the value set of Monkey. -When providing `__all__` as `all_var`, it is checked for all imports. +When providing `__all__` or `True` as `all_var`, the variable or in case of `True` the module `__all__` is checked for missing exports/imports. Returned is a dictionary in the format: @@ -27,11 +27,13 @@ Returned is a dictionary in the format: Errors: -- `all_var`: key is not in the provided `__all__` variable -- `import`: key had an ImportError -- `search_path_extra`: key (here a path) is not included in lazy imports. -- `search_path_import`: import of key (here the search path) failed -- `search_path_all_var`: module imported as search path had no `__all__`. This error can be disabled with `require_search_path_all_var=False` +- `not_in_all_var`: Key is not in the provided `__all__` variable. +- `missing_attr`: Key (path) which was defined in an `__all__` variable does not exist or raises an AttributeError. +- `missing_all_var`: Key (search path or main module) had no `__all__`. For search pathes this error can be disabled via `require_search_path_all_var=False`. +- `import`: Key (module or function) raised an ImportError. +- `shadowed`: Key is defined as lazy_import but defined in the main module so the lazy import is not used. +- `search_path_extra`: Key (path) is not included in lazy imports. +- `search_path_import`: Import of key (here the search path) failed. ### Ignore import errors when lazy import is deprecated @@ -46,7 +48,11 @@ Using Monkay for tests is confortable and easy: import edgy def test_edgy_lazy_imports(): - assert not edgy.monkay.find_missing(all_var=edgy.__all__, search_pathes=["edgy.core.files", "edgy.core.db.fields", "edgy.core.connection"]) + missing = edgy.monkay.find_missing(all_var=edgy.__all__, search_pathes=["edgy.core.files", "edgy.core.db.fields", "edgy.core.connection"]) + # remove false positives + if missing["AutoNowMixin"] == {"AutoNowMixin"}: + del missing["AutoNowMixin"] + assert not missing ``` diff --git a/docs/tutorial.md b/docs/tutorial.md index 4438d8a..cf8afc2 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -48,6 +48,8 @@ When providing your own `__all__` variable **after** providing Monkay or you wan and update the `__all__` value via `Monkay.update_all_var` if wanted. + + #### Lazy imports When using lazy imports the globals get an `__getattr__` injected. A potential old `__getattr__` is used as fallback when provided **before** @@ -79,7 +81,7 @@ There is also a method: Which can be used to clear the caches. -##### Monkey +##### Monkey #### Using settings @@ -255,3 +257,86 @@ There is a second more complicated way to reorder: via the parameter `extension_order_key_fn`. It takes a key function which is expected to return a lexicographic key capable for ordering. You can however intermix both. + + +## Tricks + +### Type-checker friendly lazy imports + +Additionally to the lazy imports you can define in the `TYPE_CHECKING` scope the imports of the types. + +They are not loaded except when type checking. + +``` python + +from typing import TYPE_CHECKING + +from monkay import Monkay + +if TYPE_CHECKING: + from tests.targets.fn_module import bar + +monkay = Monkay( + # required for autohooking + globals(), + lazy_imports={ + "bar": "tests.targets.fn_module:bar", + }, +) +``` + + +### Static `__all__` + +For autocompletions it is helpful to have a static `__all__` variable because many tools parse the sourcecode. +Handling the `__all__` manually is for small imports easy but for bigger projects problematic. + +Let's extend the former example: + + +``` python + +import os +from typing import TYPE_CHECKING + +from monkay import Monkay + +if TYPE_CHECKING: + from tests.targets.fn_module import bar + +__all__ =["bar", "monkay", "stringify_all", "check"] + +monkay = Monkay( + # required for autohooking + globals(), + lazy_imports={ + "bar": "tests.targets.fn_module:bar", + }, + skip_all_update=not os.environ.get("DEBUG") + # when printing all, lazy imports automatically add to __all__ + post_add_lazy_import_hook=__all__.append if __name__ == "__main__" else None +) + + +def print_stringify_all(separate_by_category: bool=True) -> None: + print("__all__ = [\n{}\n]".format( + "\n,".join( + f'"{t[1]}"' + for t in monkay.sorted_exports(separate_by_category=separate_by_category) + ) + )) + +def check() -> None: + if monkay.find_missing(search_pathes=["tests.targets.fn_module"]): + raise Exception() + +if __name__ == "__main__": + # refresh __all__ to contain all lazy imports + __all__ = monkay.update_all_var(__all__) + print_stringify_all() +elif os.environ.get("DEBUG"): + check() +``` + +This way in a debug environment the imports are automatically checked. +And via `python -m mod` the new `__all__` can be exported. diff --git a/monkay/__about__.py b/monkay/__about__.py index 537a067..23f4b09 100644 --- a/monkay/__about__.py +++ b/monkay/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2024-present alex # # SPDX-License-Identifier: BSD-3-Clauses -__version__ = "0.0.4" +__version__ = "0.0.5" diff --git a/monkay/__init__.py b/monkay/__init__.py index 3001856..7df3868 100644 --- a/monkay/__init__.py +++ b/monkay/__init__.py @@ -2,11 +2,20 @@ # # SPDX-License-Identifier: BSD-3-Clauses -from .base import DeprecatedImport, ExtensionProtocol, Monkay, load, load_any +from .base import ( + PRE_ADD_LAZY_IMPORT_HOOK, + DeprecatedImport, + ExtensionProtocol, + Monkay, + load, + load_any, +) __all__ = [ "Monkay", + "SortedExportsEntry", "DeprecatedImport", + "PRE_ADD_LAZY_IMPORT_HOOK", "ExtensionProtocol", "load", "load_any", diff --git a/monkay/base.py b/monkay/base.py index 9ea0276..92e69ff 100644 --- a/monkay/base.py +++ b/monkay/base.py @@ -1,22 +1,24 @@ from __future__ import annotations import warnings -from collections.abc import Callable, Generator, Iterable, Sequence +from collections.abc import Callable, Collection, Generator, Iterable from contextlib import contextmanager from contextvars import ContextVar from functools import cached_property, partial from importlib import import_module -from inspect import isclass +from inspect import isclass, ismodule from itertools import chain from typing import ( TYPE_CHECKING, Any, Generic, Literal, + NamedTuple, Protocol, TypedDict, TypeVar, cast, + overload, runtime_checkable, ) @@ -27,6 +29,12 @@ SETTINGS = TypeVar("SETTINGS", bound="BaseSettings") +class SortedExportsEntry(NamedTuple): + category: Literal["other", "lazy_import", "deprecated_lazy_import"] + export_name: str + path: str + + class DeprecatedImport(TypedDict, total=False): path: str | Callable[[], Any] reason: str @@ -36,6 +44,34 @@ class DeprecatedImport(TypedDict, total=False): DeprecatedImport.__required_keys__ = frozenset({"deprecated"}) +class PRE_ADD_LAZY_IMPORT_HOOK(Protocol): + @overload + @staticmethod + def __call__( + key: str, + value: str | Callable[[], Any], + type_: Literal["lazy_import"], + /, + ) -> tuple[str, str | Callable[[], Any]]: ... + + @overload + @staticmethod + def __call__( + key: str, + value: DeprecatedImport, + type_: Literal["deprecated_lazy_import"], + /, + ) -> tuple[str, DeprecatedImport]: ... + + @staticmethod + def __call__( + key: str, + value: str | Callable[[], Any] | DeprecatedImport, + type_: Literal["lazy_import", "deprecated_lazy_import"], + /, + ) -> tuple[str, str | Callable[[], Any] | DeprecatedImport]: ... + + def load(path: str, *, allow_splits: str = ":.", package: None | str = None) -> Any: splitted = path.rsplit(":", 1) if ":" in allow_splits else [] if len(splitted) < 2 and "." in allow_splits: @@ -48,7 +84,7 @@ def load(path: str, *, allow_splits: str = ":.", package: None | str = None) -> def load_any( path: str, - attrs: Sequence[str], + attrs: Collection[str], *, non_first_deprecated: bool = False, package: None | str = None, @@ -70,23 +106,6 @@ def load_any( raise ImportError(f"Could not import any of the attributes:.{', '.join(attrs)}") -@runtime_checkable -class ExtensionProtocol(Protocol[INSTANCE, SETTINGS]): - name: str - - def apply(self, monkay_instance: Monkay[INSTANCE, SETTINGS]) -> None: ... - - -def _stub_previous_getattr(name: str) -> Any: - raise AttributeError(f'Module has no attribute: "{name}" (Monkay).') - - -def _obj_to_full_name(obj: Any) -> str: - if not isclass(obj): - obj = type(obj) - return f"{obj.__module__}.{obj.__qualname__}" - - def absolutify_import(import_path: str, package: str | None) -> str: if not package or not import_path: return import_path @@ -106,6 +125,29 @@ def absolutify_import(import_path: str, package: str | None) -> str: return f"{package}.{import_path.lstrip('.')}" +@runtime_checkable +class ExtensionProtocol(Protocol[INSTANCE, SETTINGS]): + name: str + + def apply(self, monkay_instance: Monkay[INSTANCE, SETTINGS]) -> None: ... + + +class InGlobalsDict(Exception): + pass + + +def _stub_previous_getattr(name: str) -> Any: + raise AttributeError(f'Module has no attribute: "{name}" (Monkay).') + + +def _obj_to_full_name(obj: Any) -> str: + if ismodule(obj): + return obj.__spec__.name # type: ignore + if not isclass(obj): + obj = type(obj) + return f"{obj.__module__}.{obj.__qualname__}" + + class Monkay(Generic[INSTANCE, SETTINGS]): getter: Callable[..., Any] _instance: None | INSTANCE = None @@ -118,7 +160,7 @@ class Monkay(Generic[INSTANCE, SETTINGS]): def __init__( self, - global_dict: dict, + globals_dict: dict, *, with_instance: str | bool = False, with_extensions: str | bool = False, @@ -134,31 +176,36 @@ def __init__( settings_ctx_name: str = "monkay_settings_ctx", extensions_applied_ctx_name: str = "monkay_extensions_applied_ctx", skip_all_update: bool = False, + pre_add_lazy_import_hook: None | PRE_ADD_LAZY_IMPORT_HOOK = None, + post_add_lazy_import_hook: None | Callable[[str], None] = None, package: str | None = "", ) -> None: + self.globals_dict = globals_dict if with_instance is True: with_instance = "monkay_instance_ctx" with_instance = with_instance if with_extensions is True: with_extensions = "monkay_extensions_ctx" with_extensions = with_extensions - if package == "" and global_dict.get("__spec__"): - package = global_dict["__spec__"].parent + if package == "" and globals_dict.get("__spec__"): + package = globals_dict["__spec__"].parent self.package = package or None self._cached_imports: dict[str, Any] = {} + self.pre_add_lazy_import_hook: None | PRE_ADD_LAZY_IMPORT_HOOK = pre_add_lazy_import_hook + self.post_add_lazy_import_hook = post_add_lazy_import_hook self.uncached_imports: set[str] = set(uncached_imports) self.lazy_imports: dict[str, str | Callable[[], Any]] = {} self.deprecated_lazy_imports: dict[str, DeprecatedImport] = {} if lazy_imports: for name, lazy_import in lazy_imports.items(): - self.add_lazy_import(name, lazy_import) + self.add_lazy_import(name, lazy_import, no_hooks=True) if deprecated_lazy_imports: for name, deprecated_import in deprecated_lazy_imports.items(): - self.add_deprecated_lazy_import(name, deprecated_import) + self.add_deprecated_lazy_import(name, deprecated_import, no_hooks=True) self.settings_path = settings_path if self.settings_path: - self._settings_var = global_dict[settings_ctx_name] = ContextVar(settings_ctx_name, default=None) + self._settings_var = globals_dict[settings_ctx_name] = ContextVar(settings_ctx_name, default=None) if settings_preload_name: warnings.warn( @@ -172,24 +219,24 @@ def __init__( self.settings_extensions_name = settings_extensions_name self._handle_preloads(preloads) - if self.lazy_imports or self.deprecated_lazy_imports: - getter: Callable[..., Any] = self.module_getter - if "__getattr__" in global_dict: - getter = partial(getter, chained_getter=global_dict["__getattr__"]) - global_dict["__getattr__"] = self.getter = getter - if not skip_all_update: - all_var = global_dict.setdefault("__all__", []) - global_dict["__all__"] = self.update_all_var(all_var) if with_instance: - self._instance_var = global_dict[with_instance] = ContextVar(with_instance, default=None) + self._instance_var = globals_dict[with_instance] = ContextVar(with_instance, default=None) if with_extensions: self.extension_order_key_fn = extension_order_key_fn self._extensions = {} - self._extensions_var = global_dict[with_extensions] = ContextVar(with_extensions, default=None) - self._extensions_applied_var = global_dict[extensions_applied_ctx_name] = ContextVar( + self._extensions_var = globals_dict[with_extensions] = ContextVar(with_extensions, default=None) + self._extensions_applied_var = globals_dict[extensions_applied_ctx_name] = ContextVar( extensions_applied_ctx_name, default=None ) self._handle_extensions() + if self.lazy_imports or self.deprecated_lazy_imports: + getter: Callable[..., Any] = self.module_getter + if "__getattr__" in globals_dict: + getter = partial(getter, chained_getter=globals_dict["__getattr__"]) + globals_dict["__getattr__"] = self.getter = getter + if not skip_all_update: + all_var = globals_dict.setdefault("__all__", []) + globals_dict["__all__"] = self.update_all_var(all_var) def clear_caches(self, settings_cache: bool = True, import_cache: bool = True) -> None: if settings_cache: @@ -330,35 +377,45 @@ def with_extensions( finally: self._extensions_var.reset(token) - def update_all_var(self, all_var: Sequence[str]) -> list[str]: - if not isinstance(all_var, list): - all_var = list(all_var) - all_var_set = set(all_var) + def update_all_var(self, all_var: Collection[str]) -> list[str] | set[str]: + if isinstance(all_var, set): + all_var_set = all_var + else: + if not isinstance(all_var, list): + all_var = list(all_var) + all_var_set = set(all_var) + if self.lazy_imports or self.deprecated_lazy_imports: for var in chain( self.lazy_imports, self.deprecated_lazy_imports, ): if var not in all_var_set: - all_var.append(var) - return all_var + if isinstance(all_var, list): + all_var.append(var) + else: + cast(set[str], all_var).add(var) + + return cast("list[str] | set[str]", all_var) def find_missing( self, *, - all_var: None | Sequence[str] = None, - search_pathes: None | Sequence[str] = None, + all_var: bool | Collection[str] = True, + search_pathes: None | Collection[str] = None, ignore_deprecated_import_errors: bool = False, require_search_path_all_var: bool = True, ) -> dict[ str, set[ Literal[ - "all_var", + "not_in_all_var", + "missing_attr", + "missing_all_var", "import", + "shadowed", "search_path_extra", "search_path_import", - "search_path_all_var", ] ], ]: @@ -369,14 +426,22 @@ def find_missing( str, set[ Literal[ - "all_var", + "not_in_all_var", + "missing_attr", + "missing_all_var", "import", + "shadowed", "search_path_extra", "search_path_import", - "search_path_all_var", ] ], ] = {} + if all_var is True: + try: + all_var = self.getter("__all__", check_globals_dict=True) + except AttributeError: + missing.setdefault(self.globals_dict["__spec__"].name, set()).add("missing_all_var") + all_var = [] key_set = set(chain(self.lazy_imports.keys(), self.deprecated_lazy_imports.keys())) value_pathes_set: set[str] = set() for name in key_set: @@ -388,12 +453,24 @@ def find_missing( if found_path: value_pathes_set.add(absolutify_import(found_path, self.package)) try: - returnobj = self.getter(name, no_warn_deprecated=True) + obj = self.getter(name, no_warn_deprecated=True, check_globals_dict="fail") if not found_path: - value_pathes_set.add(_obj_to_full_name(returnobj)) + value_pathes_set.add(_obj_to_full_name(obj)) + except InGlobalsDict: + missing.setdefault(name, set()).add("shadowed") except ImportError: if not ignore_deprecated_import_errors or name not in self.deprecated_lazy_imports: missing.setdefault(name, set()).add("import") + if all_var is not False: + for export_name in cast(Collection[str], all_var): + try: + obj = self.getter(export_name, no_warn_deprecated=True, check_globals_dict=True) + except AttributeError: + missing.setdefault(export_name, set()).add("missing_attr") + continue + if export_name not in key_set: + value_pathes_set.add(_obj_to_full_name(obj)) + if search_pathes: for search_path in search_pathes: try: @@ -405,16 +482,21 @@ def find_missing( all_var_search = mod.__all__ except AttributeError: if require_search_path_all_var: - missing.setdefault(search_path, set()).add("search_path_all_var") + missing.setdefault(search_path, set()).add("missing_all_var") continue for export_name in all_var_search: export_path = absolutify_import(f"{search_path}.{export_name}", self.package) if export_path not in value_pathes_set: missing.setdefault(export_path, set()).add("search_path_extra") - if all_var is not None: - for name in key_set.difference(all_var): - missing.setdefault(name, set()).add("all_var") + try: + getattr(mod, export_name) + except AttributeError: + missing.setdefault(export_path, set()).add("missing_attr") + + if all_var is not False: + for name in key_set.difference(cast(Collection[str], all_var)): + missing.setdefault(name, set()).add("not_in_all_var") return missing @@ -443,27 +525,85 @@ def with_settings(self, settings: SETTINGS | None) -> Generator: finally: self._settings_var.reset(token) - def add_lazy_import( - self, - name: str, - value: str | Callable[[], Any], - ) -> None: + def add_lazy_import(self, name: str, value: str | Callable[[], Any], *, no_hooks: bool = False) -> None: + if not no_hooks and self.pre_add_lazy_import_hook is not None: + name, value = self.pre_add_lazy_import_hook(name, value, "lazy_import") if name in self.lazy_imports: raise KeyError(f'"{name}" is already a lazy import') if name in self.deprecated_lazy_imports: raise KeyError(f'"{name}" is already a deprecated lazy import') self.lazy_imports[name] = value + if not no_hooks and self.post_add_lazy_import_hook is not None: + self.post_add_lazy_import_hook(name) - def add_deprecated_lazy_import( - self, - name: str, - value: DeprecatedImport, - ) -> None: + def add_deprecated_lazy_import(self, name: str, value: DeprecatedImport, *, no_hooks: bool = False) -> None: + if not no_hooks and self.pre_add_lazy_import_hook is not None: + name, value = self.pre_add_lazy_import_hook(name, value, "deprecated_lazy_import") if name in self.lazy_imports: raise KeyError(f'"{name}" is already a lazy import') if name in self.deprecated_lazy_imports: raise KeyError(f'"{name}" is already a deprecated lazy import') self.deprecated_lazy_imports[name] = value + if not no_hooks and self.post_add_lazy_import_hook is not None: + self.post_add_lazy_import_hook(name) + + def sorted_exports( + self, + all_var: Collection[str] | None = None, + *, + separate_by_category: bool = True, + sort_by: Literal["export_name", "path"] = "path", + ) -> list[SortedExportsEntry]: + if all_var is None: + all_var = self.globals_dict["__all__"] + sorted_exports: list[SortedExportsEntry] = [] + # ensure all entries are only returned once + for name in set(all_var): + if name in self.lazy_imports: + sorted_exports.append( + SortedExportsEntry( + "lazy_import", + name, + cast( + str, + self.lazy_imports[name] + if isinstance(self.lazy_imports[name], str) + else f"{self.globals_dict['__spec__'].name}.{name}", + ), + ) + ) + elif name in self.deprecated_lazy_imports: + sorted_exports.append( + SortedExportsEntry( + "deprecated_lazy_import", + name, + cast( + str, + self.deprecated_lazy_imports[name]["path"] + if isinstance(self.deprecated_lazy_imports[name]["path"], str) + else f"{self.globals_dict['__spec__'].name}.{name}", + ), + ) + ) + else: + sorted_exports.append( + SortedExportsEntry( + "other", + name, + f"{self.globals_dict['__spec__'].name}.{name}", + ) + ) + if separate_by_category: + + def key_fn(ordertuple: SortedExportsEntry) -> tuple: + return ordertuple.category, getattr(ordertuple, sort_by) + else: + + def key_fn(ordertuple: SortedExportsEntry) -> tuple: + return (getattr(ordertuple, sort_by),) + + sorted_exports.sort(key=key_fn) + return sorted_exports def module_getter( self, @@ -471,12 +611,17 @@ def module_getter( *, chained_getter: Callable[[str], Any] = _stub_previous_getattr, no_warn_deprecated: bool = False, + check_globals_dict: bool | Literal["fail"] = False, ) -> Any: """ Module Getter which handles lazy imports. The injected version containing a potential found __getattr__ handler as chained_getter is availabe as getter attribute. """ + if check_globals_dict and key in self.globals_dict: + if check_globals_dict == "fail": + raise InGlobalsDict(f'"{key}" is defined as real variable.') + return self.globals_dict[key] lazy_import = self.lazy_imports.get(key) if lazy_import is None: deprecated = self.deprecated_lazy_imports.get(key) diff --git a/tests/targets/module_broken_all.py b/tests/targets/module_broken_all.py new file mode 100644 index 0000000..d06567c --- /dev/null +++ b/tests/targets/module_broken_all.py @@ -0,0 +1 @@ +__all__ = ["broken"] # noqa diff --git a/tests/targets/module_full.py b/tests/targets/module_full.py index 0b90e8d..c7a1345 100644 --- a/tests/targets/module_full.py +++ b/tests/targets/module_full.py @@ -14,6 +14,7 @@ class FakeApp: is_fake_app: bool = True +__all__ = ["foo", "stringify_all"] # noqa monkay = Monkay( globals(), with_extensions=True, @@ -37,3 +38,13 @@ class FakeApp: } }, ) + + +def stringify_all_plain(separate_by_category: bool): + return "[\n{}\n]".format( + "\n,".join(f'"{t[1]}"' for t in monkay.sorted_exports(separate_by_category=separate_by_category)) + ) + + +def stringify_all(separate_by_category: bool): + return f"__all__ = {stringify_all_plain(separate_by_category)}" diff --git a/tests/targets/module_missing_all.py b/tests/targets/module_missing_all.py new file mode 100644 index 0000000..a0b0b59 --- /dev/null +++ b/tests/targets/module_missing_all.py @@ -0,0 +1,9 @@ +from monkay import Monkay + +monkay = Monkay( + globals(), + lazy_imports={ + "bar": ".fn_module:bar", + }, + skip_all_update=True, +) diff --git a/tests/targets/module_prefixed.py b/tests/targets/module_prefixed.py new file mode 100644 index 0000000..37df2af --- /dev/null +++ b/tests/targets/module_prefixed.py @@ -0,0 +1,25 @@ +from typing import Any + +from monkay import Monkay + + +def prefix_fn(name: str, value: Any, type_: str) -> tuple[str, Any]: + return f"{type_}_{name}", value + + +__all__ = {"foo", "monkay"} # noqa +monkay = Monkay( + globals(), + lazy_imports={ + "bar": ".fn_module:bar", + }, + deprecated_lazy_imports={ + "deprecated": { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + } + }, + pre_add_lazy_import_hook=prefix_fn, + post_add_lazy_import_hook=__all__.add, +) diff --git a/tests/targets/module_shadowed.py b/tests/targets/module_shadowed.py new file mode 100644 index 0000000..cd434ab --- /dev/null +++ b/tests/targets/module_shadowed.py @@ -0,0 +1,13 @@ +from monkay import Monkay + + +def bar(): + return "notbar" + + +monkay = Monkay( + globals(), + lazy_imports={ + "bar": ".fn_module:bar", + }, +) diff --git a/tests/test_basic.py b/tests/test_basic.py index 49093d8..9f2efd7 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,6 +1,7 @@ import contextlib import sys from io import StringIO +from pathlib import Path import pytest from pydantic_settings import BaseSettings @@ -10,14 +11,8 @@ @pytest.fixture(autouse=True, scope="function") def cleanup(): - for name in [ - "module_full_preloaded1_fn", - "module_full_preloaded1", - "module_preloaded1", - "module_full", - "fn_module", - ]: - sys.modules.pop(f"tests.targets.{name}", None) + for p in (Path(__file__).parent / "targets").iterdir(): + sys.modules.pop(f"tests.targets.{p.stem}", None) yield @@ -193,27 +188,16 @@ def test_caches(): assert "_settings" not in mod.monkay.__dict__ -def test_find_missing(): +def test_sorted_exports(): import tests.targets.module_full as mod - # __all__ is autogenerated - assert not mod.monkay.find_missing(all_var=mod.__all__, search_pathes=["tests.targets.fn_module"]) - assert mod.monkay.find_missing( - all_var=mod.__all__, - search_pathes=["tests.targets.not_existing", "tests.targets.module_preloaded1"], - ) == { - "tests.targets.not_existing": {"search_path_import"}, - "tests.targets.module_preloaded1.not_included_export": {"search_path_extra"}, - } - assert mod.monkay.find_missing(all_var={}, search_pathes=["tests.targets.module_full_preloaded1"]) == { - "bar": {"all_var"}, - "bar2": { - "all_var", - }, - "dynamic": {"all_var"}, - "settings": {"all_var"}, - "deprecated": {"all_var"}, - "tests.targets.module_full_preloaded1": { - "search_path_all_var", - }, - } + eval(mod.stringify_all_plain(separate_by_category=True)) + eval(mod.stringify_all_plain(separate_by_category=False)) + assert ( + mod.stringify_all(separate_by_category=True) + == '__all__ = [\n"deprecated"\n,"bar2"\n,"bar"\n,"dynamic"\n,"settings"\n,"foo"\n,"stringify_all"\n]' + ) + assert ( + mod.stringify_all(separate_by_category=False) + == '__all__ = [\n"bar2"\n,"bar"\n,"deprecated"\n,"dynamic"\n,"foo"\n,"settings"\n,"stringify_all"\n]' + ) diff --git a/tests/test_hooks.py b/tests/test_hooks.py new file mode 100644 index 0000000..c9517b7 --- /dev/null +++ b/tests/test_hooks.py @@ -0,0 +1,89 @@ +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True, scope="function") +def cleanup(): + for p in (Path(__file__).parent / "targets").iterdir(): + sys.modules.pop(f"tests.targets.{p.stem}", None) + yield + + +def test_hooks(): + import tests.targets.module_prefixed as mod + + assert "bar" in mod.__all__ + assert "deprecated" in mod.__all__ + + +@pytest.mark.parametrize( + "monkay_fn", + [ + lambda m: m.add_lazy_import("bar", ".fn_module:bar", no_hooks=True), + lambda m: m.add_deprecated_lazy_import( + "deprecated", + { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + }, + no_hooks=True, + ), + ], +) +def test_no_hooks_collisions(monkay_fn): + import tests.targets.module_prefixed as mod + + with pytest.raises(KeyError): + monkay_fn(mod.monkay) + + +@pytest.mark.parametrize( + "monkay_fn,export,in_all", + [ + (lambda m: m.add_lazy_import("bar", ".fn_module:bar"), "lazy_import_bar", True), + ( + lambda m: m.add_deprecated_lazy_import( + "deprecated", + { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + }, + ), + "deprecated_lazy_import_deprecated", + True, + ), + ( + lambda m: m.add_lazy_import( + "bar2", + ".fn_module:bar", + no_hooks=True, + ), + "bar2", + False, + ), + ( + lambda m: m.add_deprecated_lazy_import( + "deprecated2", + { + "path": "tests.targets.fn_module:deprecated", + "reason": "old", + "new_attribute": "super_new", + }, + no_hooks=True, + ), + "deprecated2", + False, + ), + ], +) +def test_add(monkay_fn, export, in_all): + import tests.targets.module_prefixed as mod + + monkay_fn(mod.monkay) + if in_all: + assert export in mod.__all__ + assert mod.monkay.getter(export, no_warn_deprecated=True, check_globals_dict=True) is not None diff --git a/tests/test_missing.py b/tests/test_missing.py new file mode 100644 index 0000000..866e36f --- /dev/null +++ b/tests/test_missing.py @@ -0,0 +1,62 @@ +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True, scope="function") +def cleanup(): + for p in (Path(__file__).parent / "targets").iterdir(): + sys.modules.pop(f"tests.targets.{p.stem}", None) + yield + + +def test_find_missing(): + import tests.targets.module_full as mod + + # __all__ is autogenerated + assert not mod.monkay.find_missing(all_var=mod.__all__, search_pathes=["tests.targets.fn_module"]) + # we can also use bools + assert not mod.monkay.find_missing(all_var=True, search_pathes=["tests.targets.fn_module"]) + assert mod.monkay.find_missing( + search_pathes=["tests.targets.not_existing", "tests.targets.module_preloaded1"], + ) == { + "tests.targets.not_existing": {"search_path_import"}, + "tests.targets.module_preloaded1.not_included_export": {"search_path_extra"}, + } + assert mod.monkay.find_missing(all_var={}, search_pathes=["tests.targets.module_full_preloaded1"]) == { + "bar": {"not_in_all_var"}, + "bar2": { + "not_in_all_var", + }, + "dynamic": {"not_in_all_var"}, + "settings": {"not_in_all_var"}, + "deprecated": {"not_in_all_var"}, + "tests.targets.module_full_preloaded1": { + "missing_all_var", + }, + } + assert mod.monkay.find_missing(search_pathes=["tests.targets.module_missing_all"]) == { + "tests.targets.module_missing_all": {"missing_all_var"}, + } + + assert mod.monkay.find_missing(search_pathes=["tests.targets.module_broken_all"]) == { + "tests.targets.module_broken_all.broken": {"missing_attr", "search_path_extra"}, + } + + +def test_find_missing_no_all(): + from .targets import module_missing_all as mod + + assert mod.monkay.find_missing() == { + "bar": {"not_in_all_var"}, + "tests.targets.module_missing_all": {"missing_all_var"}, + } + + +def test_find_missing_shadowed(): + from .targets import module_shadowed as mod + + assert mod.monkay.find_missing() == { + "bar": {"shadowed"}, + }