Skip to content

Commit

Permalink
Reject invalid ParamSpec locations (#18278)
Browse files Browse the repository at this point in the history 
Fixes #14832, fixes #13966, fixes #14622.

Still does not report error in #14777, I'll work separately on that.

Move all `ParamSpec` validity checking to `typeanal.py`. Stop treating
`P.args` and `P.kwargs` as binding - only bare typevar makes it
available in scope. Reject keyword arguments following `P.args`.

This also makes one more conformance test pass.
  • Loading branch information
@sterliakov
sterliakov authored Jan 2, 2025
1 parent e05770d commit 025642b
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 147 deletions.
60 changes: 0 additions & 60 deletions mypy/semanal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@
from mypy.nodes import (
ARG_NAMED,
ARG_POS,
ARG_STAR,
ARG_STAR2,
CONTRAVARIANT,
COVARIANT,
Expand Down Expand Up @@ -981,7 +980,6 @@ def analyze_func_def(self, defn: FuncDef) -> None:
defn.type = result
self.add_type_alias_deps(analyzer.aliases_used)
self.check_function_signature(defn)
self.check_paramspec_definition(defn)
if isinstance(defn, FuncDef):
assert isinstance(defn.type, CallableType)
defn.type = set_callable_name(defn.type, defn)
Expand Down Expand Up @@ -1610,64 +1608,6 @@ def check_function_signature(self, fdef: FuncItem) -> None:
elif len(sig.arg_types) > len(fdef.arguments):
self.fail("Type signature has too many arguments", fdef, blocker=True)

def check_paramspec_definition(self, defn: FuncDef) -> None:
func = defn.type
assert isinstance(func, CallableType)

if not any(isinstance(var, ParamSpecType) for var in func.variables):
return # Function does not have param spec variables

args = func.var_arg()
kwargs = func.kw_arg()
if args is None and kwargs is None:
return # Looks like this function does not have starred args

args_defn_type = None
kwargs_defn_type = None
for arg_def, arg_kind in zip(defn.arguments, defn.arg_kinds):
if arg_kind == ARG_STAR:
args_defn_type = arg_def.type_annotation
elif arg_kind == ARG_STAR2:
kwargs_defn_type = arg_def.type_annotation

# This may happen on invalid `ParamSpec` args / kwargs definition,
# type analyzer sets types of arguments to `Any`, but keeps
# definition types as `UnboundType` for now.
if not (
(isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"))
or (
isinstance(kwargs_defn_type, UnboundType)
and kwargs_defn_type.name.endswith(".kwargs")
)
):
# Looks like both `*args` and `**kwargs` are not `ParamSpec`
# It might be something else, skipping.
return

args_type = args.typ if args is not None else None
kwargs_type = kwargs.typ if kwargs is not None else None

if (
not isinstance(args_type, ParamSpecType)
or not isinstance(kwargs_type, ParamSpecType)
or args_type.name != kwargs_type.name
):
if isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"):
param_name = args_defn_type.name.split(".")[0]
elif isinstance(kwargs_defn_type, UnboundType) and kwargs_defn_type.name.endswith(
".kwargs"
):
param_name = kwargs_defn_type.name.split(".")[0]
else:
# Fallback for cases that probably should not ever happen:
param_name = "P"

self.fail(
f'ParamSpec must have "*args" typed as "{param_name}.args" and "**kwargs" typed as "{param_name}.kwargs"',
func,
code=codes.VALID_TYPE,
)

def visit_decorator(self, dec: Decorator) -> None:
self.statement = dec
# TODO: better don't modify them at all.
Expand Down
119 changes: 67 additions & 52 deletions mypy/typeanal.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,15 @@ def not_declared_in_type_params(self, tvar_name: str) -> bool:

def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) -> Type:
sym = self.lookup_qualified(t.name, t)
param_spec_name = None
if t.name.endswith((".args", ".kwargs")):
param_spec_name = t.name.rsplit(".", 1)[0]
maybe_param_spec = self.lookup_qualified(param_spec_name, t)
if maybe_param_spec and isinstance(maybe_param_spec.node, ParamSpecExpr):
sym = maybe_param_spec
else:
param_spec_name = None

if sym is not None:
node = sym.node
if isinstance(node, PlaceholderNode):
Expand Down Expand Up @@ -362,17 +371,23 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool)
if tvar_def is None:
if self.allow_unbound_tvars:
return t
name = param_spec_name or t.name
if self.defining_alias and self.not_declared_in_type_params(t.name):
msg = f'ParamSpec "{t.name}" is not included in type_params'
msg = f'ParamSpec "{name}" is not included in type_params'
else:
msg = f'ParamSpec "{t.name}" is unbound'
msg = f'ParamSpec "{name}" is unbound'
self.fail(msg, t, code=codes.VALID_TYPE)
return AnyType(TypeOfAny.from_error)
assert isinstance(tvar_def, ParamSpecType)
if len(t.args) > 0:
self.fail(
f'ParamSpec "{t.name}" used with arguments', t, code=codes.VALID_TYPE
)
if param_spec_name is not None and not self.allow_param_spec_literals:
self.fail(
"ParamSpec components are not allowed here", t, code=codes.VALID_TYPE
)
return AnyType(TypeOfAny.from_error)
# Change the line number
return ParamSpecType(
tvar_def.name,
Expand Down Expand Up @@ -1113,46 +1128,57 @@ def visit_callable_type(
variables, _ = self.bind_function_type_variables(t, t)
type_guard = self.anal_type_guard(t.ret_type)
type_is = self.anal_type_is(t.ret_type)

arg_kinds = t.arg_kinds
if len(arg_kinds) >= 2 and arg_kinds[-2] == ARG_STAR and arg_kinds[-1] == ARG_STAR2:
arg_types = self.anal_array(t.arg_types[:-2], nested=nested) + [
self.anal_star_arg_type(t.arg_types[-2], ARG_STAR, nested=nested),
self.anal_star_arg_type(t.arg_types[-1], ARG_STAR2, nested=nested),
]
# If nested is True, it means we are analyzing a Callable[...] type, rather
# than a function definition type. We need to "unpack" ** TypedDict annotation
# here (for function definitions it is done in semanal).
if nested and isinstance(arg_types[-1], UnpackType):
arg_types = []
param_spec_with_args = param_spec_with_kwargs = None
param_spec_invalid = False
for kind, ut in zip(arg_kinds, t.arg_types):
if kind == ARG_STAR:
param_spec_with_args, at = self.anal_star_arg_type(ut, kind, nested=nested)
elif kind == ARG_STAR2:
param_spec_with_kwargs, at = self.anal_star_arg_type(ut, kind, nested=nested)
else:
if param_spec_with_args:
param_spec_invalid = True
self.fail(
"Arguments not allowed after ParamSpec.args", t, code=codes.VALID_TYPE
)
at = self.anal_type(ut, nested=nested, allow_unpack=False)
arg_types.append(at)

if nested and arg_types:
# If we've got a Callable[[Unpack[SomeTypedDict]], None], make sure
# Unpack is interpreted as `**` and not as `*`.
last = arg_types[-1]
if isinstance(last, UnpackType):
# TODO: it would be better to avoid this get_proper_type() call.
unpacked = get_proper_type(arg_types[-1].type)
if isinstance(unpacked, TypedDictType):
arg_types[-1] = unpacked
p_at = get_proper_type(last.type)
if isinstance(p_at, TypedDictType) and not last.from_star_syntax:
# Automatically detect Unpack[Foo] in Callable as backwards
# compatible syntax for **Foo, if Foo is a TypedDict.
arg_kinds[-1] = ARG_STAR2
arg_types[-1] = p_at
unpacked_kwargs = True
arg_types = self.check_unpacks_in_list(arg_types)
else:
star_index = None
arg_types = self.check_unpacks_in_list(arg_types)

if not param_spec_invalid and param_spec_with_args != param_spec_with_kwargs:
# If already invalid, do not report more errors - definition has
# to be fixed anyway
name = param_spec_with_args or param_spec_with_kwargs
self.fail(
f'ParamSpec must have "*args" typed as "{name}.args" and "**kwargs" typed as "{name}.kwargs"',
t,
code=codes.VALID_TYPE,
)
param_spec_invalid = True

if param_spec_invalid:
if ARG_STAR in arg_kinds:
star_index = arg_kinds.index(ARG_STAR)
star2_index = None
arg_types[arg_kinds.index(ARG_STAR)] = AnyType(TypeOfAny.from_error)
if ARG_STAR2 in arg_kinds:
star2_index = arg_kinds.index(ARG_STAR2)
arg_types = []
for i, ut in enumerate(t.arg_types):
at = self.anal_type(
ut, nested=nested, allow_unpack=i in (star_index, star2_index)
)
if nested and isinstance(at, UnpackType) and i == star_index:
# TODO: it would be better to avoid this get_proper_type() call.
p_at = get_proper_type(at.type)
if isinstance(p_at, TypedDictType) and not at.from_star_syntax:
# Automatically detect Unpack[Foo] in Callable as backwards
# compatible syntax for **Foo, if Foo is a TypedDict.
at = p_at
arg_kinds[i] = ARG_STAR2
unpacked_kwargs = True
arg_types.append(at)
if nested:
arg_types = self.check_unpacks_in_list(arg_types)
arg_types[arg_kinds.index(ARG_STAR2)] = AnyType(TypeOfAny.from_error)

# If there were multiple (invalid) unpacks, the arg types list will become shorter,
# we need to trim the kinds/names as well to avoid crashes.
arg_kinds = t.arg_kinds[: len(arg_types)]
Expand Down Expand Up @@ -1207,7 +1233,7 @@ def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None:
return self.anal_type(t.args[0])
return None

def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> tuple[str | None, Type]:
"""Analyze signature argument type for *args and **kwargs argument."""
if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args:
components = t.name.split(".")
Expand All @@ -1234,15 +1260,15 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
)
else:
assert False, kind
return make_paramspec(
return tvar_name, make_paramspec(
tvar_def.name,
tvar_def.fullname,
tvar_def.id,
named_type_func=self.named_type,
line=t.line,
column=t.column,
)
return self.anal_type(t, nested=nested, allow_unpack=True)
return None, self.anal_type(t, nested=nested, allow_unpack=True)

def visit_overloaded(self, t: Overloaded) -> Type:
# Overloaded types are manually constructed in semanal.py by analyzing the
Expand Down Expand Up @@ -2586,18 +2612,7 @@ def _seems_like_callable(self, type: UnboundType) -> bool:

def visit_unbound_type(self, t: UnboundType) -> None:
name = t.name
node = None

# Special case P.args and P.kwargs for ParamSpecs only.
if name.endswith("args"):
if name.endswith((".args", ".kwargs")):
base = ".".join(name.split(".")[:-1])
n = self.api.lookup_qualified(base, t)
if n is not None and isinstance(n.node, ParamSpecExpr):
node = n
name = base
if node is None:
node = self.api.lookup_qualified(name, t)
node = self.api.lookup_qualified(name, t)
if node and node.fullname in SELF_TYPE_NAMES:
self.has_self_type = True
if (
Expand Down
Loading

0 comments on commit 025642b

Please sign in to comment.