Fix mypy issues (#3979)

* Fix all mypy issues

* Ran black

* Exclude tox from mypy check

* Fix all mypy issues again

* Address PR comments

* Fix accidental line ending changes

* Update .gitignore

* No unused type: ignore

* TypeError: 'ABCMeta' object is not subscriptable

* Fix RuffError

* Fix post-merge mypy issues

* RUff format

* Ignore more generated files

* Disable more mypy errors

* Globally ignore attr-defined for now

* Update more comments

* Address PR comments and fix new exposed typing issues

* Comments updates and don't touch vendored

* Accidentally removed noqa

* Update setuptools/tests/integration/test_pip_install_sdist.py

Co-authored-by: Anderson Bravalheri <andersonbravalheri+github@gmail.com>

* Post merge comments

Update setuptools/tests/integration/test_pip_install_sdist.py

Co-authored-by: Anderson Bravalheri <andersonbravalheri+github@gmail.com>

* Document that usage of _config_vars is very purposeful Closes #4228
+ try to resolve doc issue

* sort nitpick_ignore

* Make only comment on newline like others

* Forgot to re-ignore

---------

Co-authored-by: Anderson Bravalheri <andersonbravalheri+github@gmail.com>

docs/conf.py

@@ -161,22 +161,23 @@
 # Ref: https://stackoverflow.com/a/30624034/595220
 nitpick_ignore = [
     ('c:func', 'SHGetSpecialFolderPath'),  # ref to MS docs
+    ('envvar', 'DIST_EXTRA_CONFIG'),  # undocumented
     ('envvar', 'DISTUTILS_DEBUG'),  # undocumented
     ('envvar', 'HOME'),  # undocumented
     ('envvar', 'PLAT'),  # undocumented
-    ('envvar', 'DIST_EXTRA_CONFIG'),  # undocumented
     ('py:attr', 'CCompiler.language_map'),  # undocumented
     ('py:attr', 'CCompiler.language_order'),  # undocumented
-    ('py:class', 'distutils.dist.Distribution'),  # undocumented
-    ('py:class', 'distutils.extension.Extension'),  # undocumented
     ('py:class', 'BorlandCCompiler'),  # undocumented
     ('py:class', 'CCompiler'),  # undocumented
     ('py:class', 'CygwinCCompiler'),  # undocumented
+    ('py:class', 'distutils.dist.Distribution'),  # undocumented
     ('py:class', 'distutils.dist.DistributionMetadata'),  # undocumented
+    ('py:class', 'distutils.extension.Extension'),  # undocumented
     ('py:class', 'FileList'),  # undocumented
     ('py:class', 'IShellLink'),  # ref to MS docs
     ('py:class', 'MSVCCompiler'),  # undocumented
     ('py:class', 'OptionDummy'),  # undocumented
+    ('py:class', 'setuptools.dist.Distribution'),  # undocumented
     ('py:class', 'UnixCCompiler'),  # undocumented
     ('py:exc', 'CompileError'),  # undocumented
     ('py:exc', 'DistutilsExecError'),  # undocumented
@@ -186,8 +187,7 @@
     ('py:exc', 'PreprocessError'),  # undocumented
     ('py:exc', 'setuptools.errors.PlatformError'),  # sphinx cannot find it
     ('py:func', 'distutils.CCompiler.new_compiler'),  # undocumented
-    # undocumented:
-    ('py:func', 'distutils.dist.DistributionMetadata.read_pkg_file'),
+    ('py:func', 'distutils.dist.DistributionMetadata.read_pkg_file'),  # undocumented
     ('py:func', 'distutils.file_util._copy_file_contents'),  # undocumented
     ('py:func', 'distutils.log.debug'),  # undocumented
     ('py:func', 'distutils.spawn.find_executable'),  # undocumented

mypy.ini

@@ -1,5 +1,42 @@
 [mypy]
-ignore_missing_imports = True
-# required to support namespace packages
-# https://github.com/python/mypy/issues/14057
+# CI should test for all versions, local development gets hints for oldest supported
+python_version = 3.8
+strict = False
+warn_unused_ignores = True
+# required to support namespace packages: https://github.com/python/mypy/issues/14057
 explicit_package_bases = True
+exclude = (?x)(
+	^build/
+	| ^.tox/
+	| ^pkg_resources/tests/data/my-test-package-source/setup.py$ # Duplicate module name
+	| ^.+?/(_vendor|extern)/ # Vendored
+	| ^setuptools/_distutils/ # Vendored
+	| ^setuptools/config/_validate_pyproject/ # Auto-generated
+	)
+disable_error_code = 
+	# TODO: Test environment is not yet properly configured to install all imported packages
+	# import-not-found, # This can be left commented out for local runs until we enforce running mypy in the CI
+	# TODO: Not all dependencies are typed. Namely: distutils._modified, wheel.wheelfile, and jaraco.*
+	import-untyped,
+	# Ignoring attr-defined because setuptools wraps a lot of distutils classes, adding new attributes,
+	# w/o updating all the attributes and return types from the base classes for type-checkers to understand
+	# Especially with setuptools.dist.command vs distutils.dist.command vs setuptools._distutils.dist.command
+	# *.extern modules that actually live in *._vendor will also cause attr-defined issues on import
+	attr-defined,
+
+# Avoid raising issues when importing from "extern" modules, as those are added to path dynamically.
+# https://github.com/pypa/setuptools/pull/3979#discussion_r1367968993
+[mypy-pkg_resources.extern.*,setuptools.extern.*]
+ignore_missing_imports = True
+
+[mypy-pkg_resources.tests.*,setuptools.tests.*]
+disable_error_code =
+	# Tests include creating dynamic modules that won't exists statically before the test is run.
+	# Let's ignore all "import-not-found", as if an import really wasn't found, then the test would fail.
+	import-not-found,
+	# mmany untyped "jaraco" modules
+	import-untyped,
+
+# Mypy issue, this vendored module is already excluded!
+[mypy-setuptools._vendor.packaging._manylinux]
+disable_error_code = import-not-found

pkg_resources/tests/test_pkg_resources.py

@@ -9,6 +9,7 @@
 import stat
 import distutils.dist
 import distutils.command.install_egg_info
+from typing import List
 
 from unittest import mock
 
@@ -32,7 +33,7 @@ def __call__(self):
 
 
 class TestZipProvider:
-    finalizers = []
+    finalizers: List[EggRemover] = []
 
     ref_time = datetime.datetime(2013, 5, 12, 13, 25, 0)
     "A reference time for a file modification"

setup.py

@@ -88,5 +88,6 @@ def _restore_install_lib(self):
 
 if __name__ == '__main__':
     # allow setup.py to run from another directory
-    here and os.chdir(here)
+    # TODO: Use a proper conditonal statement here
+    here and os.chdir(here)  # type: ignore[func-returns-value]
     dist = setuptools.setup(**setup_params)

setuptools/__init__.py

@@ -3,6 +3,7 @@
 import functools
 import os
 import re
+from typing import TYPE_CHECKING
 
 import _distutils_hack.override  # noqa: F401
 import distutils.core
@@ -105,8 +106,11 @@ def setup(**attrs):
 
 setup.__doc__ = distutils.core.setup.__doc__
 
-
-_Command = monkey.get_unpatched(distutils.core.Command)
+if TYPE_CHECKING:
+    # Work around a mypy issue where type[T] can't be used as a base: https://github.com/python/mypy/issues/10962
+    _Command = distutils.core.Command
+else:
+    _Command = monkey.get_unpatched(distutils.core.Command)
 
 
 class Command(_Command):
@@ -165,8 +169,9 @@ class Command(_Command):
     """
 
     command_consumes_arguments = False
+    distribution: Distribution  # override distutils.dist.Distribution with setuptools.dist.Distribution
 
-    def __init__(self, dist, **kw):
+    def __init__(self, dist: Distribution, **kw):
         """
         Construct the command for dist, updating
         vars(self) with any keyword parameters.

setuptools/command/_requirestxt.py

@@ -35,7 +35,7 @@ def _prepare(
 
 
 def _convert_extras_requirements(
-    extras_require: _StrOrIter,
+    extras_require: Mapping[str, _StrOrIter],
 ) -> Mapping[str, _Ordered[Requirement]]:
     """
     Convert requirements in `extras_require` of the form

setuptools/command/build_ext.py

@@ -26,7 +26,9 @@
 
 # make sure _config_vars is initialized
 get_config_var("LDSHARED")
-from distutils.sysconfig import _config_vars as _CONFIG_VARS  # noqa
+# Not publicly exposed in typeshed distutils stubs, but this is done on purpose
+# See https://github.com/pypa/setuptools/pull/4228#issuecomment-1959856400
+from distutils.sysconfig import _config_vars as _CONFIG_VARS  # type: ignore # noqa
 
 
 def _customize_compiler_for_shlib(compiler):
@@ -58,7 +60,7 @@ def _customize_compiler_for_shlib(compiler):
     use_stubs = True
 elif os.name != 'nt':
     try:
-        import dl
+        import dl  # type: ignore[import-not-found] # https://github.com/python/mypy/issues/13002
 
         use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')
     except ImportError:
@@ -378,7 +380,10 @@ def _compile_and_remove_stub(self, stub_file: str):
         optimize = self.get_finalized_command('install_lib').optimize
         if optimize > 0:
             byte_compile(
-                [stub_file], optimize=optimize, force=True, dry_run=self.dry_run
+                [stub_file],
+                optimize=optimize,
+                force=True,
+                dry_run=self.dry_run,
             )
         if os.path.exists(stub_file) and not self.dry_run:
             os.unlink(stub_file)

setuptools/command/dist_info.py

@@ -9,8 +9,10 @@
 from distutils import log
 from distutils.core import Command
 from pathlib import Path
+from typing import cast
 
 from .. import _normalization
+from .egg_info import egg_info as egg_info_cls
 
 
 class dist_info(Command):
@@ -50,7 +52,7 @@ def finalize_options(self):
         project_dir = dist.src_root or os.curdir
         self.output_dir = Path(self.output_dir or project_dir)
 
-        egg_info = self.reinitialize_command("egg_info")
+        egg_info = cast(egg_info_cls, self.reinitialize_command("egg_info"))
         egg_info.egg_base = str(self.output_dir)
 
         if self.tag_date:

setuptools/command/easy_install.py

@@ -25,6 +25,7 @@
 from distutils.command import install
 import sys
 import os
+from typing import Dict, List
 import zipimport
 import shutil
 import tempfile
@@ -43,7 +44,6 @@
 import configparser
 import sysconfig
 
-
 from sysconfig import get_path
 
 from setuptools import Command
@@ -1765,7 +1765,7 @@ def _wrap_lines(cls, lines):
 
 
 if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'raw') == 'rewrite':
-    PthDistributions = RewritePthDistributions
+    PthDistributions = RewritePthDistributions  # type: ignore[misc]  # Overwriting type
 
 
 def _first_line_re():
@@ -2015,7 +2015,7 @@ def is_python_script(script_text, filename):
     from os import chmod as _chmod
 except ImportError:
     # Jython compatibility
-    def _chmod(*args):
+    def _chmod(*args: object, **kwargs: object) -> None:  # type: ignore[misc] # Mypy re-uses the imported definition anyway
         pass
 
 
@@ -2033,8 +2033,8 @@ class CommandSpec(list):
     those passed to Popen.
     """
 
-    options = []
-    split_args = dict()
+    options: List[str] = []
+    split_args: Dict[str, bool] = dict()
 
     @classmethod
     def best(cls):

setuptools/command/editable_wheel.py

@@ -33,6 +33,7 @@
     Protocol,
     Tuple,
     TypeVar,
+    cast,
 )
 
 from .. import (
@@ -50,7 +51,12 @@
     SetuptoolsDeprecationWarning,
     SetuptoolsWarning,
 )
+from .build import build as build_cls
 from .build_py import build_py as build_py_cls
+from .dist_info import dist_info as dist_info_cls
+from .egg_info import egg_info as egg_info_cls
+from .install import install as install_cls
+from .install_scripts import install_scripts as install_scripts_cls
 
 if TYPE_CHECKING:
     from wheel.wheelfile import WheelFile  # noqa
@@ -155,7 +161,7 @@ def run(self):
 
     def _ensure_dist_info(self):
         if self.dist_info_dir is None:
-            dist_info = self.reinitialize_command("dist_info")
+            dist_info = cast(dist_info_cls, self.reinitialize_command("dist_info"))
             dist_info.output_dir = self.dist_dir
             dist_info.ensure_finalized()
             dist_info.run()
@@ -202,25 +208,33 @@ def _configure_build(
         scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts"))
 
         # egg-info may be generated again to create a manifest (used for package data)
-        egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True)
+        egg_info = cast(
+            egg_info_cls, dist.reinitialize_command("egg_info", reinit_subcommands=True)
+        )
         egg_info.egg_base = str(tmp_dir)
         egg_info.ignore_egg_info_in_manifest = True
 
-        build = dist.reinitialize_command("build", reinit_subcommands=True)
-        install = dist.reinitialize_command("install", reinit_subcommands=True)
+        build = cast(
+            build_cls, dist.reinitialize_command("build", reinit_subcommands=True)
+        )
+        install = cast(
+            install_cls, dist.reinitialize_command("install", reinit_subcommands=True)
+        )
 
         build.build_platlib = build.build_purelib = build.build_lib = build_lib
         install.install_purelib = install.install_platlib = install.install_lib = wheel
         install.install_scripts = build.build_scripts = scripts
         install.install_headers = headers
         install.install_data = data
 
-        install_scripts = dist.get_command_obj("install_scripts")
+        install_scripts = cast(
+            install_scripts_cls, dist.get_command_obj("install_scripts")
+        )
         install_scripts.no_ep = True
 
         build.build_temp = str(tmp_dir)
 
-        build_py = dist.get_command_obj("build_py")
+        build_py = cast(build_py_cls, dist.get_command_obj("build_py"))
         build_py.compile = False
         build_py.existing_egg_info_dir = self._find_egg_info_dir()
 
@@ -233,6 +247,7 @@ def _set_editable_mode(self):
         """Set the ``editable_mode`` flag in the build sub-commands"""
         dist = self.distribution
         build = dist.get_command_obj("build")
+        # TODO: Update typeshed distutils stubs to overload non-None return type by default
         for cmd_name in build.get_sub_commands():
             cmd = dist.get_command_obj(cmd_name)
             if hasattr(cmd, "editable_mode"):
@@ -269,7 +284,7 @@ def _run_build_commands(
         self._run_install("data")
         return files, mapping
 
-    def _run_build_subcommands(self):
+    def _run_build_subcommands(self) -> None:
         """
         Issue #3501 indicates that some plugins/customizations might rely on:
 
@@ -283,7 +298,7 @@ def _run_build_subcommands(self):
         # TODO: Once plugins/customisations had the chance to catch up, replace
         #       `self._run_build_subcommands()` with `self.run_command("build")`.
         #       Also remove _safely_run, TestCustomBuildPy. Suggested date: Aug/2023.
-        build: Command = self.get_finalized_command("build")
+        build = self.get_finalized_command("build")
         for name in build.get_sub_commands():
             cmd = self.get_finalized_command(name)
             if name == "build_py" and type(cmd) != build_py_cls:
@@ -432,7 +447,8 @@ def __init__(
     ):
         self.auxiliary_dir = Path(auxiliary_dir)
         self.build_lib = Path(build_lib).resolve()
-        self._file = dist.get_command_obj("build_py").copy_file
+        # TODO: Update typeshed distutils stubs to overload non-None return type by default
+        self._file = dist.get_command_obj("build_py").copy_file  # type: ignore[union-attr]
         super().__init__(dist, name, [self.auxiliary_dir])
 
     def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]):
@@ -450,7 +466,9 @@ def _create_file(self, relative_output: str, src_file: str, link=None):
         dest = self.auxiliary_dir / relative_output
         if not dest.parent.is_dir():
             dest.parent.mkdir(parents=True)
-        self._file(src_file, dest, link=link)
+        # TODO: Update typeshed distutils stubs so distutils.cmd.Command.copy_file, accepts PathLike
+        # same with methods used by copy_file
+        self._file(src_file, dest, link=link)  # type: ignore[arg-type]
 
     def _create_links(self, outputs, output_mapping):
         self.auxiliary_dir.mkdir(parents=True, exist_ok=True)
@@ -603,7 +621,8 @@ def _simple_layout(
     layout = {pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages}
     if not layout:
         return set(package_dir) in ({}, {""})
-    parent = os.path.commonpath(starmap(_parent_path, layout.items()))
+    # TODO: has been fixed upstream, waiting for new mypy release https://github.com/python/typeshed/pull/11310
+    parent = os.path.commonpath(starmap(_parent_path, layout.items()))  # type: ignore[call-overload]
     return all(
         _path.same_path(Path(parent, *key.split('.')), value)
         for key, value in layout.items()

setuptools/command/install.py

@@ -3,9 +3,11 @@
 import glob
 import platform
 import distutils.command.install as orig
+from typing import cast
 
 import setuptools
 from ..warnings import SetuptoolsDeprecationWarning, SetuptoolsWarning
+from .bdist_egg import bdist_egg as bdist_egg_cls
 
 # Prior to numpy 1.9, NumPy relies on the '_install' name, so provide it for
 # now. See https://github.com/pypa/setuptools/issues/199/
@@ -135,7 +137,8 @@ def do_egg_install(self):
         cmd.package_index.scan(glob.glob('*.egg'))
 
         self.run_command('bdist_egg')
-        args = [self.distribution.get_command_obj('bdist_egg').egg_output]
+        bdist_egg = cast(bdist_egg_cls, self.distribution.get_command_obj('bdist_egg'))
+        args = [bdist_egg.egg_output]
 
         if setuptools.bootstrap_install_from:
             # Bootstrap self-installation of setuptools