From 2239e87e3c5a830177bf5ec451b37ecb88e903d8 Mon Sep 17 00:00:00 2001 From: The Pyink Maintainers Date: Fri, 30 Aug 2024 05:17:42 -0700 Subject: [PATCH] Rebase Pyink to Black v24.8.0. PiperOrigin-RevId: 669289883 --- patches/pyink.patch | 168 +++-------- pyproject.toml | 5 +- src/pyink/__init__.py | 37 +-- src/pyink/comments.py | 24 +- src/pyink/files.py | 5 + src/pyink/handle_ipynb_magics.py | 4 +- src/pyink/ink.py | 69 +++-- src/pyink/linegen.py | 97 ++++-- src/pyink/lines.py | 13 +- src/pyink/mode.py | 26 ++ src/pyink/nodes.py | 92 ++++-- src/pyink/parsing.py | 6 +- src/pyink/strings.py | 71 ++++- tests/data/cases/backslash_before_indent.py | 24 ++ .../cases/comment_after_escaped_newline.py | 4 +- tests/data/cases/dummy_implementations.py | 105 +++++++ tests/data/cases/fmtonoff6.py | 13 + tests/data/cases/form_feeds.py | 1 + tests/data/cases/pattern_matching_complex.py | 2 +- .../cases/pattern_matching_with_if_stmt.py | 72 +++++ tests/data/cases/pep_701.py | 276 ++++++++++++++++++ tests/data/cases/preview_multiline_strings.py | 14 + tests/data/cases/type_param_defaults.py | 61 ++++ .../.gitignore | 3 + .../ignore_directory_gitignore_tests/abc.py | 0 .../large_ignored_dir_two/a.py | 0 .../large_ignored_dir_two/inner/b.py | 0 .../large_ignored_dir_two/inner2/c.py | 0 .../large_ignored_dir_two/inner3/d.py | 0 .../ignore_directory_gitignore_tests/z.py | 0 tests/data/miscellaneous/debug_visitor.out | 154 +++++++++- tests/data/pyink_configs/majority_quotes.py | 2 + tests/optional.py | 8 +- tests/test_black.py | 101 +++++-- tests/test_tokenize.py | 120 ++++++++ tests/util.py | 4 +- tox.ini | 2 +- 37 files changed, 1302 insertions(+), 281 deletions(-) create mode 100644 tests/data/cases/backslash_before_indent.py create mode 100644 tests/data/cases/fmtonoff6.py create mode 100644 tests/data/cases/pattern_matching_with_if_stmt.py create mode 100644 tests/data/cases/pep_701.py create mode 100644 tests/data/cases/type_param_defaults.py create mode 100644 tests/data/ignore_directory_gitignore_tests/.gitignore create mode 100644 tests/data/ignore_directory_gitignore_tests/abc.py create mode 100644 tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/a.py create mode 100644 tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner/b.py create mode 100644 tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner2/c.py create mode 100644 tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner3/d.py create mode 100644 tests/data/ignore_directory_gitignore_tests/z.py create mode 100644 tests/test_tokenize.py diff --git a/patches/pyink.patch b/patches/pyink.patch index bb735c85722..3b546e6b705 100644 --- a/patches/pyink.patch +++ b/patches/pyink.patch @@ -49,18 +49,15 @@ from pyink.mode import Mode as Mode # re-exported -from pyink.mode import Preview, TargetVersion, supports_feature +from pyink.mode import Preview, QuoteStyle, TargetVersion, supports_feature - from pyink.nodes import ( - STARS, - is_number_token, -@@ -90,12 +91,11 @@ from pyink.ranges import ( + from pyink.nodes import STARS, is_number_token, is_simple_decorator_expression, syms + from pyink.output import color_diff, diff, dump_to_file, err, ipynb_diff, out + from pyink.parsing import ( # noqa F401 +@@ -90,9 +91,8 @@ from pyink.ranges import ( parse_line_ranges, sanitized_lines, ) +from pyink import ink from pyink.report import Changed, NothingChanged, Report - from pyink.trans import iter_fexpr_spans - - from google3.devtools.python.pyformat import import_sorting -from blib2to3.pgen2 import token -from blib2to3.pytree import Leaf, Node @@ -471,17 +468,6 @@ yield complete_line def visit_default(self, node: LN) -> Iterator[Line]: -@@ -156,7 +182,9 @@ class LineGenerator(Visitor[Line]): - node.prefix = "" - if self.mode.string_normalization and node.type == token.STRING: - node.value = normalize_string_prefix(node.value) -- node.value = normalize_string_quotes(node.value) -+ node.value = normalize_string_quotes( -+ node.value, preferred_quote=self.mode.preferred_quote -+ ) - if node.type == token.NUMBER: - normalize_numeric_literal(node) - if node.type not in WHITESPACE: @@ -166,26 +194,27 @@ class LineGenerator(Visitor[Line]): def visit_test(self, node: Node) -> Iterator[Line]: """Visit an `x if y else z` test""" @@ -562,15 +548,15 @@ + yield from self.line(_DEDENT) else: -- if not node.parent or not is_stub_suite(node.parent): +- if node.parent and is_stub_suite(node.parent): + if ( -+ not (self.mode.is_pyi or not self.mode.is_pyink) -+ or not node.parent -+ or not is_stub_suite(node.parent, self.mode) ++ (self.mode.is_pyi or not self.mode.is_pyink) ++ and node.parent ++ and is_stub_suite(node.parent, self.mode) + ): - yield from self.line() - yield from self.visit_default(node) - + node.prefix = "" + yield from self.visit_default(node) + return @@ -414,7 +453,10 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) @@ -584,9 +570,9 @@ if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): @@ -428,7 +470,9 @@ class LineGenerator(Visitor[Line]): - # formatting as visit_default() is called *after*. To avoid a - # situation where this function formats a docstring differently on - # the second pass, normalize it early. + # see padding logic below), there's a possibility for unstable + # formatting. To avoid a situation where this function formats a + # docstring differently on the second pass, normalize it early. - docstring = normalize_string_quotes(docstring) + docstring = normalize_string_quotes( + docstring, preferred_quote=self.mode.preferred_quote @@ -628,6 +614,17 @@ self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) self.visit_async_funcdef = self.visit_async_stmt self.visit_decorated = self.visit_decorators +@@ -499,7 +556,9 @@ def visit_STRING( + + if self.mode.string_normalization and leaf.type == token.STRING: + leaf.value = normalize_string_prefix(leaf.value) +- leaf.value = normalize_string_quotes(leaf.value) ++ leaf.value = normalize_string_quotes( ++ leaf.value, preferred_quote=self.mode.preferred_quote ++ ) + yield from self.visit_default(leaf) + + def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: @@ -577,10 +628,19 @@ def transform_line( ll = mode.line_length @@ -1012,70 +1009,6 @@ + return Quote.DOUBLE --- a/nodes.py +++ b/nodes.py -@@ -532,8 +532,8 @@ def first_leaf_of(node: LN) -> Optional[ - - - def is_arith_like(node: LN) -> bool: -- """Whether node is an arithmetic or a binary arithmetic expression""" -- return node.type in { -+ """Whether node is an arithmetic or a binary arithmetic expression""" -+ return node.type in { - syms.arith_expr, - syms.shift_expr, - syms.xor_expr, -@@ -542,14 +542,14 @@ def is_arith_like(node: LN) -> bool: - - - def is_docstring(leaf: Leaf, mode: Mode) -> bool: -- if leaf.type != token.STRING: -- return False -+ if leaf.type != token.STRING: -+ return False - -- prefix = get_string_prefix(leaf.value) -- if set(prefix).intersection("bBfF"): -- return False -+ prefix = get_string_prefix(leaf.value) -+ if set(prefix).intersection("bBfF"): -+ return False - -- if ( -+ if ( - Preview.unify_docstring_detection in mode - and leaf.parent - and leaf.parent.type == syms.simple_stmt -@@ -557,20 +557,22 @@ def is_docstring(leaf: Leaf, mode: Mode) - and leaf.parent.parent - and leaf.parent.parent.type == syms.file_input - ): -- return True -+ return True - -- if prev_siblings_are( -+ if prev_siblings_are( - leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] - ): -- return True -+ return True - -- # Multiline docstring on the same line as the `def`. -- if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]): -- # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python -- # grammar. We're safe to return True without further checks. -- return True -+ # Multiline docstring on the same line as the `def`. -+ if prev_siblings_are( -+ leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt] -+ ): -+ # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python -+ # grammar. We're safe to return True without further checks. -+ return True - -- return False -+ return False - - - def is_empty_tuple(node: LN) -> bool: @@ -763,9 +765,13 @@ def is_function_or_class(node: Node) -> return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} @@ -1094,7 +1027,7 @@ # If there is a comment, we want to keep it. --- a/pyproject.toml +++ b/pyproject.toml -@@ -1,51 +1,23 @@ +@@ -1,52 +1,23 @@ -# Example configuration for Black. - -# NOTE: you have to use single-quoted strings in TOML for regular expressions. @@ -1112,9 +1045,10 @@ -extend-exclude = ''' -/( - # The following are specific to Black, you probably don't want those. -- tests/data -- | profiling --)/ +- tests/data/ +- | profiling/ +- | scripts/generate_schema.py # Uses match syntax +-) -''' -# We use the unstable style for formatting Black itself. If you -# want bug-free formatting, you should keep this off. If you want @@ -1155,11 +1089,11 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", -@@ -70,51 +42,34 @@ dependencies = [ +@@ -70,53 +42,35 @@ dependencies = [ "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", -+ "black==24.3.0", ++ "black==24.8.0", ] -dynamic = ["readme", "version"] +dynamic = ["version"] @@ -1185,8 +1119,10 @@ +pyink = "pyink:patched_main" [project.urls] +-Documentation = "https://black.readthedocs.io/" -Changelog = "https://github.com/psf/black/blob/main/CHANGES.md" --Homepage = "https://github.com/psf/black" +-Repository = "https://github.com/psf/black" +-Issues = "https://github.com/psf/black/issues" - -[tool.hatch.metadata.hooks.fancy-pypi-readme] -content-type = "text/markdown" @@ -1195,7 +1131,8 @@ - { path = "CHANGES.md" }, -] +Changelog = "https://github.com/google/pyink/blob/pyink/CHANGES.md" -+Homepage = "https://github.com/google/pyink" ++Repository = "https://github.com/google/pyink" ++Issues = "https://github.com/google/pyink/issues" [tool.hatch.version] source = "vcs" @@ -1261,7 +1198,7 @@ --- a/strings.py +++ b/strings.py @@ -8,6 +8,7 @@ from functools import lru_cache - from typing import Final, List, Match, Pattern + from typing import Final, List, Match, Pattern, Tuple from pyink._width_table import WIDTH_TABLE +from pyink.mode import Quote @@ -1279,8 +1216,8 @@ + + For three quotes strings, always use double-quote. - Adds or removes backslashes as appropriate. Doesn't parse and fix - strings nested in f-strings. + Adds or removes backslashes as appropriate. + """ @@ -234,8 +237,8 @@ def normalize_string_quotes(s: str) -> s if new_escape_count > orig_escape_count: return s # Do not introduce more escaping @@ -1302,27 +1239,6 @@ +pyink = false --- a/tests/test_black.py +++ b/tests/test_black.py -@@ -1666,9 +1666,9 @@ class BlackTestCase(BlackBaseTestCase): - src_dir.mkdir() - - root_pyproject = root / "pyproject.toml" -- root_pyproject.write_text("[tool.black]", encoding="utf-8") -+ root_pyproject.write_text("[tool.pyink]", encoding="utf-8") - src_pyproject = src_dir / "pyproject.toml" -- src_pyproject.write_text("[tool.black]", encoding="utf-8") -+ src_pyproject.write_text("[tool.pyink]", encoding="utf-8") - src_python = src_dir / "foo.py" - src_python.touch() - -@@ -1699,7 +1699,7 @@ class BlackTestCase(BlackBaseTestCase): - - src_sub_python = src_sub / "bar.py" - -- # we skip src_sub_pyproject since it is missing the [tool.black] section -+ # we skip src_sub_pyproject since it is missing the [tool.pyink] section - self.assertEqual( - pyink.find_project_root((src_sub_python,)), - (src_dir.resolve(), "pyproject.toml"), @@ -2772,6 +2772,82 @@ class TestFileCollection: stdin_filename=stdin_filename, ) @@ -1432,7 +1348,7 @@ skip_install = True commands = pip install -e . -- black --check {toxinidir}/src {toxinidir}/tests +- black --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts - -[testenv:generate_schema] -setenv = PYTHONWARNDEFAULTENCODING = @@ -1441,7 +1357,7 @@ -commands = - pip install -e . - python {toxinidir}/scripts/generate_schema.py --outfile {toxinidir}/src/black/resources/black.schema.json -+ pyink --check {toxinidir}/src {toxinidir}/tests ++ pyink --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts --- a/trans.py +++ b/trans.py @@ -28,8 +28,8 @@ from typing import ( diff --git a/pyproject.toml b/pyproject.toml index 9d21a751aa3..c60626a284d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", - "black==24.3.0", + "black==24.8.0", ] dynamic = ["version"] @@ -59,7 +59,8 @@ pyink = "pyink:patched_main" [project.urls] Changelog = "https://github.com/google/pyink/blob/pyink/CHANGES.md" -Homepage = "https://github.com/google/pyink" +Repository = "https://github.com/google/pyink" +Issues = "https://github.com/google/pyink/issues" [tool.hatch.version] source = "vcs" diff --git a/src/pyink/__init__.py b/src/pyink/__init__.py index 030bccb59a9..2f0228c5883 100644 --- a/src/pyink/__init__.py +++ b/src/pyink/__init__.py @@ -70,13 +70,7 @@ from pyink.mode import FUTURE_FLAG_TO_FEATURE, Feature, VERSION_TO_FEATURES from pyink.mode import Mode as Mode # re-exported from pyink.mode import Preview, QuoteStyle, TargetVersion, supports_feature -from pyink.nodes import ( - STARS, - is_number_token, - is_simple_decorator_expression, - is_string_token, - syms, -) +from pyink.nodes import STARS, is_number_token, is_simple_decorator_expression, syms from pyink.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from pyink.parsing import ( # noqa F401 ASTSafetyError, @@ -93,7 +87,6 @@ ) from pyink import ink from pyink.report import Changed, NothingChanged, Report -from pyink.trans import iter_fexpr_spans COMPILED = Path(__file__).suffix in (".pyd", ".so") @@ -1308,7 +1301,10 @@ def _format_str_once( elt = EmptyLineTracker(mode=mode) split_line_features = { feature - for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF} + for feature in { + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + } if supports_feature(versions, feature) } block: Optional[LinesBlock] = None @@ -1380,15 +1376,14 @@ def get_features_used( # noqa: C901 } for n in node.pre_order(): - if is_string_token(n): - value_head = n.value[:2] - if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}: - features.add(Feature.F_STRINGS) - if Feature.DEBUG_F_STRINGS not in features: - for span_beg, span_end in iter_fexpr_spans(n.value): - if n.value[span_beg : span_end - 1].rstrip().endswith("="): - features.add(Feature.DEBUG_F_STRINGS) - break + if n.type == token.FSTRING_START: + features.add(Feature.F_STRINGS) + elif ( + n.type == token.RBRACE + and n.parent is not None + and any(child.type == token.EQUAL for child in n.parent.children) + ): + features.add(Feature.DEBUG_F_STRINGS) elif is_number_token(n): if "_" in n.value: @@ -1484,6 +1479,12 @@ def get_features_used( # noqa: C901 elif n.type in (syms.type_stmt, syms.typeparams): features.add(Feature.TYPE_PARAMS) + elif ( + n.type in (syms.typevartuple, syms.paramspec, syms.typevar) + and n.children[-2].type == token.EQUAL + ): + features.add(Feature.TYPE_PARAM_DEFAULTS) + return features diff --git a/src/pyink/comments.py b/src/pyink/comments.py index 2df7a897f9c..046a5623c89 100644 --- a/src/pyink/comments.py +++ b/src/pyink/comments.py @@ -184,24 +184,24 @@ def convert_one_fmt_off_pair( for leaf in node.leaves(): previous_consumed = 0 for comment in list_comments(leaf.prefix, is_endmarker=False): - should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment( - comment.value, mode - ) - if not should_pass_fmt: + is_fmt_off = comment.value in FMT_OFF + is_fmt_skip = _contains_fmt_skip_comment(comment.value, mode) + if (not is_fmt_off and not is_fmt_skip) or ( + # Invalid use when `# fmt: off` is applied before a closing bracket. + is_fmt_off + and leaf.type in CLOSING_BRACKETS + ): previous_consumed = comment.consumed continue # We only want standalone comments. If there's no previous leaf or # the previous leaf is indentation, it's a standalone comment in # disguise. - if should_pass_fmt and comment.type != STANDALONE_COMMENT: + if comment.type != STANDALONE_COMMENT: prev = preceding_leaf(leaf) if prev: - if comment.value in FMT_OFF and prev.type not in WHITESPACE: + if is_fmt_off and prev.type not in WHITESPACE: continue - if ( - _contains_fmt_skip_comment(comment.value, mode) - and prev.type in WHITESPACE - ): + if is_fmt_skip and prev.type in WHITESPACE: continue ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode)) @@ -213,7 +213,7 @@ def convert_one_fmt_off_pair( prefix = first.prefix if comment.value in FMT_OFF: first.prefix = prefix[comment.consumed :] - if _contains_fmt_skip_comment(comment.value, mode): + if is_fmt_skip: first.prefix = "" standalone_comment_prefix = prefix else: @@ -233,7 +233,7 @@ def convert_one_fmt_off_pair( fmt_off_prefix = fmt_off_prefix.split("\n")[-1] standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value - if _contains_fmt_skip_comment(comment.value, mode): + if is_fmt_skip: hidden_value += ( comment.leading_whitespace if Preview.no_normalize_fmt_skip_whitespace in mode diff --git a/src/pyink/files.py b/src/pyink/files.py index b803d4d7bec..df7c2792c4a 100644 --- a/src/pyink/files.py +++ b/src/pyink/files.py @@ -59,6 +59,9 @@ def find_project_root( ) -> Tuple[Path, str]: """Return a directory containing .git, .hg, or pyproject.toml. + pyproject.toml files are only considered if they contain a [tool.pyink] + section and are ignored otherwise. + That directory will be a common parent of all files and directories passed in `srcs`. @@ -309,6 +312,8 @@ def _path_is_ignored( for gitignore_path, pattern in gitignore_dict.items(): try: relative_path = path.relative_to(gitignore_path).as_posix() + if path.is_dir(): + relative_path = relative_path + "/" except ValueError: break if pattern.match_file(relative_path): diff --git a/src/pyink/handle_ipynb_magics.py b/src/pyink/handle_ipynb_magics.py index 552e70b1bb9..ae8c1c4f20a 100644 --- a/src/pyink/handle_ipynb_magics.py +++ b/src/pyink/handle_ipynb_magics.py @@ -294,8 +294,8 @@ def _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]: def _get_str_args(args: List[ast.expr]) -> List[str]: str_args = [] for arg in args: - assert isinstance(arg, ast.Str) - str_args.append(arg.s) + assert isinstance(arg, ast.Constant) and isinstance(arg.value, str) + str_args.append(arg.value) return str_args diff --git a/src/pyink/ink.py b/src/pyink/ink.py index 1e27fcb55ed..2604c320d5f 100644 --- a/src/pyink/ink.py +++ b/src/pyink/ink.py @@ -3,46 +3,77 @@ This is a separate module for easier patch management. """ -import sys from typing import ( Collection, + Iterator, List, Optional, Sequence, Set, Tuple, Union, - Iterator, ) -from blib2to3.pgen2.token import ASYNC, NEWLINE, STRING +from blib2to3.pgen2.token import ASYNC, FSTRING_START, NEWLINE, STRING from blib2to3.pytree import type_repr from pyink.mode import Quote -from pyink.nodes import LN, Leaf, Node, STANDALONE_COMMENT, syms, Visitor +from pyink.nodes import LN, Leaf, Node, STANDALONE_COMMENT, Visitor, syms from pyink.strings import STRING_PREFIX_CHARS -def majority_quote(node: Node) -> Quote: - """Returns the majority quote from node. +def majority_quote(node: LN) -> Quote: + """Returns the majority quote from the node. Triple quotes strings are excluded from calculation. If even, returns double quote. + + Args: + node: A graph node of Python code split by operations. + + Returns: + The majority quote of the node. """ - num_double_quotes = 0 - num_single_quotes = 0 - for leaf in node.leaves(): - if leaf.type == STRING: - value = leaf.value.lstrip(STRING_PREFIX_CHARS) - if value.startswith(("'''", '"""')): - continue - if value.startswith('"'): - num_double_quotes += 1 - else: - num_single_quotes += 1 + print([leaf.value for leaf in node.leaves()]) + print(repr(node)) + num_single_quotes, num_double_quotes = _count_quotes(node) if num_single_quotes > num_double_quotes: return Quote.SINGLE - else: - return Quote.DOUBLE + return Quote.DOUBLE + + +def _count_quotes(node: LN) -> Tuple[int, int]: + """Recursively counts quotes. + + Returns the number of single quotes and the number of double quotes used + inside the node. Quotes inside an f-string are not counted. + + Args: + node: A graph node of Python code split by operations. + + Returns: + The majority quote of the node. + """ + if isinstance(node, Leaf) and ( + node.type == STRING or node.type == FSTRING_START + ): + value = node.value.lstrip(STRING_PREFIX_CHARS) + if value.startswith(("'''", '"""')): + return (0, 0) + if value.startswith('"'): + return (0, 1) + return (1, 0) + + # Quotes of potential strings nested inside an f-string are not counted. + if type_repr(node.type) == "fstring": + return _count_quotes(node.children[0]) + + num_single_quotes = 0 + num_double_quotes = 0 + for child in node.children: + num_child_single_quotes, num_child_double_quotes = _count_quotes(child) + num_single_quotes += num_child_single_quotes + num_double_quotes += num_child_double_quotes + return num_single_quotes, num_double_quotes def convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]): diff --git a/src/pyink/linegen.py b/src/pyink/linegen.py index 32ce884d6b1..6deab0c302d 100644 --- a/src/pyink/linegen.py +++ b/src/pyink/linegen.py @@ -43,6 +43,7 @@ WHITESPACE, Visitor, ensure_visible, + fstring_to_string, get_annotation_type, is_arith_like, is_async_stmt_or_funcdef, @@ -178,13 +179,6 @@ def visit_default(self, node: LN) -> Iterator[Line]: if any_open_brackets: node.prefix = "" - if self.mode.string_normalization and node.type == token.STRING: - node.value = normalize_string_prefix(node.value) - node.value = normalize_string_quotes( - node.value, preferred_quote=self.mode.preferred_quote - ) - if node.type == token.NUMBER: - normalize_numeric_literal(node) if node.type not in WHITESPACE: self.current_line.append(node) yield from super().visit_default(node) @@ -349,11 +343,14 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: else: if ( - not (self.mode.is_pyi or not self.mode.is_pyink) - or not node.parent - or not is_stub_suite(node.parent, self.mode) + (self.mode.is_pyi or not self.mode.is_pyink) + and node.parent + and is_stub_suite(node.parent, self.mode) ): - yield from self.line() + node.prefix = "" + yield from self.visit_default(node) + return + yield from self.line() yield from self.visit_default(node) def visit_async_stmt(self, node: Node) -> Iterator[Line]: @@ -462,12 +459,11 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: # indentation of those changes the AST representation of the code. if self.mode.string_normalization: docstring = normalize_string_prefix(leaf.value) - # visit_default() does handle string normalization for us, but - # since this method acts differently depending on quote style (ex. + # We handle string normalization at the end of this method, but since + # what we do right now acts differently depending on quote style (ex. # see padding logic below), there's a possibility for unstable - # formatting as visit_default() is called *after*. To avoid a - # situation where this function formats a docstring differently on - # the second pass, normalize it early. + # formatting. To avoid a situation where this function formats a + # docstring differently on the second pass, normalize it early. docstring = normalize_string_quotes( docstring, preferred_quote=self.mode.preferred_quote ) @@ -549,8 +545,65 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: else: leaf.value = prefix + quote + docstring + quote + if self.mode.string_normalization and leaf.type == token.STRING: + leaf.value = normalize_string_prefix(leaf.value) + leaf.value = normalize_string_quotes( + leaf.value, preferred_quote=self.mode.preferred_quote + ) yield from self.visit_default(leaf) + def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: + normalize_numeric_literal(leaf) + yield from self.visit_default(leaf) + + def visit_fstring(self, node: Node) -> Iterator[Line]: + # currently we don't want to format and split f-strings at all. + string_leaf = fstring_to_string(node) + node.replace(string_leaf) + if "\\" in string_leaf.value and any( + "\\" in str(child) + for child in node.children + if child.type == syms.fstring_replacement_field + ): + # string normalization doesn't account for nested quotes, + # causing breakages. skip normalization when nested quotes exist + yield from self.visit_default(string_leaf) + return + yield from self.visit_STRING(string_leaf) + + # TODO: Uncomment Implementation to format f-string children + # fstring_start = node.children[0] + # fstring_end = node.children[-1] + # assert isinstance(fstring_start, Leaf) + # assert isinstance(fstring_end, Leaf) + + # quote_char = fstring_end.value[0] + # quote_idx = fstring_start.value.index(quote_char) + # prefix, quote = ( + # fstring_start.value[:quote_idx], + # fstring_start.value[quote_idx:] + # ) + + # if not is_docstring(node, self.mode): + # prefix = normalize_string_prefix(prefix) + + # assert quote == fstring_end.value + + # is_raw_fstring = "r" in prefix or "R" in prefix + # middles = [ + # leaf + # for leaf in node.leaves() + # if leaf.type == token.FSTRING_MIDDLE + # ] + + # if self.mode.string_normalization: + # middles, quote = normalize_fstring_quotes(quote, middles, is_raw_fstring) + + # fstring_start.value = prefix + quote + # fstring_end.value = quote + + # yield from self.visit_default(node) + def __post_init__(self) -> None: """You are in a twisty little maze of passages.""" self.current_line = Line(mode=self.mode) @@ -1369,6 +1422,16 @@ def normalize_invisible_parens( # noqa: C901 child, parens_after={"case"}, mode=mode, features=features ) + # Add parentheses around if guards in case blocks + if ( + isinstance(child, Node) + and child.type == syms.guard + and Preview.parens_for_long_if_clauses_in_case_block in mode + ): + normalize_invisible_parens( + child, parens_after={"if"}, mode=mode, features=features + ) + # Add parentheses around long tuple unpacking in assignments. if ( index == 0 @@ -1424,7 +1487,7 @@ def normalize_invisible_parens( # noqa: C901 # of case will be not parsed as a Python keyword. break - elif not (isinstance(child, Leaf) and is_multiline_string(child)): + elif not is_multiline_string(child): wrap_in_parentheses(node, child, visible=False) comma_check = child.type == token.COMMA diff --git a/src/pyink/lines.py b/src/pyink/lines.py index 55a951f40dd..37c8afa8438 100644 --- a/src/pyink/lines.py +++ b/src/pyink/lines.py @@ -92,7 +92,12 @@ def append( Inline comments are put aside. """ - has_value = leaf.type in BRACKETS or bool(leaf.value.strip()) + has_value = ( + leaf.type in BRACKETS + # empty fstring-middles must not be truncated + or leaf.type == token.FSTRING_MIDDLE + or bool(leaf.value.strip()) + ) if not has_value: return @@ -926,11 +931,13 @@ def is_line_short_enough( # noqa: C901 return False if leaf.bracket_depth <= max_level_to_update and leaf.type == token.COMMA: - # Ignore non-nested trailing comma + # Inside brackets, ignore trailing comma # directly after MLS/MLS-containing expression ignore_ctxs: List[Optional[LN]] = [None] ignore_ctxs += multiline_string_contexts - if not (leaf.prev_sibling in ignore_ctxs and i == len(line.leaves) - 1): + if (line.inside_brackets or leaf.bracket_depth > 0) and ( + i != len(line.leaves) - 1 or leaf.prev_sibling not in ignore_ctxs + ): commas[leaf.bracket_depth] += 1 if max_level_to_update != math.inf: max_level_to_update = min(max_level_to_update, leaf.bracket_depth) diff --git a/src/pyink/mode.py b/src/pyink/mode.py index dbef47f73d2..5a667b6f404 100644 --- a/src/pyink/mode.py +++ b/src/pyink/mode.py @@ -24,6 +24,7 @@ class TargetVersion(Enum): PY310 = 10 PY311 = 11 PY312 = 12 + PY313 = 13 class Feature(Enum): @@ -46,6 +47,8 @@ class Feature(Enum): DEBUG_F_STRINGS = 16 PARENTHESIZED_CONTEXT_MANAGERS = 17 TYPE_PARAMS = 18 + FSTRING_PARSING = 19 + TYPE_PARAM_DEFAULTS = 20 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -156,6 +159,28 @@ class Feature(Enum): Feature.EXCEPT_STAR, Feature.VARIADIC_GENERICS, Feature.TYPE_PARAMS, + Feature.FSTRING_PARSING, + }, + TargetVersion.PY313: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + Feature.FSTRING_PARSING, + Feature.TYPE_PARAM_DEFAULTS, }, } @@ -180,6 +205,7 @@ class Preview(Enum): is_simple_lookup_for_doublestar_expression = auto() docstring_check_for_newline = auto() remove_redundant_guard_parens = auto() + parens_for_long_if_clauses_in_case_block = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/pyink/nodes.py b/src/pyink/nodes.py index f17297dbfdb..35389380d5b 100644 --- a/src/pyink/nodes.py +++ b/src/pyink/nodes.py @@ -145,7 +145,13 @@ OPENING_BRACKETS: Final = set(BRACKET.keys()) CLOSING_BRACKETS: Final = set(BRACKET.values()) BRACKETS: Final = OPENING_BRACKETS | CLOSING_BRACKETS -ALWAYS_NO_SPACE: Final = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT} +ALWAYS_NO_SPACE: Final = CLOSING_BRACKETS | { + token.COMMA, + STANDALONE_COMMENT, + token.FSTRING_MIDDLE, + token.FSTRING_END, + token.BANG, +} RARROW = 55 @@ -211,6 +217,9 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no }: return NO + if t == token.LBRACE and p.type == syms.fstring_replacement_field: + return NO + prev = leaf.prev_sibling if not prev: prevp = preceding_leaf(p) @@ -272,6 +281,9 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no elif prev.type in OPENING_BRACKETS: return NO + elif prev.type == token.BANG: + return NO + if p.type in {syms.parameters, syms.arglist}: # untyped function signatures or calls if not prev or prev.type != token.COMMA: @@ -393,6 +405,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument: return NO + # TODO: add fstring here? elif t in {token.NAME, token.NUMBER, token.STRING}: return NO @@ -533,8 +546,8 @@ def first_leaf_of(node: LN) -> Optional[Leaf]: def is_arith_like(node: LN) -> bool: - """Whether node is an arithmetic or a binary arithmetic expression""" - return node.type in { + """Whether node is an arithmetic or a binary arithmetic expression""" + return node.type in { syms.arith_expr, syms.shift_expr, syms.xor_expr, @@ -542,38 +555,37 @@ def is_arith_like(node: LN) -> bool: } -def is_docstring(leaf: Leaf, mode: Mode) -> bool: - if leaf.type != token.STRING: - return False +def is_docstring(node: NL, mode: Mode) -> bool: + if isinstance(node, Leaf): + if node.type != token.STRING: + return False - prefix = get_string_prefix(leaf.value) - if set(prefix).intersection("bBfF"): - return False + prefix = get_string_prefix(node.value) + if set(prefix).intersection("bBfF"): + return False - if ( + if ( Preview.unify_docstring_detection in mode - and leaf.parent - and leaf.parent.type == syms.simple_stmt - and not leaf.parent.prev_sibling - and leaf.parent.parent - and leaf.parent.parent.type == syms.file_input + and node.parent + and node.parent.type == syms.simple_stmt + and not node.parent.prev_sibling + and node.parent.parent + and node.parent.parent.type == syms.file_input ): - return True + return True - if prev_siblings_are( - leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] + if prev_siblings_are( + node.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): - return True + return True - # Multiline docstring on the same line as the `def`. - if prev_siblings_are( - leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt] - ): - # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python - # grammar. We're safe to return True without further checks. - return True + # Multiline docstring on the same line as the `def`. + if prev_siblings_are(node.parent, [syms.parameters, token.COLON, syms.simple_stmt]): + # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python + # grammar. We're safe to return True without further checks. + return True - return False + return False def is_empty_tuple(node: LN) -> bool: @@ -750,8 +762,28 @@ def is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool: return p.type in within -def is_multiline_string(leaf: Leaf) -> bool: +def is_fstring(node: Node) -> bool: + """Return True if the node is an f-string""" + return node.type == syms.fstring + + +def fstring_to_string(node: Node) -> Leaf: + """Converts an fstring node back to a string node.""" + string_without_prefix = str(node)[len(node.prefix) :] + string_leaf = Leaf(token.STRING, string_without_prefix, prefix=node.prefix) + string_leaf.lineno = node.get_lineno() or 0 + return string_leaf + + +def is_multiline_string(node: LN) -> bool: """Return True if `leaf` is a multiline string that actually spans many lines.""" + if isinstance(node, Node) and is_fstring(node): + leaf = fstring_to_string(node) + elif isinstance(node, Leaf): + leaf = node + else: + return False + return has_triple_quotes(leaf.value) and "\n" in leaf.value @@ -960,10 +992,6 @@ def is_rpar_token(nl: NL) -> TypeGuard[Leaf]: return nl.type == token.RPAR -def is_string_token(nl: NL) -> TypeGuard[Leaf]: - return nl.type == token.STRING - - def is_number_token(nl: NL) -> TypeGuard[Leaf]: return nl.type == token.NUMBER diff --git a/src/pyink/parsing.py b/src/pyink/parsing.py index 3f3cb949ede..d6fb1f7f87d 100644 --- a/src/pyink/parsing.py +++ b/src/pyink/parsing.py @@ -223,11 +223,9 @@ def _stringify_ast(node: ast.AST, parent_stack: List[ast.AST]) -> Iterator[str]: and field == "value" and isinstance(value, str) and len(parent_stack) >= 2 + # Any standalone string, ideally this would + # exactly match pyink.nodes.is_docstring and isinstance(parent_stack[-1], ast.Expr) - and isinstance( - parent_stack[-2], - (ast.FunctionDef, ast.AsyncFunctionDef, ast.Module, ast.ClassDef), - ) ): # Constant strings may be indented across newlines, if they are # docstrings; fold spaces after newlines when comparing. Similarly, diff --git a/src/pyink/strings.py b/src/pyink/strings.py index d15c259e643..cfa878709e5 100644 --- a/src/pyink/strings.py +++ b/src/pyink/strings.py @@ -5,7 +5,7 @@ import re import sys from functools import lru_cache -from typing import Final, List, Match, Pattern +from typing import Final, List, Match, Pattern, Tuple from pyink._width_table import WIDTH_TABLE from pyink.mode import Quote @@ -172,8 +172,7 @@ def normalize_string_quotes(s: str, *, preferred_quote: Quote) -> str: For three quotes strings, always use double-quote. - Adds or removes backslashes as appropriate. Doesn't parse and fix - strings nested in f-strings. + Adds or removes backslashes as appropriate. """ value = s.lstrip(STRING_PREFIX_CHARS) if value[:3] == '"""': @@ -214,6 +213,7 @@ def normalize_string_quotes(s: str, *, preferred_quote: Quote) -> str: s = f"{prefix}{orig_quote}{body}{orig_quote}" new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body) new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body) + if "f" in prefix.casefold(): matches = re.findall( r""" @@ -243,6 +243,71 @@ def normalize_string_quotes(s: str, *, preferred_quote: Quote) -> str: return f"{prefix}{new_quote}{new_body}{new_quote}" +def normalize_fstring_quotes( + quote: str, + middles: List[Leaf], + is_raw_fstring: bool, +) -> Tuple[List[Leaf], str]: + """Prefer double quotes but only if it doesn't cause more escaping. + + Adds or removes backslashes as appropriate. + """ + if quote == '"""': + return middles, quote + + elif quote == "'''": + new_quote = '"""' + elif quote == '"': + new_quote = "'" + else: + new_quote = '"' + + unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}") + escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}") + escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){quote}") + if is_raw_fstring: + for middle in middles: + if unescaped_new_quote.search(middle.value): + # There's at least one unescaped new_quote in this raw string + # so converting is impossible + return middles, quote + + # Do not introduce or remove backslashes in raw strings, just use double quote + return middles, '"' + + new_segments = [] + for middle in middles: + segment = middle.value + # remove unnecessary escapes + new_segment = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", segment) + if segment != new_segment: + # Consider the string without unnecessary escapes as the original + middle.value = new_segment + + new_segment = sub_twice(escaped_orig_quote, rf"\1\2{quote}", new_segment) + new_segment = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_segment) + new_segments.append(new_segment) + + if new_quote == '"""' and new_segments[-1].endswith('"'): + # edge case: + new_segments[-1] = new_segments[-1][:-1] + '\\"' + + for middle, new_segment in zip(middles, new_segments): + orig_escape_count = middle.value.count("\\") + new_escape_count = new_segment.count("\\") + + if new_escape_count > orig_escape_count: + return middles, quote # Do not introduce more escaping + + if new_escape_count == orig_escape_count and quote == '"': + return middles, quote # Prefer double quotes + + for middle, new_segment in zip(middles, new_segments): + middle.value = new_segment + + return middles, new_quote + + def normalize_unicode_escape_sequences(leaf: Leaf) -> None: """Replace hex codes in Unicode escape sequences with lowercase representation.""" text = leaf.value diff --git a/tests/data/cases/backslash_before_indent.py b/tests/data/cases/backslash_before_indent.py new file mode 100644 index 00000000000..2d126a945e4 --- /dev/null +++ b/tests/data/cases/backslash_before_indent.py @@ -0,0 +1,24 @@ +# flags: --minimum-version=3.10 +class Plotter: +\ + pass + +class AnotherCase: + \ + """Some + \ + Docstring + """ + +# output + +class Plotter: + + pass + + +class AnotherCase: + """Some + \ + Docstring + """ diff --git a/tests/data/cases/comment_after_escaped_newline.py b/tests/data/cases/comment_after_escaped_newline.py index 133f4898a47..133c230a3a8 100644 --- a/tests/data/cases/comment_after_escaped_newline.py +++ b/tests/data/cases/comment_after_escaped_newline.py @@ -14,5 +14,7 @@ def bob(): # pylint: disable=W9016 pass -def bobtwo(): # some comment here +def bobtwo(): + + # some comment here pass diff --git a/tests/data/cases/dummy_implementations.py b/tests/data/cases/dummy_implementations.py index 0a52c081bcc..107bc2c506f 100644 --- a/tests/data/cases/dummy_implementations.py +++ b/tests/data/cases/dummy_implementations.py @@ -68,6 +68,67 @@ async def async_function(self): async def async_function(self): ... +class ClassA: + def f(self): + + ... + + +class ClassB: + def f(self): + + + + + + + + + + ... + + +class ClassC: + def f(self): + + ... + # Comment + + +class ClassD: + def f(self):# Comment 1 + + ...# Comment 2 + # Comment 3 + + +class ClassE: + def f(self): + + ... + def f2(self): + print(10) + + +class ClassF: + def f(self): + + ...# Comment 2 + + +class ClassG: + def f(self):#Comment 1 + + ...# Comment 2 + + +class ClassH: + def f(self): + #Comment + + ... + + # output from typing import NoReturn, Protocol, Union, overload @@ -142,3 +203,47 @@ async def async_function(self): ... @decorated async def async_function(self): ... + + +class ClassA: + def f(self): ... + + +class ClassB: + def f(self): ... + + +class ClassC: + def f(self): + + ... + # Comment + + +class ClassD: + def f(self): # Comment 1 + + ... # Comment 2 + # Comment 3 + + +class ClassE: + def f(self): ... + def f2(self): + print(10) + + +class ClassF: + def f(self): ... # Comment 2 + + +class ClassG: + def f(self): # Comment 1 + ... # Comment 2 + + +class ClassH: + def f(self): + # Comment + + ... diff --git a/tests/data/cases/fmtonoff6.py b/tests/data/cases/fmtonoff6.py new file mode 100644 index 00000000000..9d23925063c --- /dev/null +++ b/tests/data/cases/fmtonoff6.py @@ -0,0 +1,13 @@ +# Regression test for https://github.com/psf/black/issues/2478. +def foo(): + arr = ( + (3833567325051000, 5, 1, 2, 4229.25, 6, 0), + # fmt: off + ) + + +# Regression test for https://github.com/psf/black/issues/3458. +dependencies = { + a: b, + # fmt: off +} diff --git a/tests/data/cases/form_feeds.py b/tests/data/cases/form_feeds.py index 48ffc98106b..957b4a1db95 100644 --- a/tests/data/cases/form_feeds.py +++ b/tests/data/cases/form_feeds.py @@ -156,6 +156,7 @@ def something(self): # + # pass diff --git a/tests/data/cases/pattern_matching_complex.py b/tests/data/cases/pattern_matching_complex.py index ba64f2639a0..028832d772a 100644 --- a/tests/data/cases/pattern_matching_complex.py +++ b/tests/data/cases/pattern_matching_complex.py @@ -83,7 +83,7 @@ match x: case [0]: y = 0 - case [1, 0] if (x := x[:0]): + case [1, 0] if x := x[:0]: y = 1 case [1, 0]: y = 2 diff --git a/tests/data/cases/pattern_matching_with_if_stmt.py b/tests/data/cases/pattern_matching_with_if_stmt.py new file mode 100644 index 00000000000..ff54af91771 --- /dev/null +++ b/tests/data/cases/pattern_matching_with_if_stmt.py @@ -0,0 +1,72 @@ +# flags: --preview --minimum-version=3.10 +match match: + case "test" if case != "not very loooooooooooooog condition": # comment + pass + +match smth: + case "test" if "any long condition" != "another long condition" and "this is a long condition": + pass + case test if "any long condition" != "another long condition" and "this is a looooong condition": + pass + case test if "any long condition" != "another long condition" and "this is a looooong condition": # some additional comments + pass + case test if (True): # some comment + pass + case test if (False + ): # some comment + pass + case test if (True # some comment + ): + pass # some comment + case cases if (True # some comment + ): # some other comment + pass # some comment + case match if (True # some comment + ): + pass # some comment + +# case black_test_patma_052 (originally in the pattern_matching_complex test case) +match x: + case [1, 0] if x := x[:0]: + y = 1 + case [1, 0] if (x := x[:0]): + y = 1 + +# output + +match match: + case "test" if case != "not very loooooooooooooog condition": # comment + pass + +match smth: + case "test" if ( + "any long condition" != "another long condition" and "this is a long condition" + ): + pass + case test if ( + "any long condition" != "another long condition" + and "this is a looooong condition" + ): + pass + case test if ( + "any long condition" != "another long condition" + and "this is a looooong condition" + ): # some additional comments + pass + case test if True: # some comment + pass + case test if False: # some comment + pass + case test if True: # some comment + pass # some comment + case cases if True: # some comment # some other comment + pass # some comment + case match if True: # some comment + pass # some comment + +# case black_test_patma_052 (originally in the pattern_matching_complex test case) +match x: + case [1, 0] if x := x[:0]: + y = 1 + case [1, 0] if x := x[:0]: + y = 1 diff --git a/tests/data/cases/pep_701.py b/tests/data/cases/pep_701.py new file mode 100644 index 00000000000..9acee951e71 --- /dev/null +++ b/tests/data/cases/pep_701.py @@ -0,0 +1,276 @@ +# flags: --minimum-version=3.12 +x = f"foo" +x = f'foo' +x = f"""foo""" +x = f'''foo''' +x = f"foo {{ bar {{ baz" +x = f"foo {{ {2 + 2}bar {{ baz" +x = f'foo {{ {2 + 2}bar {{ baz' +x = f"""foo {{ {2 + 2}bar {{ baz""" +x = f'''foo {{ {2 + 2}bar {{ baz''' + +# edge case: FSTRING_MIDDLE containing only whitespace should not be stripped +x = f"{a} {b}" + +x = f"foo { + 2 + 2 +} bar baz" + +x = f"foo {{ {"a {2 + 2} b"}bar {{ baz" +x = f"foo {{ {f'a {2 + 2} b'}bar {{ baz" +x = f"foo {{ {f"a {2 + 2} b"}bar {{ baz" + +x = f"foo {{ {f'a {f"a {2 + 2} b"} b'}bar {{ baz" +x = f"foo {{ {f"a {f"a {2 + 2} b"} b"}bar {{ baz" + +x = """foo {{ {2 + 2}bar +baz""" + + +x = f"""foo {{ {2 + 2}bar {{ baz""" + +x = f"""foo {{ { + 2 + 2 +}bar {{ baz""" + + +x = f"""foo {{ { + 2 + 2 +}bar +baz""" + +x = f"""foo {{ a + foo {2 + 2}bar {{ baz + + x = f"foo {{ { + 2 + 2 # comment + }bar" + + {{ baz + + }} buzz + + {print("abc" + "def" +)} +abc""" + +# edge case: end triple quotes at index zero +f"""foo {2+2} bar +""" + +f' \' {f"'"} \' ' +f" \" {f'"'} \" " + +x = f"a{2+2:=^72}b" +x = f"a{2+2:x}b" + +rf'foo' +rf'{foo}' + +f"{x:{y}d}" + +x = f"a{2+2:=^{x}}b" +x = f"a{2+2:=^{foo(x+y**2):something else}}b" +x = f"a{2+2:=^{foo(x+y**2):something else}one more}b" +f'{(abc:=10)}' + +f"This is a really long string, but just make sure that you reflow fstrings { + 2+2:d +}" +f"This is a really long string, but just make sure that you reflow fstrings correctly {2+2:d}" + +f"{2+2=}" +f"{2+2 = }" +f"{ 2 + 2 = }" + +f"""foo { + datetime.datetime.now():%Y +%m +%d +}""" + +f"{ +X +!r +}" + +raise ValueError( + "xxxxxxxxxxxIncorrect --line-ranges format, expect START-END, found" + f" {lines_str!r}" + ) + +f"`escape` only permitted in {{'html', 'latex', 'latex-math'}}, \ +got {escape}" + +x = f'\N{GREEK CAPITAL LETTER DELTA} \N{SNOWMAN} {x}' +fr'\{{\}}' + +f""" + WITH {f''' + {1}_cte AS ()'''} +""" + +value: str = f'''foo +''' + +log( + f"Received operation {server_operation.name} from " + f"{self.writer._transport.get_extra_info('peername')}", # type: ignore[attr-defined] + level=0, +) + +f"{1:{f'{2}'}}" +f'{1:{f'{2}'}}' +f'{1:{2}d}' + +f'{{\\"kind\\":\\"ConfigMap\\",\\"metadata\\":{{\\"annotations\\":{{}},\\"name\\":\\"cluster-info\\",\\"namespace\\":\\"amazon-cloudwatch\\"}}}}' + +f"""{''' +'''}""" + +f"{'\''}" +f"{f'\''}" + +f'{1}\{{' +f'{2} foo \{{[\}}' +f'\{3}' +rf"\{"a"}" + +# output + +x = f"foo" +x = f"foo" +x = f"""foo""" +x = f"""foo""" +x = f"foo {{ bar {{ baz" +x = f"foo {{ {2 + 2}bar {{ baz" +x = f"foo {{ {2 + 2}bar {{ baz" +x = f"""foo {{ {2 + 2}bar {{ baz""" +x = f"""foo {{ {2 + 2}bar {{ baz""" + +# edge case: FSTRING_MIDDLE containing only whitespace should not be stripped +x = f"{a} {b}" + +x = f"foo { + 2 + 2 +} bar baz" + +x = f"foo {{ {"a {2 + 2} b"}bar {{ baz" +x = f"foo {{ {f'a {2 + 2} b'}bar {{ baz" +x = f"foo {{ {f"a {2 + 2} b"}bar {{ baz" + +x = f"foo {{ {f'a {f"a {2 + 2} b"} b'}bar {{ baz" +x = f"foo {{ {f"a {f"a {2 + 2} b"} b"}bar {{ baz" + +x = """foo {{ {2 + 2}bar +baz""" + + +x = f"""foo {{ {2 + 2}bar {{ baz""" + +x = f"""foo {{ { + 2 + 2 +}bar {{ baz""" + + +x = f"""foo {{ { + 2 + 2 +}bar +baz""" + +x = f"""foo {{ a + foo {2 + 2}bar {{ baz + + x = f"foo {{ { + 2 + 2 # comment + }bar" + + {{ baz + + }} buzz + + {print("abc" + "def" +)} +abc""" + +# edge case: end triple quotes at index zero +f"""foo {2+2} bar +""" + +f' \' {f"'"} \' ' +f" \" {f'"'} \" " + +x = f"a{2+2:=^72}b" +x = f"a{2+2:x}b" + +rf"foo" +rf"{foo}" + +f"{x:{y}d}" + +x = f"a{2+2:=^{x}}b" +x = f"a{2+2:=^{foo(x+y**2):something else}}b" +x = f"a{2+2:=^{foo(x+y**2):something else}one more}b" +f"{(abc:=10)}" + +f"This is a really long string, but just make sure that you reflow fstrings { + 2+2:d +}" +f"This is a really long string, but just make sure that you reflow fstrings correctly {2+2:d}" + +f"{2+2=}" +f"{2+2 = }" +f"{ 2 + 2 = }" + +f"""foo { + datetime.datetime.now():%Y +%m +%d +}""" + +f"{ +X +!r +}" + +raise ValueError( + "xxxxxxxxxxxIncorrect --line-ranges format, expect START-END, found" + f" {lines_str!r}" +) + +f"`escape` only permitted in {{'html', 'latex', 'latex-math'}}, \ +got {escape}" + +x = f"\N{GREEK CAPITAL LETTER DELTA} \N{SNOWMAN} {x}" +rf"\{{\}}" + +f""" + WITH {f''' + {1}_cte AS ()'''} +""" + +value: str = f"""foo +""" + +log( + f"Received operation {server_operation.name} from " + f"{self.writer._transport.get_extra_info('peername')}", # type: ignore[attr-defined] + level=0, +) + +f"{1:{f'{2}'}}" +f"{1:{f'{2}'}}" +f"{1:{2}d}" + +f'{{\\"kind\\":\\"ConfigMap\\",\\"metadata\\":{{\\"annotations\\":{{}},\\"name\\":\\"cluster-info\\",\\"namespace\\":\\"amazon-cloudwatch\\"}}}}' + +f"""{''' +'''}""" + +f"{'\''}" +f"{f'\''}" + +f"{1}\{{" +f"{2} foo \{{[\}}" +f"\{3}" +rf"\{"a"}" diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py index 9288f6991bd..1daf6f4d784 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -175,6 +175,13 @@ def dastardly_default_value( "c" ) +assert some_var == expected_result, """ +test +""" +assert some_var == expected_result, f""" +expected: {expected_result} +actual: {some_var}""" + # output """cow say""", @@ -385,3 +392,10 @@ def dastardly_default_value( ) this_will_also_become_one_line = "abc" # comment + +assert some_var == expected_result, """ +test +""" +assert some_var == expected_result, f""" +expected: {expected_result} +actual: {some_var}""" diff --git a/tests/data/cases/type_param_defaults.py b/tests/data/cases/type_param_defaults.py new file mode 100644 index 00000000000..cd844fe0746 --- /dev/null +++ b/tests/data/cases/type_param_defaults.py @@ -0,0 +1,61 @@ +# flags: --minimum-version=3.13 + +type A[T=int] = float +type B[*P=int] = float +type C[*Ts=int] = float +type D[*Ts=*int] = float +type D[something_that_is_very_very_very_long=something_that_is_very_very_very_long] = float +type D[*something_that_is_very_very_very_long=*something_that_is_very_very_very_long] = float +type something_that_is_long[something_that_is_long=something_that_is_long] = something_that_is_long + +def simple[T=something_that_is_long](short1: int, short2: str, short3: bytes) -> float: + pass + +def longer[something_that_is_long=something_that_is_long](something_that_is_long: something_that_is_long) -> something_that_is_long: + pass + +def trailing_comma1[T=int,](a: str): + pass + +def trailing_comma2[T=int](a: str,): + pass + +# output + +type A[T = int] = float +type B[*P = int] = float +type C[*Ts = int] = float +type D[*Ts = *int] = float +type D[ + something_that_is_very_very_very_long = something_that_is_very_very_very_long +] = float +type D[ + *something_that_is_very_very_very_long = *something_that_is_very_very_very_long +] = float +type something_that_is_long[ + something_that_is_long = something_that_is_long +] = something_that_is_long + + +def simple[ + T = something_that_is_long +](short1: int, short2: str, short3: bytes) -> float: + pass + + +def longer[ + something_that_is_long = something_that_is_long +](something_that_is_long: something_that_is_long) -> something_that_is_long: + pass + + +def trailing_comma1[ + T = int, +](a: str): + pass + + +def trailing_comma2[ + T = int +](a: str,): + pass diff --git a/tests/data/ignore_directory_gitignore_tests/.gitignore b/tests/data/ignore_directory_gitignore_tests/.gitignore new file mode 100644 index 00000000000..4573ac5b3ac --- /dev/null +++ b/tests/data/ignore_directory_gitignore_tests/.gitignore @@ -0,0 +1,3 @@ +large_ignored_dir/ +large_ignored_dir_two +abc.py diff --git a/tests/data/ignore_directory_gitignore_tests/abc.py b/tests/data/ignore_directory_gitignore_tests/abc.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/a.py b/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/a.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner/b.py b/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner/b.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner2/c.py b/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner2/c.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner3/d.py b/tests/data/ignore_directory_gitignore_tests/large_ignored_dir_two/inner3/d.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/ignore_directory_gitignore_tests/z.py b/tests/data/ignore_directory_gitignore_tests/z.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/miscellaneous/debug_visitor.out b/tests/data/miscellaneous/debug_visitor.out index fa60010d421..24d7ed82472 100644 --- a/tests/data/miscellaneous/debug_visitor.out +++ b/tests/data/miscellaneous/debug_visitor.out @@ -229,8 +229,34 @@ file_input LPAR '(' arglist - STRING - "f'{indent}{_type}'" + fstring + FSTRING_START + "f'" + FSTRING_MIDDLE + '' + fstring_replacement_field + LBRACE + '{' + NAME + 'indent' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + fstring_replacement_field + LBRACE + '{' + NAME + '_type' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + FSTRING_END + "'" + /fstring COMMA ',' argument @@ -370,8 +396,34 @@ file_input LPAR '(' arglist - STRING - "f'{indent}/{_type}'" + fstring + FSTRING_START + "f'" + FSTRING_MIDDLE + '' + fstring_replacement_field + LBRACE + '{' + NAME + 'indent' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '/' + fstring_replacement_field + LBRACE + '{' + NAME + '_type' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + FSTRING_END + "'" + /fstring COMMA ',' argument @@ -494,8 +546,34 @@ file_input LPAR '(' arglist - STRING - "f'{indent}{_type}'" + fstring + FSTRING_START + "f'" + FSTRING_MIDDLE + '' + fstring_replacement_field + LBRACE + '{' + NAME + 'indent' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + fstring_replacement_field + LBRACE + '{' + NAME + '_type' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + FSTRING_END + "'" + /fstring COMMA ',' argument @@ -557,8 +635,36 @@ file_input LPAR '(' arglist - STRING - "f' {node.prefix!r}'" + fstring + FSTRING_START + "f'" + FSTRING_MIDDLE + ' ' + fstring_replacement_field + LBRACE + '{' + power + NAME + 'node' + trailer + DOT + '.' + NAME + 'prefix' + /trailer + /power + BANG + '!' + NAME + 'r' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + FSTRING_END + "'" + /fstring COMMA ',' argument @@ -613,8 +719,36 @@ file_input LPAR '(' arglist - STRING - "f' {node.value!r}'" + fstring + FSTRING_START + "f'" + FSTRING_MIDDLE + ' ' + fstring_replacement_field + LBRACE + '{' + power + NAME + 'node' + trailer + DOT + '.' + NAME + 'value' + /trailer + /power + BANG + '!' + NAME + 'r' + RBRACE + '}' + /fstring_replacement_field + FSTRING_MIDDLE + '' + FSTRING_END + "'" + /fstring COMMA ',' argument diff --git a/tests/data/pyink_configs/majority_quotes.py b/tests/data/pyink_configs/majority_quotes.py index 26e2cd67c0e..606d7504b4b 100644 --- a/tests/data/pyink_configs/majority_quotes.py +++ b/tests/data/pyink_configs/majority_quotes.py @@ -5,3 +5,5 @@ _single = 'Single' _another_single = 'Another Single' _double = "Double" +_fstring_single = f'fstring single {4 + 2}' +_nested_fstring_double = f"nested fstring {f'double {4 + 2}'}" diff --git a/tests/optional.py b/tests/optional.py index 70ee823e316..142da844898 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -93,7 +93,7 @@ def pytest_configure(config: "Config") -> None: ot_run |= {no(excluded) for excluded in ot_markers - ot_run} ot_markers |= {no(m) for m in ot_markers} - log.info("optional tests to run:", ot_run) + log.info("optional tests to run: %s", ot_run) unknown_tests = ot_run - ot_markers if unknown_tests: raise ValueError(f"Unknown optional tests wanted: {unknown_tests!r}") @@ -115,17 +115,17 @@ def pytest_collection_modifyitems(config: "Config", items: "List[Node]") -> None optional_markers_on_test & enabled_optional_markers ): continue - log.info("skipping non-requested optional", item) + log.info("skipping non-requested optional: %s", item) item.add_marker(skip_mark(frozenset(optional_markers_on_test))) -@lru_cache() +@lru_cache def skip_mark(tests: FrozenSet[str]) -> "MarkDecorator": names = ", ".join(sorted(tests)) return pytest.mark.skip(reason=f"Marked with disabled optional tests ({names})") -@lru_cache() +@lru_cache def no(name: str) -> str: if name.startswith("no_"): return name[len("no_") :] diff --git a/tests/test_black.py b/tests/test_black.py index f2bf32cc13c..7a00fbd31fe 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -263,6 +263,21 @@ def test_pep_695_version_detection(self) -> None: versions = pyink.detect_target_versions(root) self.assertIn(pyink.TargetVersion.PY312, versions) + def test_pep_696_version_detection(self) -> None: + source, _ = read_data("cases", "type_param_defaults") + samples = [ + source, + "type X[T=int] = float", + "type X[T:int=int]=int", + "type X[*Ts=int]=int", + "type X[*Ts=*int]=int", + "type X[**P=int]=int", + ] + for sample in samples: + root = pyink.lib2to3_parse(sample) + features = pyink.get_features_used(root) + self.assertIn(pyink.Feature.TYPE_PARAM_DEFAULTS, features) + def test_expression_ff(self) -> None: source, expected = read_data("cases", "expression.py") tmp_file = Path(pyink.dump_to_file(source)) @@ -343,12 +358,11 @@ def test_detect_debug_f_strings(self) -> None: features = pyink.get_features_used(root) self.assertNotIn(pyink.Feature.DEBUG_F_STRINGS, features) - # We don't yet support feature version detection in nested f-strings root = pyink.lib2to3_parse( """f"heard a rumour that { f'{1+1=}' } ... seems like it could be true" """ ) features = pyink.get_features_used(root) - self.assertNotIn(pyink.Feature.DEBUG_F_STRINGS, features) + self.assertIn(pyink.Feature.DEBUG_F_STRINGS, features) @patch("pyink.dump_to_file", dump_to_stderr) def test_string_quotes(self) -> None: @@ -474,20 +488,8 @@ def test_false_positive_symlink_output_issue_3384(self) -> None: # running from CLI, but fails when running the tests because cwd is different project_root = Path(THIS_DIR / "data" / "nested_gitignore_tests") working_directory = project_root / "root" - target_abspath = working_directory / "child" - target_contents = list(target_abspath.iterdir()) - - def mock_n_calls(responses: List[bool]) -> Callable[[], bool]: - def _mocked_calls() -> bool: - if responses: - return responses.pop(0) - return False - return _mocked_calls - - with patch("pathlib.Path.iterdir", return_value=target_contents), patch( - "pathlib.Path.resolve", return_value=target_abspath - ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): + with change_directory(working_directory): # Note that the root folder (project_root) isn't the folder # named "root" (aka working_directory) report = MagicMock(verbose=True) @@ -1544,16 +1546,34 @@ def test_infer_target_version(self) -> None: for version, expected in [ ("3.6", [TargetVersion.PY36]), ("3.11.0rc1", [TargetVersion.PY311]), - (">=3.10", [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312]), + ( + ">=3.10", + [ + TargetVersion.PY310, + TargetVersion.PY311, + TargetVersion.PY312, + TargetVersion.PY313, + ], + ), ( ">=3.10.6", - [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + [ + TargetVersion.PY310, + TargetVersion.PY311, + TargetVersion.PY312, + TargetVersion.PY313, + ], ), ("<3.6", [TargetVersion.PY33, TargetVersion.PY34, TargetVersion.PY35]), (">3.7,<3.10", [TargetVersion.PY38, TargetVersion.PY39]), ( ">3.7,!=3.8,!=3.9", - [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + [ + TargetVersion.PY310, + TargetVersion.PY311, + TargetVersion.PY312, + TargetVersion.PY313, + ], ), ( "> 3.9.4, != 3.10.3", @@ -1562,6 +1582,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312, + TargetVersion.PY313, ], ), ( @@ -1575,6 +1596,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312, + TargetVersion.PY313, ], ), ( @@ -1590,6 +1612,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312, + TargetVersion.PY313, ], ), ("==3.8.*", [TargetVersion.PY38]), @@ -2514,6 +2537,12 @@ def test_gitignore_that_ignores_subfolders(self) -> None: expected = [target / "b.py"] assert_collected_sources([target], expected, root=root) + def test_gitignore_that_ignores_directory(self) -> None: + # If gitignore with a directory is in root + root = Path(DATA_DIR, "ignore_directory_gitignore_tests") + expected = [root / "z.py"] + assert_collected_sources([root], expected, root=root) + def test_empty_include(self) -> None: path = DATA_DIR / "include_exclude_tests" src = [path] @@ -2692,11 +2721,19 @@ def test_get_sources_symlink_and_force_exclude(self) -> None: def test_get_sources_with_stdin_symlink_outside_root( self, ) -> None: - path = THIS_DIR / "data" / "include_exclude_tests" - stdin_filename = str(path / "b/exclude/a.py") - outside_root_symlink = Path("/target_directory/a.py") - root = Path("target_dir/").resolve().absolute() - with patch("pathlib.Path.resolve", return_value=outside_root_symlink): + with TemporaryDirectory() as tempdir: + tmp = Path(tempdir).resolve() + + root = tmp / "root" + root.mkdir() + (root / "pyproject.toml").write_text("[tool.pyink]", encoding="utf-8") + + target = tmp / "outside_root" / "a.py" + target.parent.mkdir() + target.write_text("print('hello')", encoding="utf-8") + (root / "a.py").symlink_to(target) + + stdin_filename = str(root / "a.py") assert_collected_sources( root=root, src=["-"], @@ -2984,6 +3021,22 @@ async def f(): """docstring""" ''', ) + self.check_ast_equivalence( + """ + if __name__ == "__main__": + " docstring-like " + """, + ''' + if __name__ == "__main__": + """docstring-like""" + ''', + ) + self.check_ast_equivalence(r'def f(): r" \n "', r'def f(): "\\n"') + self.check_ast_equivalence('try: pass\nexcept: " x "', 'try: pass\nexcept: "x"') + + self.check_ast_equivalence( + 'def foo(): return " x "', 'def foo(): return "x"', should_fail=True + ) def test_assert_equivalent_fstring(self) -> None: major, minor = sys.version_info[:2] @@ -3013,7 +3066,7 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: try: - with open(pyink.__file__, "r", encoding="utf-8") as _bf: + with open(pyink.__file__, encoding="utf-8") as _bf: black_source_lines = _bf.readlines() except UnicodeDecodeError: if not pyink.COMPILED: diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py new file mode 100644 index 00000000000..216669bf9b3 --- /dev/null +++ b/tests/test_tokenize.py @@ -0,0 +1,120 @@ +"""Tests for the blib2to3 tokenizer.""" + +import io +import sys +import textwrap +from dataclasses import dataclass +from typing import List + +import pyink +from blib2to3.pgen2 import token, tokenize + + +@dataclass +class Token: + type: str + string: str + start: tokenize.Coord + end: tokenize.Coord + + +def get_tokens(text: str) -> List[Token]: + """Return the tokens produced by the tokenizer.""" + readline = io.StringIO(text).readline + tokens: List[Token] = [] + + def tokeneater( + type: int, string: str, start: tokenize.Coord, end: tokenize.Coord, line: str + ) -> None: + tokens.append(Token(token.tok_name[type], string, start, end)) + + tokenize.tokenize(readline, tokeneater) + return tokens + + +def assert_tokenizes(text: str, tokens: List[Token]) -> None: + """Assert that the tokenizer produces the expected tokens.""" + actual_tokens = get_tokens(text) + assert actual_tokens == tokens + + +def test_simple() -> None: + assert_tokenizes( + "1", + [Token("NUMBER", "1", (1, 0), (1, 1)), Token("ENDMARKER", "", (2, 0), (2, 0))], + ) + assert_tokenizes( + "'a'", + [ + Token("STRING", "'a'", (1, 0), (1, 3)), + Token("ENDMARKER", "", (2, 0), (2, 0)), + ], + ) + assert_tokenizes( + "a", + [Token("NAME", "a", (1, 0), (1, 1)), Token("ENDMARKER", "", (2, 0), (2, 0))], + ) + + +def test_fstring() -> None: + assert_tokenizes( + 'f"x"', + [ + Token("FSTRING_START", 'f"', (1, 0), (1, 2)), + Token("FSTRING_MIDDLE", "x", (1, 2), (1, 3)), + Token("FSTRING_END", '"', (1, 3), (1, 4)), + Token("ENDMARKER", "", (2, 0), (2, 0)), + ], + ) + assert_tokenizes( + 'f"{x}"', + [ + Token("FSTRING_START", 'f"', (1, 0), (1, 2)), + Token("FSTRING_MIDDLE", "", (1, 2), (1, 2)), + Token("LBRACE", "{", (1, 2), (1, 3)), + Token("NAME", "x", (1, 3), (1, 4)), + Token("RBRACE", "}", (1, 4), (1, 5)), + Token("FSTRING_MIDDLE", "", (1, 5), (1, 5)), + Token("FSTRING_END", '"', (1, 5), (1, 6)), + Token("ENDMARKER", "", (2, 0), (2, 0)), + ], + ) + assert_tokenizes( + 'f"{x:y}"\n', + [ + Token(type="FSTRING_START", string='f"', start=(1, 0), end=(1, 2)), + Token(type="FSTRING_MIDDLE", string="", start=(1, 2), end=(1, 2)), + Token(type="LBRACE", string="{", start=(1, 2), end=(1, 3)), + Token(type="NAME", string="x", start=(1, 3), end=(1, 4)), + Token(type="OP", string=":", start=(1, 4), end=(1, 5)), + Token(type="FSTRING_MIDDLE", string="y", start=(1, 5), end=(1, 6)), + Token(type="RBRACE", string="}", start=(1, 6), end=(1, 7)), + Token(type="FSTRING_MIDDLE", string="", start=(1, 7), end=(1, 7)), + Token(type="FSTRING_END", string='"', start=(1, 7), end=(1, 8)), + Token(type="NEWLINE", string="\n", start=(1, 8), end=(1, 9)), + Token(type="ENDMARKER", string="", start=(2, 0), end=(2, 0)), + ], + ) + assert_tokenizes( + 'f"x\\\n{a}"\n', + [ + Token(type="FSTRING_START", string='f"', start=(1, 0), end=(1, 2)), + Token(type="FSTRING_MIDDLE", string="x\\\n", start=(1, 2), end=(2, 0)), + Token(type="LBRACE", string="{", start=(2, 0), end=(2, 1)), + Token(type="NAME", string="a", start=(2, 1), end=(2, 2)), + Token(type="RBRACE", string="}", start=(2, 2), end=(2, 3)), + Token(type="FSTRING_MIDDLE", string="", start=(2, 3), end=(2, 3)), + Token(type="FSTRING_END", string='"', start=(2, 3), end=(2, 4)), + Token(type="NEWLINE", string="\n", start=(2, 4), end=(2, 5)), + Token(type="ENDMARKER", string="", start=(3, 0), end=(3, 0)), + ], + ) + + +# Run "echo some code | python tests/test_tokenize.py" to generate test cases. +if __name__ == "__main__": + code = sys.stdin.read() + tokens = get_tokens(code) + text = f"assert_tokenizes({code!r}, {tokens!r})" + text = pyink.format_str(text, mode=pyink.Mode()) + print(textwrap.indent(text, " ")) diff --git a/tests/util.py b/tests/util.py index aef25ec15d1..081517a1bf9 100644 --- a/tests/util.py +++ b/tests/util.py @@ -230,7 +230,7 @@ def _parse_minimum_version(version: str) -> Tuple[int, int]: return int(major), int(minor) -@functools.lru_cache() +@functools.lru_cache def get_flags_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument( @@ -307,7 +307,7 @@ def parse_mode(flags_line: str) -> TestCaseArgs: def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: - with open(file_name, "r", encoding="utf8") as test: + with open(file_name, encoding="utf8") as test: lines = test.readlines() _input: List[str] = [] _output: List[str] = [] diff --git a/tox.ini b/tox.ini index 081f92794c4..ca51e590d7f 100644 --- a/tox.ini +++ b/tox.ini @@ -95,4 +95,4 @@ setenv = PYTHONPATH = {toxinidir}/src skip_install = True commands = pip install -e . - pyink --check {toxinidir}/src {toxinidir}/tests + pyink --check {toxinidir}/src {toxinidir}/tests {toxinidir}/docs {toxinidir}/scripts