From 4ef6271fe4386df7b85dce858c3dcd8ff4d153fc Mon Sep 17 00:00:00 2001 From: Matej Aleksandrov Date: Fri, 27 Sep 2024 05:18:50 -0700 Subject: [PATCH] Handle comments with annotations PiperOrigin-RevId: 679548102 --- patches/pyink.patch | 585 ++++++++++++++++++------ src/pyink/__init__.py | 24 +- src/pyink/comments.py | 5 +- src/pyink/ink.py | 24 +- src/pyink/linegen.py | 39 +- src/pyink/lines.py | 52 +-- src/pyink/mode.py | 8 + src/pyink/nodes.py | 7 +- src/pyink/trans.py | 4 +- tests/data/cases/torture.py | 4 +- tests/data/pyink_configs/overrides.toml | 1 + tests/test_black.py | 14 + 12 files changed, 551 insertions(+), 216 deletions(-) diff --git a/patches/pyink.patch b/patches/pyink.patch index 7bfcd6442f5..eb4fd7f3508 100644 --- a/patches/pyink.patch +++ b/patches/pyink.patch @@ -40,7 +40,7 @@ from pyink._pyink_version import version as __version__ from pyink.cache import Cache from pyink.comments import normalize_fmt_off -@@ -66,9 +67,9 @@ from pyink.handle_ipynb_magics import ( +@@ -66,9 +67,15 @@ from pyink.handle_ipynb_magics import ( ) from pyink.linegen import LN, LineGenerator, transform_line from pyink.lines import EmptyLineTracker, LinesBlock @@ -48,11 +48,17 @@ +from pyink.mode import FUTURE_FLAG_TO_FEATURE, Feature, VERSION_TO_FEATURES from pyink.mode import Mode as Mode # re-exported -from pyink.mode import Preview, TargetVersion, supports_feature -+from pyink.mode import Preview, QuoteStyle, TargetVersion, supports_feature ++from pyink.mode import ( ++ DEFAULT_ANNOTATION_PRAGMAS, ++ Preview, ++ QuoteStyle, ++ TargetVersion, ++ supports_feature, ++) from pyink.nodes import STARS, is_number_token, is_simple_decorator_expression, syms from pyink.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from pyink.parsing import ( # noqa F401 -@@ -90,9 +91,8 @@ from pyink.ranges import ( +@@ -84,9 +91,8 @@ from pyink.ranges import ( parse_line_ranges, sanitized_lines, ) @@ -63,7 +69,7 @@ COMPILED = Path(__file__).suffix in (".pyd", ".so") -@@ -273,25 +273,26 @@ def validate_regex( +@@ -264,25 +270,26 @@ def validate_regex( multiple=True, help=( "Python versions that should be supported by Black's output. You should" @@ -97,7 +103,7 @@ ), ) @click.option( -@@ -326,17 +327,17 @@ def validate_regex( +@@ -317,17 +324,17 @@ def validate_regex( "--preview", is_flag=True, help=( @@ -120,7 +126,7 @@ ), ) @click.option( -@@ -351,20 +352,56 @@ def validate_regex( +@@ -342,20 +349,68 @@ def validate_regex( ), ) @click.option( @@ -150,6 +156,18 @@ + ), +) +@click.option( ++ "--pyink-annotation-pragmas", ++ type=str, ++ multiple=True, ++ help=( ++ "Pyink won't split too long lines if they contain a comment with any of" ++ " the given annotation pragmas because it doesn't know to which part of" ++ " the line the annotation applies. The default annotation pragmas are:" ++ f" {', '.join(DEFAULT_ANNOTATION_PRAGMAS)}." ++ ), ++ default=[], ++) ++@click.option( + "--pyink-use-majority-quotes", + is_flag=True, + help=( @@ -182,7 +200,7 @@ ), ) @click.option( -@@ -377,11 +404,11 @@ def validate_regex( +@@ -368,11 +423,11 @@ def validate_regex( multiple=True, metavar="START-END", help=( @@ -199,7 +217,7 @@ ), default=(), ) -@@ -389,9 +416,9 @@ def validate_regex( +@@ -380,9 +435,9 @@ def validate_regex( "--fast/--safe", is_flag=True, help=( @@ -212,7 +230,7 @@ ), ) @click.option( -@@ -401,8 +428,8 @@ def validate_regex( +@@ -392,8 +447,8 @@ def validate_regex( "Require a specific version of Black to be running. This is useful for" " ensuring that all contributors to your project are using the same" " version, because different versions of Black may format code a little" @@ -223,7 +241,7 @@ ), ) @click.option( -@@ -410,11 +437,12 @@ def validate_regex( +@@ -401,11 +456,12 @@ def validate_regex( type=str, callback=validate_regex, help=( @@ -241,7 +259,7 @@ ), show_default=False, ) -@@ -423,8 +451,8 @@ def validate_regex( +@@ -414,8 +470,8 @@ def validate_regex( type=str, callback=validate_regex, help=( @@ -252,7 +270,7 @@ ), ) @click.option( -@@ -432,10 +460,10 @@ def validate_regex( +@@ -423,10 +479,10 @@ def validate_regex( type=str, callback=validate_regex, help=( @@ -267,7 +285,7 @@ ), ) @click.option( -@@ -443,9 +471,9 @@ def validate_regex( +@@ -434,9 +490,9 @@ def validate_regex( type=str, is_eager=True, help=( @@ -280,7 +298,7 @@ ), ) @click.option( -@@ -455,10 +483,10 @@ def validate_regex( +@@ -446,10 +502,10 @@ def validate_regex( callback=validate_regex, help=( "A regular expression that matches files and directories that should be" @@ -295,7 +313,7 @@ ), show_default=True, ) -@@ -468,10 +496,10 @@ def validate_regex( +@@ -459,10 +515,10 @@ def validate_regex( type=click.IntRange(min=1), default=None, help=( @@ -310,7 +328,7 @@ ), ) @click.option( -@@ -479,8 +507,8 @@ def validate_regex( +@@ -470,8 +526,8 @@ def validate_regex( "--quiet", is_flag=True, help=( @@ -321,7 +339,7 @@ ), ) @click.option( -@@ -496,15 +524,20 @@ def validate_regex( +@@ -487,15 +543,20 @@ def validate_regex( @click.version_option( version=__version__, message=( @@ -345,18 +363,19 @@ ), is_eager=True, metavar="SRC ...", -@@ -543,6 +576,10 @@ def main( # noqa: C901 +@@ -534,6 +595,11 @@ def main( # noqa: C901 preview: bool, unstable: bool, enable_unstable_feature: List[Preview], + pyink: bool, + pyink_indentation: str, + pyink_ipynb_indentation: str, ++ pyink_annotation_pragmas: List[str], + pyink_use_majority_quotes: bool, quiet: bool, verbose: bool, required_version: Optional[str], -@@ -640,7 +677,12 @@ def main( # noqa: C901 +@@ -631,7 +697,15 @@ def main( # noqa: C901 preview=preview, unstable=unstable, python_cell_magics=set(python_cell_magics), @@ -364,13 +383,16 @@ + is_pyink=pyink, + pyink_indentation=int(pyink_indentation), + pyink_ipynb_indentation=int(pyink_ipynb_indentation), ++ pyink_annotation_pragmas=( ++ tuple(pyink_annotation_pragmas) or DEFAULT_ANNOTATION_PRAGMAS ++ ), + quote_style=( + QuoteStyle.MAJORITY if pyink_use_majority_quotes else QuoteStyle.DOUBLE + ), ) lines: List[Tuple[int, int]] = [] -@@ -1153,9 +677,10 @@ +@@ -1098,9 +1172,10 @@ def validate_cell(src: str, mode: Mode) """ if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS): raise NothingChanged @@ -384,7 +406,7 @@ ): raise NothingChanged -@@ -1175,7 +1219,6 @@ +@@ -1164,7 +1239,6 @@ def format_ipynb_string(src_contents: st raise NothingChanged trailing_newline = src_contents[-1] == "\n" @@ -392,7 +414,7 @@ nb = json.loads(src_contents) validate_metadata(nb) for cell in nb["cells"]: -@@ -1184,14 +1230,15 @@ +@@ -1176,14 +1250,15 @@ def format_ipynb_string(src_contents: st pass else: cell["source"] = dst.splitlines(keepends=True) @@ -416,7 +438,7 @@ def format_str( -@@ -1253,6 +1294,8 @@ def _format_str_once( +@@ -1244,6 +1319,8 @@ def _format_str_once( future_imports = get_future_imports(src_node) versions = detect_target_versions(src_node, future_imports=future_imports) @@ -442,9 +464,48 @@ (917760, 917999, 0), -] +) +--- a/comments.py ++++ b/comments.py +@@ -3,6 +3,7 @@ from dataclasses import dataclass + from functools import lru_cache + from typing import Collection, Final, Iterator, List, Optional, Tuple, Union + ++from pyink import ink + from pyink.mode import Mode, Preview + from pyink.nodes import ( + CLOSING_BRACKETS, +@@ -376,7 +377,7 @@ def children_contains_fmt_on(container: + return False + + +-def contains_pragma_comment(comment_list: List[Leaf]) -> bool: ++def contains_pragma_comment(comment_list: List[Leaf], mode: Mode) -> bool: + """ + Returns: + True iff one of the comments in @comment_list is a pragma used by one +@@ -384,7 +385,7 @@ def contains_pragma_comment(comment_list + pylint). + """ + for comment in comment_list: +- if comment.value.startswith(("# type:", "# noqa", "# pylint:")): ++ if ink.comment_contains_pragma(comment.value, mode): + return True + + return False +--- a/files.py ++++ b/files.py +@@ -231,7 +231,7 @@ def strip_specifier_set(specifier_set: S + def find_user_pyproject_toml() -> Path: + r"""Return the path to the top-level user configuration for pyink. + +- This looks for ~\.black on Windows and ~/.config/black on Linux and other ++ This looks for ~\.pyink on Windows and ~/.config/pyink on Linux and other + Unix systems. + + May raise: --- a/handle_ipynb_magics.py +++ b/handle_ipynb_magics.py -@@ -263,6 +263,8 @@ +@@ -273,6 +273,8 @@ def unmask_cell(src: str, replacements: """ for replacement in replacements: src = src.replace(replacement.mask, replacement.src) @@ -455,7 +516,7 @@ --- a/linegen.py +++ b/linegen.py -@@ -9,6 +9,11 @@ from enum import Enum, auto +@@ -9,6 +9,12 @@ from enum import Enum, auto from functools import partial, wraps from typing import Collection, Iterator, List, Optional, Set, Union, cast @@ -464,10 +525,11 @@ +else: + from typing import Final, Literal + ++from pyink import ink from pyink.brackets import ( COMMA_PRIORITY, DOT_PRIORITY, -@@ -18,6 +23,7 @@ from pyink.brackets import ( +@@ -18,6 +24,7 @@ from pyink.brackets import ( ) from pyink.comments import FMT_OFF, generate_comments, list_comments from pyink.lines import ( @@ -475,7 +537,15 @@ Line, RHSResult, append_leaves, -@@ -88,6 +94,15 @@ LeafID = int +@@ -55,7 +62,6 @@ from pyink.nodes import ( + is_stub_body, + is_stub_suite, + is_tuple_containing_walrus, +- is_type_ignore_comment_string, + is_vararg, + is_walrus_assignment, + is_yield, +@@ -87,6 +93,15 @@ LeafID = int LN = Union[Leaf, Node] @@ -491,7 +561,7 @@ class CannotSplit(CannotTransform): """A readable split that fits the allotted line length is impossible.""" -@@ -107,7 +122,9 @@ class LineGenerator(Visitor[Line]): +@@ -106,7 +121,9 @@ class LineGenerator(Visitor[Line]): self.current_line: Line self.__post_init__() @@ -502,7 +572,7 @@ """Generate a line. If the line is empty, only emit if it makes sense. -@@ -116,7 +133,10 @@ class LineGenerator(Visitor[Line]): +@@ -115,7 +132,10 @@ class LineGenerator(Visitor[Line]): If any lines were generated, set up a new current_line. """ if not self.current_line: @@ -514,7 +584,7 @@ return # Line is empty, don't emit. Creating a new one unnecessary. if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef( -@@ -129,7 +149,13 @@ class LineGenerator(Visitor[Line]): +@@ -128,7 +148,13 @@ class LineGenerator(Visitor[Line]): return complete_line = self.current_line @@ -529,7 +599,7 @@ yield complete_line def visit_default(self, node: LN) -> Iterator[Line]: -@@ -166,26 +194,27 @@ class LineGenerator(Visitor[Line]): +@@ -160,26 +186,27 @@ class LineGenerator(Visitor[Line]): def visit_test(self, node: Node) -> Iterator[Line]: """Visit an `x if y else z` test""" @@ -570,7 +640,7 @@ yield from self.visit_default(node) def visit_DEDENT(self, node: Leaf) -> Iterator[Line]: -@@ -200,7 +229,7 @@ class LineGenerator(Visitor[Line]): +@@ -194,7 +221,7 @@ class LineGenerator(Visitor[Line]): yield from self.visit_default(node) # Finally, emit the dedent. @@ -579,18 +649,36 @@ def visit_stmt( self, node: Node, keywords: Set[str], parens: Set[str] -@@ -293,7 +322,9 @@ class LineGenerator(Visitor[Line]): +@@ -245,6 +272,7 @@ class LineGenerator(Visitor[Line]): + maybe_make_parens_invisible_in_atom( + child, + parent=node, ++ mode=self.mode, + remove_brackets_around_comma=False, + ) + else: +@@ -265,6 +293,7 @@ class LineGenerator(Visitor[Line]): + if maybe_make_parens_invisible_in_atom( + child, + parent=node, ++ mode=self.mode, + remove_brackets_around_comma=False, + ): + wrap_in_parentheses(node, child, visible=False) +@@ -287,7 +316,11 @@ class LineGenerator(Visitor[Line]): def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" - if is_stub_suite(node): + if ( -+ self.mode.is_pyi or not self.mode.is_pyink -+ ) and is_stub_suite(node, self.mode): ++ self.mode.is_pyi ++ or not self.mode.is_pyink ++ and is_stub_suite(node, self.mode) ++ ): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) -@@ -307,15 +338,23 @@ class LineGenerator(Visitor[Line]): +@@ -301,15 +334,23 @@ class LineGenerator(Visitor[Line]): prev_type = child.type if node.parent and node.parent.type in STATEMENT: @@ -618,7 +706,25 @@ node.prefix = "" yield from self.visit_default(node) return -@@ -414,7 +453,10 @@ class LineGenerator(Visitor[Line]): +@@ -358,7 +399,7 @@ class LineGenerator(Visitor[Line]): + ): + wrap_in_parentheses(node, leaf) + +- remove_await_parens(node) ++ remove_await_parens(node, mode=self.mode) + + yield from self.visit_default(node) + +@@ -405,13 +446,18 @@ class LineGenerator(Visitor[Line]): + def foo(a: (int), b: (float) = 7): ... + """ + assert len(node.children) == 3 +- if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): ++ if maybe_make_parens_invisible_in_atom( ++ node.children[2], parent=node, mode=self.mode ++ ): + wrap_in_parentheses(node, node.children[2], visible=False) + yield from self.visit_default(node) def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: @@ -630,7 +736,7 @@ normalize_unicode_escape_sequences(leaf) if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): -@@ -428,7 +470,9 @@ class LineGenerator(Visitor[Line]): +@@ -424,7 +470,9 @@ class LineGenerator(Visitor[Line]): # see padding logic below), there's a possibility for unstable # formatting. To avoid a situation where this function formats a # docstring differently on the second pass, normalize it early. @@ -641,7 +747,7 @@ else: docstring = leaf.value prefix = get_string_prefix(docstring) -@@ -442,7 +486,7 @@ class LineGenerator(Visitor[Line]): +@@ -438,7 +486,7 @@ class LineGenerator(Visitor[Line]): quote_len = 1 if docstring[1] != quote_char else 3 docstring = docstring[quote_len:-quote_len] docstring_started_empty = not docstring @@ -650,7 +756,7 @@ if is_multiline_string(leaf): docstring = fix_docstring(docstring, indent) -@@ -484,7 +528,13 @@ class LineGenerator(Visitor[Line]): +@@ -473,7 +521,13 @@ class LineGenerator(Visitor[Line]): # If docstring is one line, we don't put the closing quotes on a # separate line because it looks ugly (#3320). lines = docstring.splitlines() @@ -665,17 +771,7 @@ # If adding closing quotes would cause the last line to exceed # the maximum line length, and the closing quote is not -@@ -531,7 +581,8 @@ class LineGenerator(Visitor[Line]): - - self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) - self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) -- self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) -+ if not self.mode.is_pyink: -+ self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) - self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) - self.visit_async_funcdef = self.visit_async_stmt - self.visit_decorated = self.visit_decorators -@@ -499,7 +556,9 @@ def visit_STRING( +@@ -499,7 +553,9 @@ class LineGenerator(Visitor[Line]): if self.mode.string_normalization and leaf.type == token.STRING: leaf.value = normalize_string_prefix(leaf.value) @@ -686,7 +782,17 @@ yield from self.visit_default(leaf) def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: -@@ -577,10 +628,19 @@ def transform_line( +@@ -575,7 +631,8 @@ class LineGenerator(Visitor[Line]): + + self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) + self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) +- self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) ++ if not self.mode.is_pyink: ++ self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) + self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) + self.visit_async_funcdef = self.visit_async_stmt + self.visit_decorated = self.visit_decorators +@@ -621,14 +678,23 @@ def transform_line( ll = mode.line_length sn = mode.string_normalization @@ -710,7 +816,12 @@ transformers: List[Transformer] if ( -@@ -787,7 +847,6 @@ def _first_right_hand_split( +- not line.contains_uncollapsable_type_comments() ++ not line.contains_uncollapsable_pragma_comments() + and not line.should_split_rhs + and not line.magic_trailing_comma + and ( +@@ -831,7 +897,6 @@ def _first_right_hand_split( omit: Collection[LeafID] = (), ) -> RHSResult: """Split the line into head, body, tail starting with the last bracket pair. @@ -718,7 +829,7 @@ Note: this function should not have side effects. It's relied upon by _maybe_split_omitting_optional_parens to get an opinion whether to prefer splitting on the right side of an assignment statement. -@@ -1054,7 +1113,7 @@ def bracket_split_build_line( +@@ -1098,7 +1163,7 @@ def bracket_split_build_line( result = Line(mode=original.mode, depth=original.depth) if component is _BracketSplitComponent.body: result.inside_brackets = True @@ -727,7 +838,103 @@ if leaves: no_commas = ( # Ensure a trailing comma for imports and standalone function arguments -@@ -1637,7 +1696,7 @@ def generate_trailers_to_omit(line: Line +@@ -1392,15 +1457,17 @@ def normalize_invisible_parens( # noqa: + if maybe_make_parens_invisible_in_atom( + child, + parent=node, ++ mode=mode, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(node, child, visible=False) + elif isinstance(child, Node) and node.type == syms.with_stmt: +- remove_with_parens(child, node) ++ remove_with_parens(child, node, mode=mode) + elif child.type == syms.atom: + if maybe_make_parens_invisible_in_atom( + child, + parent=node, ++ mode=mode, + ): + wrap_in_parentheses(node, child, visible=False) + elif is_one_tuple(child): +@@ -1452,7 +1519,7 @@ def _normalize_import_from(parent: Node, + parent.append_child(Leaf(token.RPAR, "")) + + +-def remove_await_parens(node: Node) -> None: ++def remove_await_parens(node: Node, mode: Mode) -> None: + if node.children[0].type == token.AWAIT and len(node.children) > 1: + if ( + node.children[1].type == syms.atom +@@ -1461,6 +1528,7 @@ def remove_await_parens(node: Node) -> N + if maybe_make_parens_invisible_in_atom( + node.children[1], + parent=node, ++ mode=mode, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(node, node.children[1], visible=False) +@@ -1529,7 +1597,7 @@ def _maybe_wrap_cms_in_parens( + node.insert_child(1, new_child) + + +-def remove_with_parens(node: Node, parent: Node) -> None: ++def remove_with_parens(node: Node, parent: Node, mode: Mode) -> None: + """Recursively hide optional parens in `with` statements.""" + # Removing all unnecessary parentheses in with statements in one pass is a tad + # complex as different variations of bracketed statements result in pretty +@@ -1551,21 +1619,23 @@ def remove_with_parens(node: Node, paren + if maybe_make_parens_invisible_in_atom( + node, + parent=parent, ++ mode=mode, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(parent, node, visible=False) + if isinstance(node.children[1], Node): +- remove_with_parens(node.children[1], node) ++ remove_with_parens(node.children[1], node, mode=mode) + elif node.type == syms.testlist_gexp: + for child in node.children: + if isinstance(child, Node): +- remove_with_parens(child, node) ++ remove_with_parens(child, node, mode=mode) + elif node.type == syms.asexpr_test and not any( + leaf.type == token.COLONEQUAL for leaf in node.leaves() + ): + if maybe_make_parens_invisible_in_atom( + node.children[0], + parent=node, ++ mode=mode, + remove_brackets_around_comma=True, + ): + wrap_in_parentheses(node, node.children[0], visible=False) +@@ -1574,6 +1644,7 @@ def remove_with_parens(node: Node, paren + def maybe_make_parens_invisible_in_atom( + node: LN, + parent: LN, ++ mode: Mode, + remove_brackets_around_comma: bool = False, + ) -> bool: + """If it's safe, make the parens in the atom `node` invisible, recursively. +@@ -1623,7 +1694,7 @@ def maybe_make_parens_invisible_in_atom( + if ( + # If the prefix of `middle` includes a type comment with + # ignore annotation, then we do not remove the parentheses +- not is_type_ignore_comment_string(middle.prefix.strip()) ++ not ink.comment_contains_pragma(middle.prefix.strip(), mode) + ): + first.value = "" + if first.prefix.strip(): +@@ -1633,6 +1704,7 @@ def maybe_make_parens_invisible_in_atom( + maybe_make_parens_invisible_in_atom( + middle, + parent=parent, ++ mode=mode, + remove_brackets_around_comma=remove_brackets_around_comma, + ) + +@@ -1691,7 +1763,7 @@ def generate_trailers_to_omit(line: Line if not line.magic_trailing_comma: yield omit @@ -736,24 +943,35 @@ opening_bracket: Optional[Leaf] = None closing_bracket: Optional[Leaf] = None inner_brackets: Set[LeafID] = set() +@@ -1776,7 +1848,7 @@ def run_transformer( + or not line.bracket_tracker.invisible + or any(bracket.value for bracket in line.bracket_tracker.invisible) + or line.contains_multiline_strings() +- or result[0].contains_uncollapsable_type_comments() ++ or result[0].contains_uncollapsable_pragma_comments() + or result[0].contains_unsplittable_type_ignore() + or is_line_short_enough(result[0], mode=mode) + # If any leaves have no parents (which _can_ occur since --- a/lines.py +++ b/lines.py -@@ -1,5 +1,7 @@ +@@ -1,3 +1,4 @@ +from enum import Enum, auto import itertools import math -+import re from dataclasses import dataclass, field - from typing import ( - Callable, -@@ -45,13 +47,28 @@ Index = int - LeafID = int +@@ -28,7 +29,7 @@ from pyink.nodes import ( + is_multiline_string, + is_one_sequence_between, + is_type_comment, +- is_type_ignore_comment, ++ is_pragma_comment, + is_with_or_async_with_stmt, + make_simple_prefix, + replace_child, +@@ -46,12 +47,24 @@ LeafID = int LN = Union[Leaf, Node] -+# This regex should contain a single capture group capturing the entire match. -+_PRAGMA_REGEX = re.compile("( *# (?:pylint|pytype):)") -+ -+ + +class Indentation(Enum): + SCOPE = auto() # Scope indentation. + CONTINUATION = auto() # Continuation/hanging indentation. @@ -765,7 +983,7 @@ + # Both pyink and black use 4 spaces for continuations. + return 4 + - ++ @dataclass class Line: """Holds leaves and comments. Can be printed with `str(line)`.""" @@ -776,7 +994,7 @@ leaves: List[Leaf] = field(default_factory=list) # keys ordered like `leaves` comments: Dict[LeafID, List[Leaf]] = field(default_factory=dict) -@@ -60,6 +77,9 @@ class Line: +@@ -60,6 +73,9 @@ class Line: should_split_rhs: bool = False magic_trailing_comma: Optional[Leaf] = None @@ -786,7 +1004,7 @@ def append( self, leaf: Leaf, preformatted: bool = False, track_bracket: bool = False ) -> None: -@@ -103,7 +123,7 @@ class Line: +@@ -108,7 +124,7 @@ class Line: or when a standalone comment is not the first leaf on the line. """ if ( @@ -795,37 +1013,40 @@ or self.bracket_tracker.any_open_for_or_lambda() ): if self.is_comment: -@@ -337,6 +357,29 @@ class Line: - +@@ -273,7 +289,7 @@ class Line: + return True return False -+ def trailing_pragma_comment_length(self) -> int: -+ if not self.leaves: -+ return 0 -+ -+ trailing_comments = self.comments.get(id(self.leaves[-1]), []) -+ if ( -+ not trailing_comments -+ and len(self.leaves) > 1 -+ and self.leaves[-1].type == token.RPAR -+ and not self.leaves[-1].value -+ ): -+ # When last leaf is an invisible paren, the trailing comment is -+ # attached to the leaf before. -+ trailing_comments = self.comments.get(id(self.leaves[-2]), []) -+ length = 0 -+ for comment in trailing_comments: -+ # str(comment) contains the whitespace preceding the `#` -+ comment_str = str(comment) -+ parts = _PRAGMA_REGEX.split(comment_str, maxsplit=1) -+ if len(parts) == 3: -+ length += len(parts[1]) + len(parts[2]) -+ return length -+ - def contains_multiline_strings(self) -> bool: - return any(is_multiline_string(leaf) for leaf in self.leaves) +- def contains_uncollapsable_type_comments(self) -> bool: ++ def contains_uncollapsable_pragma_comments(self) -> bool: + ignored_ids = set() + try: + last_leaf = self.leaves[-1] +@@ -298,11 +314,9 @@ class Line: + comment_seen = False + for leaf_id, comments in self.comments.items(): + for comment in comments: +- if is_type_comment(comment): +- if comment_seen or ( +- not is_type_ignore_comment(comment) +- and leaf_id not in ignored_ids +- ): ++ is_pragma = is_pragma_comment(comment, self.mode) ++ if is_type_comment(comment) or is_pragma: ++ if comment_seen or (not is_pragma and leaf_id not in ignored_ids): + return True + + comment_seen = True +@@ -337,7 +351,7 @@ class Line: + # line. + for node in self.leaves[-2:]: + for comment in self.comments.get(id(node), []): +- if is_type_ignore_comment(comment): ++ if is_pragma_comment(comment, self.mode): + return True -@@ -487,7 +530,7 @@ class Line: + return False +@@ -492,7 +506,7 @@ class Line: if not self: return "\n" @@ -834,7 +1055,7 @@ leaves = iter(self.leaves) first = next(leaves) res = f"{first.prefix}{indent}{first.value}" -@@ -559,7 +602,7 @@ class EmptyLineTracker: +@@ -564,7 +578,7 @@ class EmptyLineTracker: lines (two on module-level). """ form_feed = ( @@ -843,7 +1064,7 @@ and bool(current_line.leaves) and "\f\n" in current_line.leaves[0].prefix ) -@@ -604,7 +647,7 @@ class EmptyLineTracker: +@@ -609,7 +623,7 @@ class EmptyLineTracker: def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: # noqa: C901 max_allowed = 1 @@ -852,7 +1073,7 @@ max_allowed = 1 if self.mode.is_pyi else 2 if current_line.leaves: -@@ -621,7 +664,7 @@ class EmptyLineTracker: +@@ -626,7 +640,7 @@ class EmptyLineTracker: # Mutate self.previous_defs, remainder of this function should be pure previous_def = None @@ -861,7 +1082,7 @@ previous_def = self.previous_defs.pop() if current_line.is_def or current_line.is_class: self.previous_defs.append(current_line) -@@ -677,10 +720,25 @@ class EmptyLineTracker: +@@ -682,10 +696,25 @@ class EmptyLineTracker: ) if ( @@ -871,7 +1092,7 @@ + or self.previous_line.is_fmt_pass_converted( + first_leaf_matches=is_import + ) -+ ) ++ ) and not current_line.is_import + and not ( + # Should not add empty lines before a STANDALONE_COMMENT. @@ -889,7 +1110,7 @@ ): return (before or 1), 0 -@@ -697,8 +755,9 @@ class EmptyLineTracker: +@@ -702,8 +731,9 @@ class EmptyLineTracker: return 0, 1 return 0, 0 @@ -901,7 +1122,7 @@ ): if self.mode.is_pyi: return 0, 0 -@@ -707,7 +766,7 @@ class EmptyLineTracker: +@@ -712,7 +742,7 @@ class EmptyLineTracker: comment_to_add_newlines: Optional[LinesBlock] = None if ( self.previous_line.is_comment @@ -910,7 +1131,7 @@ and before == 0 ): slc = self.semantic_leading_comment -@@ -724,9 +783,9 @@ class EmptyLineTracker: +@@ -729,9 +759,9 @@ class EmptyLineTracker: if self.mode.is_pyi: if current_line.is_class or self.previous_line.is_class: @@ -922,7 +1143,7 @@ newlines = 1 elif current_line.is_stub_class and self.previous_line.is_stub_class: # No blank line between classes with an empty body -@@ -755,7 +814,11 @@ class EmptyLineTracker: +@@ -760,7 +790,11 @@ class EmptyLineTracker: newlines = 1 if current_line.depth else 2 # If a user has left no space after a dummy implementation, don't insert # new lines. This is useful for instance for @overload or Protocols. @@ -935,32 +1156,7 @@ newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block -@@ -810,9 +873,14 @@ def is_line_short_enough( # noqa: C901 - if not line_str: - line_str = line_to_string(line) - -+ if line.mode.is_pyink: -+ effective_length = str_width(line_str) - line.trailing_pragma_comment_length() -+ else: -+ effective_length = str_width(line_str) -+ - if Preview.multiline_string_handling not in mode: - return ( -- str_width(line_str) <= mode.line_length -+ effective_length <= mode.line_length - and "\n" not in line_str # multiline strings - and not line.contains_standalone_comments() - ) -@@ -821,7 +889,7 @@ def is_line_short_enough( # noqa: C901 - return False - if "\n" not in line_str: - # No multiline strings (MLS) present -- return str_width(line_str) <= mode.line_length -+ return effective_length <= mode.line_length - - first, *_, last = line_str.split("\n") - if str_width(first) > mode.line_length or str_width(last) > mode.line_length: -@@ -1024,7 +1092,7 @@ def can_omit_invisible_parens( +@@ -1031,7 +1065,7 @@ def can_omit_invisible_parens( def _can_omit_opening_paren(line: Line, *, first: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" remainder = False @@ -969,7 +1165,7 @@ _index = -1 for _index, leaf, leaf_length in line.enumerate_with_length(): if leaf.type in CLOSING_BRACKETS and leaf.opening_bracket is first: -@@ -1048,7 +1116,7 @@ def _can_omit_opening_paren(line: Line, +@@ -1055,7 +1089,7 @@ def _can_omit_opening_paren(line: Line, def _can_omit_closing_paren(line: Line, *, last: Leaf, line_length: int) -> bool: """See `can_omit_invisible_parens`.""" @@ -989,7 +1185,7 @@ from pyink.const import DEFAULT_LINE_LENGTH -@@ -198,6 +198,24 @@ class Deprecated(UserWarning): +@@ -224,20 +224,52 @@ class Deprecated(UserWarning): """Visible deprecation warning.""" @@ -1013,8 +1209,15 @@ + _MAX_CACHE_KEY_PART_LENGTH: Final = 32 ++DEFAULT_ANNOTATION_PRAGMAS = ( ++ "noqa", # flake8 ++ "pylint:", ++ "type: ignore", ++) ++ -@@ -206,12 +224,19 @@ class Mode: + @dataclass + class Mode: target_versions: Set[TargetVersion] = field(default_factory=set) line_length: int = DEFAULT_LINE_LENGTH string_normalization: bool = True @@ -1031,10 +1234,11 @@ + is_pyink: bool = False + pyink_indentation: Literal[2, 4] = 4 + pyink_ipynb_indentation: Literal[1, 2] = 1 ++ pyink_annotation_pragmas: tuple[str, ...] = DEFAULT_ANNOTATION_PRAGMAS unstable: bool = False enabled_features: Set[Preview] = field(default_factory=set) -@@ -223,6 +247,9 @@ class Mode: +@@ -249,6 +281,9 @@ class Mode: except those in UNSTABLE_FEATURES are enabled. Any features in `self.enabled_features` are also enabled. """ @@ -1044,7 +1248,7 @@ if self.unstable: return True if feature in self.enabled_features: -@@ -254,11 +281,25 @@ class Mode: +@@ -280,11 +315,26 @@ class Mode: version_str, str(self.line_length), str(int(self.string_normalization)), @@ -1058,6 +1262,7 @@ + str(int(self.is_pyink)), + str(self.pyink_indentation), + str(self.pyink_ipynb_indentation), ++ str(self.pyink_annotation_pragmas), features_and_magics, ] return ".".join(parts) @@ -1072,7 +1277,15 @@ + return Quote.DOUBLE --- a/nodes.py +++ b/nodes.py -@@ -763,9 +765,13 @@ def is_function_or_class(node: Node) -> +@@ -23,6 +23,7 @@ else: + + from mypy_extensions import mypyc_attr + ++from pyink import ink + from pyink.cache import CACHE_DIR + from pyink.mode import Mode, Preview + from pyink.strings import get_string_prefix, has_triple_quotes +@@ -798,9 +799,13 @@ def is_function_or_class(node: Node) -> return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} @@ -1088,6 +1301,22 @@ return False # If there is a comment, we want to keep it. +@@ -919,11 +924,13 @@ def is_type_comment(leaf: Leaf) -> bool: + return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") + + +-def is_type_ignore_comment(leaf: Leaf) -> bool: ++def is_pragma_comment(leaf: Leaf, mode: Mode) -> bool: + """Return True if the given leaf is a type comment with ignore annotation.""" + t = leaf.type + v = leaf.value +- return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v) ++ return t in {token.COMMENT, STANDALONE_COMMENT} and ( ++ ink.comment_contains_pragma(v, mode) ++ ) + + + def is_type_ignore_comment_string(value: str) -> bool: --- a/pyproject.toml +++ b/pyproject.toml @@ -1,52 +1,23 @@ @@ -1152,7 +1381,7 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", -@@ -70,53 +42,38 @@ dependencies = [ +@@ -71,53 +42,38 @@ dependencies = [ "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", "typing_extensions>=4.0.1; python_version < '3.11'", @@ -1177,7 +1406,7 @@ -black = "black:patched_main" -blackd = "blackd:patched_main [d]" +pyink = "pyink:patched_main" - + [project.entry-points."validate_pyproject.tool_schema"] -black = "black.schema:get_schema" +pyink = "pyink.schema:get_schema" @@ -1214,7 +1443,7 @@ [tool.hatch.build.targets.wheel] only-include = ["src"] sources = ["src"] -@@ -125,7 +80,6 @@ macos-max-compat = true +@@ -128,7 +84,6 @@ macos-max-compat = true # Option below requires `tests/optional.py` addopts = "--strict-config --strict-markers" optional-tests = [ @@ -1222,7 +1451,7 @@ "no_jupyter: run when `jupyter` extra NOT installed", ] markers = [ -@@ -149,36 +103,3 @@ filterwarnings = [ +@@ -152,36 +107,3 @@ filterwarnings = [ # https://github.com/aio-libs/aiohttp/pull/7302 "ignore:datetime.*utcfromtimestamp\\(\\) is deprecated and scheduled for removal:DeprecationWarning", ] @@ -1314,6 +1543,16 @@ return f"{prefix}{new_quote}{new_body}{new_quote}" +--- a/tests/conftest.py ++++ b/tests/conftest.py +@@ -1,6 +1,6 @@ + import pytest + +-pytest_plugins = ["tests.optional"] ++pytest_plugins = ["pyink.tests.optional"] + + PRINT_FULL_TREE: bool = False + PRINT_TREE_DIFF: bool = True --- a/tests/empty.toml +++ b/tests/empty.toml @@ -1 +1,5 @@ @@ -1324,7 +1563,7 @@ +pyink = false --- a/tests/test_black.py +++ b/tests/test_black.py -@@ -2772,6 +2772,82 @@ class TestFileCollection: +@@ -2809,6 +2809,96 @@ class TestFileCollection: stdin_filename=stdin_filename, ) @@ -1403,13 +1642,27 @@ + + diff = """-_double = "Double"\n+_double = 'Double'\n""" + assert diff in self.decode_and_normalized(result.stdout_bytes) ++ ++ @pytest.mark.parametrize( ++ "cli_args", ++ [ ++ [ ++ "--pyink-annotation-pragmas", ++ "@param", ++ "--pyink-annotation-pragmas", ++ "type: ignore", ++ ], ++ ], ++ ) ++ def test_pyink_cli_args(self, cli_args: List[str]) -> None: ++ invokeBlack([*cli_args, "-c", "0"], exit_code=0) + def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( self, ) -> None: --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py -@@ -12,6 +12,7 @@ +@@ -12,6 +12,7 @@ from click.testing import CliRunner from pyink import ( Mode, NothingChanged, @@ -1417,7 +1670,7 @@ format_cell, format_file_contents, format_file_in_place, -@@ -26,8 +26,10 @@ +@@ -27,8 +28,10 @@ pytest.importorskip("IPython", reason="I pytest.importorskip("tokenize_rt", reason="tokenize-rt is an optional dependency") JUPYTER_MODE = Mode(is_ipynb=True) @@ -1428,7 +1681,7 @@ runner = CliRunner() -@@ -209,6 +209,29 @@ +@@ -208,6 +211,29 @@ def test_cell_magic_with_custom_python_m assert result == expected_output @@ -1458,7 +1711,7 @@ def test_cell_magic_nested() -> None: src = "%%time\n%%time\n2+2" result = format_cell(src, fast=True, mode=JUPYTER_MODE) -@@ -385,6 +385,45 @@ +@@ -381,6 +407,45 @@ def test_entire_notebook_no_trailing_new assert result == expected @@ -1504,7 +1757,7 @@ def test_entire_notebook_without_changes() -> None: content = read_jupyter_notebook("jupyter", "notebook_without_changes") with pytest.raises(NothingChanged): -@@ -472,6 +472,30 @@ +@@ -432,6 +497,30 @@ def test_ipynb_diff_with_no_change() -> assert expected in result.output @@ -1537,7 +1790,7 @@ ) -> None: --- a/tests/util.py +++ b/tests/util.py -@@ -271,6 +271,11 @@ def get_flags_parser() -> argparse.Argum +@@ -264,6 +264,11 @@ def get_flags_parser() -> argparse.Argum ), ) parser.add_argument("--line-ranges", action="append") @@ -1549,7 +1802,7 @@ parser.add_argument( "--no-preview-line-length-1", default=False, -@@ -294,6 +296,9 @@ def parse_mode(flags_line: str) -> TestC +@@ -287,6 +292,9 @@ def parse_mode(flags_line: str) -> TestC is_ipynb=args.ipynb, magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, @@ -1559,7 +1812,7 @@ unstable=args.unstable, ) if args.line_ranges: -@@ -355,7 +355,8 @@ +@@ -340,7 +348,8 @@ def read_jupyter_notebook(subdir_name: s def read_jupyter_notebook_from_file(file_name: Path) -> str: with open(file_name, mode="rb") as fd: content_bytes = fd.read() @@ -1629,6 +1882,15 @@ # Fill the 'custom_splits' list with the appropriate CustomSplit objects. temp_string = S_leaf.value[len(prefix) + 1 : -1] +@@ -860,7 +871,7 @@ class StringMerger(StringTransformer, Cu + + if id(leaf) in line.comments: + num_of_inline_string_comments += 1 +- if contains_pragma_comment(line.comments[id(leaf)]): ++ if contains_pragma_comment(line.comments[id(leaf)], line.mode): + return TErr("Cannot merge strings which have pragma comments.") + + if num_of_strings < 2: @@ -1000,7 +1011,13 @@ class StringParenStripper(StringTransfor idx += 1 @@ -1644,6 +1906,15 @@ return TErr("This line has no strings wrapped in parens.") def do_transform( +@@ -1162,7 +1179,7 @@ class BaseStringSplitter(StringTransform + ) + + if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( +- line.comments[id(line.leaves[string_idx])] ++ line.comments[id(line.leaves[string_idx])], line.mode + ): + return TErr( + "Line appears to end with an inline pragma comment. Splitting the line" @@ -1204,7 +1221,7 @@ class BaseStringSplitter(StringTransform # NN: The leaf that is after N. @@ -1706,3 +1977,17 @@ inside_brackets=True, should_split_rhs=line.should_split_rhs, magic_trailing_comma=line.magic_trailing_comma, +--- a/tests/data/cases/torture.py ++++ b/tests/data/cases/torture.py +@@ -57,9 +57,9 @@ importA + class A: + def foo(self): + for _ in range(10): +- aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( ++ aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( # pylint: disable=no-member + xxxxxxxxxxxx +- ) # pylint: disable=no-member ++ ) + + + def test(self, othr): diff --git a/src/pyink/__init__.py b/src/pyink/__init__.py index 0aab91e7e11..4bbab2625c1 100644 --- a/src/pyink/__init__.py +++ b/src/pyink/__init__.py @@ -69,7 +69,13 @@ from pyink.lines import EmptyLineTracker, LinesBlock from pyink.mode import FUTURE_FLAG_TO_FEATURE, Feature, VERSION_TO_FEATURES from pyink.mode import Mode as Mode # re-exported -from pyink.mode import Preview, QuoteStyle, TargetVersion, supports_feature +from pyink.mode import ( + DEFAULT_ANNOTATION_PRAGMAS, + Preview, + QuoteStyle, + TargetVersion, + supports_feature, +) from pyink.nodes import STARS, is_number_token, is_simple_decorator_expression, syms from pyink.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from pyink.parsing import ( # noqa F401 @@ -368,6 +374,18 @@ def validate_regex( " notebook." ), ) +@click.option( + "--pyink-annotation-pragmas", + type=str, + multiple=True, + help=( + "Pyink won't split too long lines if they contain a comment with any of" + " the given annotation pragmas because it doesn't know to which part of" + " the line the annotation applies. The default annotation pragmas are:" + f" {', '.join(DEFAULT_ANNOTATION_PRAGMAS)}." + ), + default=[], +) @click.option( "--pyink-use-majority-quotes", is_flag=True, @@ -580,6 +598,7 @@ def main( # noqa: C901 pyink: bool, pyink_indentation: str, pyink_ipynb_indentation: str, + pyink_annotation_pragmas: List[str], pyink_use_majority_quotes: bool, quiet: bool, verbose: bool, @@ -681,6 +700,9 @@ def main( # noqa: C901 is_pyink=pyink, pyink_indentation=int(pyink_indentation), pyink_ipynb_indentation=int(pyink_ipynb_indentation), + pyink_annotation_pragmas=( + tuple(pyink_annotation_pragmas) or DEFAULT_ANNOTATION_PRAGMAS + ), quote_style=( QuoteStyle.MAJORITY if pyink_use_majority_quotes else QuoteStyle.DOUBLE ), diff --git a/src/pyink/comments.py b/src/pyink/comments.py index 046a5623c89..f89f374db4b 100644 --- a/src/pyink/comments.py +++ b/src/pyink/comments.py @@ -3,6 +3,7 @@ from functools import lru_cache from typing import Collection, Final, Iterator, List, Optional, Tuple, Union +from pyink import ink from pyink.mode import Mode, Preview from pyink.nodes import ( CLOSING_BRACKETS, @@ -376,7 +377,7 @@ def children_contains_fmt_on(container: LN) -> bool: return False -def contains_pragma_comment(comment_list: List[Leaf]) -> bool: +def contains_pragma_comment(comment_list: List[Leaf], mode: Mode) -> bool: """ Returns: True iff one of the comments in @comment_list is a pragma used by one @@ -384,7 +385,7 @@ def contains_pragma_comment(comment_list: List[Leaf]) -> bool: pylint). """ for comment in comment_list: - if comment.value.startswith(("# type:", "# noqa", "# pylint:")): + if ink.comment_contains_pragma(comment.value, mode): return True return False diff --git a/src/pyink/ink.py b/src/pyink/ink.py index 2cb219972e6..71cd6ad45e9 100644 --- a/src/pyink/ink.py +++ b/src/pyink/ink.py @@ -17,7 +17,7 @@ from blib2to3.pgen2.token import ASYNC, FSTRING_START, NEWLINE, STRING from blib2to3.pytree import type_repr -from pyink.mode import Quote +from pyink.mode import Mode, Quote from pyink.nodes import LN, Leaf, Node, STANDALONE_COMMENT, Visitor, syms from pyink.strings import STRING_PREFIX_CHARS @@ -63,6 +63,28 @@ def majority_quote(node: LN) -> Quote: return Quote.DOUBLE +def comment_contains_pragma(comment: str, mode: Mode) -> bool: + """Check if the given string contains one of the pragma forms. + + A pragma form can appear at the beginning of a comment: + # pytype: disable=attribute-error + or somewhere in the middle: + # some comment # type: ignore # another comment + or the comments can even be separated by a semicolon: + # some comment; noqa: E111; another comment + + Args: + comment: The comment to check. + mode: The mode that defines which pragma forms to check for. + + Returns: + True if the comment contains one of the pragma forms. + """ + joined_pragma_expression = "|".join(mode.pyink_annotation_pragmas) + pragma_regex = re.compile(rf"([#|;] ?(?:{joined_pragma_expression}))") + return pragma_regex.search(comment) is not None + + def get_code_start(src: str) -> str: """Provides the first line where the code starts. diff --git a/src/pyink/linegen.py b/src/pyink/linegen.py index 6deab0c302d..021a1389839 100644 --- a/src/pyink/linegen.py +++ b/src/pyink/linegen.py @@ -14,6 +14,7 @@ else: from typing import Final, Literal +from pyink import ink from pyink.brackets import ( COMMA_PRIORITY, DOT_PRIORITY, @@ -61,7 +62,6 @@ is_stub_body, is_stub_suite, is_tuple_containing_walrus, - is_type_ignore_comment_string, is_vararg, is_walrus_assignment, is_yield, @@ -272,6 +272,7 @@ def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: maybe_make_parens_invisible_in_atom( child, parent=node, + mode=self.mode, remove_brackets_around_comma=False, ) else: @@ -292,6 +293,7 @@ def visit_funcdef(self, node: Node) -> Iterator[Line]: if maybe_make_parens_invisible_in_atom( child, parent=node, + mode=self.mode, remove_brackets_around_comma=False, ): wrap_in_parentheses(node, child, visible=False) @@ -315,8 +317,10 @@ def visit_match_case(self, node: Node) -> Iterator[Line]: def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" if ( - self.mode.is_pyi or not self.mode.is_pyink - ) and is_stub_suite(node, self.mode): + self.mode.is_pyi + or not self.mode.is_pyink + and is_stub_suite(node, self.mode) + ): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -395,7 +399,7 @@ def visit_power(self, node: Node) -> Iterator[Line]: ): wrap_in_parentheses(node, leaf) - remove_await_parens(node) + remove_await_parens(node, mode=self.mode) yield from self.visit_default(node) @@ -442,7 +446,9 @@ def foo(a: int, b: float = 7): ... def foo(a: (int), b: (float) = 7): ... """ assert len(node.children) == 3 - if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + if maybe_make_parens_invisible_in_atom( + node.children[2], parent=node, mode=self.mode + ): wrap_in_parentheses(node, node.children[2], visible=False) yield from self.visit_default(node) @@ -688,7 +694,7 @@ def transform_line( transformers: List[Transformer] if ( - not line.contains_uncollapsable_type_comments() + not line.contains_uncollapsable_pragma_comments() and not line.should_split_rhs and not line.magic_trailing_comma and ( @@ -1451,15 +1457,17 @@ def normalize_invisible_parens( # noqa: C901 if maybe_make_parens_invisible_in_atom( child, parent=node, + mode=mode, remove_brackets_around_comma=True, ): wrap_in_parentheses(node, child, visible=False) elif isinstance(child, Node) and node.type == syms.with_stmt: - remove_with_parens(child, node) + remove_with_parens(child, node, mode=mode) elif child.type == syms.atom: if maybe_make_parens_invisible_in_atom( child, parent=node, + mode=mode, ): wrap_in_parentheses(node, child, visible=False) elif is_one_tuple(child): @@ -1511,7 +1519,7 @@ def _normalize_import_from(parent: Node, child: LN, index: int) -> None: parent.append_child(Leaf(token.RPAR, "")) -def remove_await_parens(node: Node) -> None: +def remove_await_parens(node: Node, mode: Mode) -> None: if node.children[0].type == token.AWAIT and len(node.children) > 1: if ( node.children[1].type == syms.atom @@ -1520,6 +1528,7 @@ def remove_await_parens(node: Node) -> None: if maybe_make_parens_invisible_in_atom( node.children[1], parent=node, + mode=mode, remove_brackets_around_comma=True, ): wrap_in_parentheses(node, node.children[1], visible=False) @@ -1588,7 +1597,7 @@ def _maybe_wrap_cms_in_parens( node.insert_child(1, new_child) -def remove_with_parens(node: Node, parent: Node) -> None: +def remove_with_parens(node: Node, parent: Node, mode: Mode) -> None: """Recursively hide optional parens in `with` statements.""" # Removing all unnecessary parentheses in with statements in one pass is a tad # complex as different variations of bracketed statements result in pretty @@ -1610,21 +1619,23 @@ def remove_with_parens(node: Node, parent: Node) -> None: if maybe_make_parens_invisible_in_atom( node, parent=parent, + mode=mode, remove_brackets_around_comma=True, ): wrap_in_parentheses(parent, node, visible=False) if isinstance(node.children[1], Node): - remove_with_parens(node.children[1], node) + remove_with_parens(node.children[1], node, mode=mode) elif node.type == syms.testlist_gexp: for child in node.children: if isinstance(child, Node): - remove_with_parens(child, node) + remove_with_parens(child, node, mode=mode) elif node.type == syms.asexpr_test and not any( leaf.type == token.COLONEQUAL for leaf in node.leaves() ): if maybe_make_parens_invisible_in_atom( node.children[0], parent=node, + mode=mode, remove_brackets_around_comma=True, ): wrap_in_parentheses(node, node.children[0], visible=False) @@ -1633,6 +1644,7 @@ def remove_with_parens(node: Node, parent: Node) -> None: def maybe_make_parens_invisible_in_atom( node: LN, parent: LN, + mode: Mode, remove_brackets_around_comma: bool = False, ) -> bool: """If it's safe, make the parens in the atom `node` invisible, recursively. @@ -1682,7 +1694,7 @@ def maybe_make_parens_invisible_in_atom( if ( # If the prefix of `middle` includes a type comment with # ignore annotation, then we do not remove the parentheses - not is_type_ignore_comment_string(middle.prefix.strip()) + not ink.comment_contains_pragma(middle.prefix.strip(), mode) ): first.value = "" if first.prefix.strip(): @@ -1692,6 +1704,7 @@ def maybe_make_parens_invisible_in_atom( maybe_make_parens_invisible_in_atom( middle, parent=parent, + mode=mode, remove_brackets_around_comma=remove_brackets_around_comma, ) @@ -1835,7 +1848,7 @@ def run_transformer( or not line.bracket_tracker.invisible or any(bracket.value for bracket in line.bracket_tracker.invisible) or line.contains_multiline_strings() - or result[0].contains_uncollapsable_type_comments() + or result[0].contains_uncollapsable_pragma_comments() or result[0].contains_unsplittable_type_ignore() or is_line_short_enough(result[0], mode=mode) # If any leaves have no parents (which _can_ occur since diff --git a/src/pyink/lines.py b/src/pyink/lines.py index 37c8afa8438..225ba37445b 100644 --- a/src/pyink/lines.py +++ b/src/pyink/lines.py @@ -1,7 +1,6 @@ from enum import Enum, auto import itertools import math -import re from dataclasses import dataclass, field from typing import ( Callable, @@ -30,7 +29,7 @@ is_multiline_string, is_one_sequence_between, is_type_comment, - is_type_ignore_comment, + is_pragma_comment, is_with_or_async_with_stmt, make_simple_prefix, replace_child, @@ -47,9 +46,6 @@ LeafID = int LN = Union[Leaf, Node] -# This regex should contain a single capture group capturing the entire match. -_PRAGMA_REGEX = re.compile("( *# (?:pylint|pytype):)") - class Indentation(Enum): SCOPE = auto() # Scope indentation. @@ -293,7 +289,7 @@ def contains_implicit_multiline_string_with_comments(self) -> bool: return True return False - def contains_uncollapsable_type_comments(self) -> bool: + def contains_uncollapsable_pragma_comments(self) -> bool: ignored_ids = set() try: last_leaf = self.leaves[-1] @@ -318,11 +314,9 @@ def contains_uncollapsable_type_comments(self) -> bool: comment_seen = False for leaf_id, comments in self.comments.items(): for comment in comments: - if is_type_comment(comment): - if comment_seen or ( - not is_type_ignore_comment(comment) - and leaf_id not in ignored_ids - ): + is_pragma = is_pragma_comment(comment, self.mode) + if is_type_comment(comment) or is_pragma: + if comment_seen or (not is_pragma and leaf_id not in ignored_ids): return True comment_seen = True @@ -357,34 +351,11 @@ def contains_unsplittable_type_ignore(self) -> bool: # line. for node in self.leaves[-2:]: for comment in self.comments.get(id(node), []): - if is_type_ignore_comment(comment): + if is_pragma_comment(comment, self.mode): return True return False - def trailing_pragma_comment_length(self) -> int: - if not self.leaves: - return 0 - - trailing_comments = self.comments.get(id(self.leaves[-1]), []) - if ( - not trailing_comments - and len(self.leaves) > 1 - and self.leaves[-1].type == token.RPAR - and not self.leaves[-1].value - ): - # When last leaf is an invisible paren, the trailing comment is - # attached to the leaf before. - trailing_comments = self.comments.get(id(self.leaves[-2]), []) - length = 0 - for comment in trailing_comments: - # str(comment) contains the whitespace preceding the `#` - comment_str = str(comment) - parts = _PRAGMA_REGEX.split(comment_str, maxsplit=1) - if len(parts) == 3: - length += len(parts[1]) + len(parts[2]) - return length - def contains_multiline_strings(self) -> bool: return any(is_multiline_string(leaf) for leaf in self.leaves) @@ -730,7 +701,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: # noqa: C9 or self.previous_line.is_fmt_pass_converted( first_leaf_matches=is_import ) - ) + ) and not current_line.is_import and not ( # Should not add empty lines before a STANDALONE_COMMENT. @@ -878,14 +849,9 @@ def is_line_short_enough( # noqa: C901 if not line_str: line_str = line_to_string(line) - if line.mode.is_pyink: - effective_length = str_width(line_str) - line.trailing_pragma_comment_length() - else: - effective_length = str_width(line_str) - if Preview.multiline_string_handling not in mode: return ( - effective_length <= mode.line_length + str_width(line_str) <= mode.line_length and "\n" not in line_str # multiline strings and not line.contains_standalone_comments() ) @@ -894,7 +860,7 @@ def is_line_short_enough( # noqa: C901 return False if "\n" not in line_str: # No multiline strings (MLS) present - return effective_length <= mode.line_length + return str_width(line_str) <= mode.line_length first, *_, last = line_str.split("\n") if str_width(first) > mode.line_length or str_width(last) > mode.line_length: diff --git a/src/pyink/mode.py b/src/pyink/mode.py index 9a4c230bf42..464f67d1a31 100644 --- a/src/pyink/mode.py +++ b/src/pyink/mode.py @@ -244,6 +244,12 @@ class QuoteStyle(Enum): _MAX_CACHE_KEY_PART_LENGTH: Final = 32 +DEFAULT_ANNOTATION_PRAGMAS = ( + "noqa", # flake8 + "pylint:", + "type: ignore", +) + @dataclass class Mode: @@ -263,6 +269,7 @@ class Mode: is_pyink: bool = False pyink_indentation: Literal[2, 4] = 4 pyink_ipynb_indentation: Literal[1, 2] = 1 + pyink_annotation_pragmas: tuple[str, ...] = DEFAULT_ANNOTATION_PRAGMAS unstable: bool = False enabled_features: Set[Preview] = field(default_factory=set) @@ -318,6 +325,7 @@ def get_cache_key(self) -> str: str(int(self.is_pyink)), str(self.pyink_indentation), str(self.pyink_ipynb_indentation), + str(self.pyink_annotation_pragmas), features_and_magics, ] return ".".join(parts) diff --git a/src/pyink/nodes.py b/src/pyink/nodes.py index 35389380d5b..e9148c1cce6 100644 --- a/src/pyink/nodes.py +++ b/src/pyink/nodes.py @@ -23,6 +23,7 @@ from mypy_extensions import mypyc_attr +from pyink import ink from pyink.cache import CACHE_DIR from pyink.mode import Mode, Preview from pyink.strings import get_string_prefix, has_triple_quotes @@ -923,11 +924,13 @@ def is_type_comment(leaf: Leaf) -> bool: return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") -def is_type_ignore_comment(leaf: Leaf) -> bool: +def is_pragma_comment(leaf: Leaf, mode: Mode) -> bool: """Return True if the given leaf is a type comment with ignore annotation.""" t = leaf.type v = leaf.value - return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v) + return t in {token.COMMENT, STANDALONE_COMMENT} and ( + ink.comment_contains_pragma(v, mode) + ) def is_type_ignore_comment_string(value: str) -> bool: diff --git a/src/pyink/trans.py b/src/pyink/trans.py index f32418ba6c7..a063b9ddf75 100644 --- a/src/pyink/trans.py +++ b/src/pyink/trans.py @@ -871,7 +871,7 @@ def _validate_msg(line: Line, string_idx: int) -> TResult[None]: if id(leaf) in line.comments: num_of_inline_string_comments += 1 - if contains_pragma_comment(line.comments[id(leaf)]): + if contains_pragma_comment(line.comments[id(leaf)], line.mode): return TErr("Cannot merge strings which have pragma comments.") if num_of_strings < 2: @@ -1179,7 +1179,7 @@ def _validate(self, line: Line, string_idx: int) -> TResult[None]: ) if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment( - line.comments[id(line.leaves[string_idx])] + line.comments[id(line.leaves[string_idx])], line.mode ): return TErr( "Line appears to end with an inline pragma comment. Splitting the line" diff --git a/tests/data/cases/torture.py b/tests/data/cases/torture.py index 2a194759a82..037719a8669 100644 --- a/tests/data/cases/torture.py +++ b/tests/data/cases/torture.py @@ -57,9 +57,9 @@ def test(self, othr): class A: def foo(self): for _ in range(10): - aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( + aaaaaaaaaaaaaaaaaaa = bbbbbbbbbbbbbbb.cccccccccc( # pylint: disable=no-member xxxxxxxxxxxx - ) # pylint: disable=no-member + ) def test(self, othr): diff --git a/tests/data/pyink_configs/overrides.toml b/tests/data/pyink_configs/overrides.toml index 10fb3be190b..067f882d16b 100644 --- a/tests/data/pyink_configs/overrides.toml +++ b/tests/data/pyink_configs/overrides.toml @@ -1,3 +1,4 @@ [tool.pyink] pyink-indentation = 2 pyink-ipynb-indentation = 2 +pyink-annotation-pragmas = ["@param", "type: ignore"] diff --git a/tests/test_black.py b/tests/test_black.py index 7a00fbd31fe..f8f52a19293 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2885,6 +2885,20 @@ def test_pyink_use_majority_quotes(self) -> None: diff = """-_double = "Double"\n+_double = 'Double'\n""" assert diff in self.decode_and_normalized(result.stdout_bytes) + @pytest.mark.parametrize( + "cli_args", + [ + [ + "--pyink-annotation-pragmas", + "@param", + "--pyink-annotation-pragmas", + "type: ignore", + ], + ], + ) + def test_pyink_cli_args(self, cli_args: List[str]) -> None: + invokeBlack([*cli_args, "-c", "0"], exit_code=0) + def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( self, ) -> None: