From eac5767cadbaffe5b073db109ffc843aa062c37e Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Sat, 7 Sep 2024 15:25:26 +0300 Subject: [PATCH 1/5] chore(poetry): bump dependencies --- poetry.lock | 59 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index d69ed0a..12dc967 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "black" @@ -52,13 +52,13 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -78,13 +78,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [package.extras] @@ -174,16 +174,17 @@ test = ["pytest"] [[package]] name = "flake8-quotes" -version = "3.3.2" +version = "3.4.0" description = "Flake8 lint for quotes." optional = false python-versions = "*" files = [ - {file = "flake8-quotes-3.3.2.tar.gz", hash = "sha256:6e26892b632dacba517bf27219c459a8396dcfac0f5e8204904c5a4ba9b480e1"}, + {file = "flake8-quotes-3.4.0.tar.gz", hash = "sha256:aad8492fb710a2d3eabe68c5f86a1428de650c8484127e14c43d0504ba30276c"}, ] [package.dependencies] flake8 = "*" +setuptools = "*" [[package]] name = "importlib-metadata" @@ -256,13 +257,13 @@ files = [ [[package]] name = "packaging" -version = "23.1" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -278,13 +279,13 @@ files = [ [[package]] name = "platformdirs" -version = "3.10.0" +version = "4.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, - {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, + {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, + {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, ] [package.dependencies] @@ -336,13 +337,13 @@ files = [ [[package]] name = "pyparsing" -version = "3.1.1" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, - {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] @@ -350,13 +351,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -371,6 +372,22 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "setuptools" +version = "68.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, + {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "tomli" version = "2.0.1" From 5ed8b033f87917a3ca01aeaba0b93e927fe1a022 Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Sat, 7 Sep 2024 15:29:11 +0300 Subject: [PATCH 2/5] fix(jira2markdown): process windows line breaks as a Unix line breaks --- jira2markdown/parser.py | 1 + tests/markup/test_lists.py | 6 ++++++ tests/markup/test_tables.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/jira2markdown/parser.py b/jira2markdown/parser.py index a2a9325..8cc2a53 100644 --- a/jira2markdown/parser.py +++ b/jira2markdown/parser.py @@ -17,4 +17,5 @@ def convert(text: str, usernames: Optional[dict] = None, elements: Optional[Mark inline_markup << elements.expr(inline_markup, markup, usernames, filter(lambda e: e.is_inline_element, elements)) markup << elements.expr(inline_markup, markup, usernames, elements) + text = text.replace("\r\n", "\n") return markup.transform_string(text) diff --git a/tests/markup/test_lists.py b/tests/markup/test_lists.py index 2d2c5da..7a79ff6 100644 --- a/tests/markup/test_lists.py +++ b/tests/markup/test_lists.py @@ -314,3 +314,9 @@ def test_list_indent(self): 1. Three """ ) + + def test_windows_line_breaks(self): + assert ( + convert("Line before list\r\n * Bulleted item 1\r\n * Bulleted item 2\r\n\r\nLine after list") + == "Line before list\n- Bulleted item 1\n- Bulleted item 2\n\nLine after list" + ) diff --git a/tests/markup/test_tables.py b/tests/markup/test_tables.py index 22e8a01..331486b 100644 --- a/tests/markup/test_tables.py +++ b/tests/markup/test_tables.py @@ -139,3 +139,19 @@ def test_empty_start_lines(self): assert convert(" \n|header") == " \n|header|\n|---|\n" assert convert(" \n \t \n|header") == " \n \t \n|header|\n|---|\n" assert convert(" \n text \n|header") == " \n text \n\n|header|\n|---|\n" + + def test_windows_line_breaks(self): + assert convert( + "text before table:\r\n\r\n" + "||header 1||header 2||header 3||\r\n" + "|cell 1-1|cell 1-2|cell 1-3|\r\n" + "|cell 2-1|cell 2-2|cell 2-3|\r\n\r\n" + "text after table" + ) == ( + "text before table:\n\n" + "|header 1|header 2|header 3|\n" + "|---|---|---|\n" + "|cell 1-1|cell 1-2|cell 1-3|\n" + "|cell 2-1|cell 2-2|cell 2-3|\n\n" + "text after table" + ) From 6252a4714edcc4c190c8875accaac20d4a605120 Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Wed, 2 Oct 2024 18:39:50 +0300 Subject: [PATCH 3/5] fixup! fix(jira2markdown): process windows line breaks as a Unix line breaks --- jira2markdown/markup/headings.py | 5 +++-- jira2markdown/markup/lists.py | 4 ++-- jira2markdown/markup/tables.py | 6 +++--- jira2markdown/markup/text_breaks.py | 5 +++-- jira2markdown/markup/text_effects.py | 4 ++-- jira2markdown/parser.py | 1 - jira2markdown/tokens.py | 20 ++++++++++++++++++++ 7 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 jira2markdown/tokens.py diff --git a/jira2markdown/markup/headings.py b/jira2markdown/markup/headings.py index 43d2ca2..e6f6a95 100644 --- a/jira2markdown/markup/headings.py +++ b/jira2markdown/markup/headings.py @@ -1,6 +1,7 @@ -from pyparsing import Combine, LineEnd, LineStart, Optional, ParserElement, ParseResults, SkipTo, StringEnd, White, Word +from pyparsing import Combine, LineStart, Optional, ParserElement, ParseResults, SkipTo, StringEnd, White, Word from jira2markdown.markup.base import AbstractMarkup +from jira2markdown.tokens import UniversalLineEnd class Headings(AbstractMarkup): @@ -17,6 +18,6 @@ def expr(self) -> ParserElement: + Combine( Word("h", "123456", exact=2).set_results_name("level") + ". " - + SkipTo(LineEnd() | StringEnd()).set_results_name("text"), + + SkipTo(UniversalLineEnd() | StringEnd()).set_results_name("text"), ).set_parse_action(self.action) ) diff --git a/jira2markdown/markup/lists.py b/jira2markdown/markup/lists.py index cf37c01..3bb6127 100644 --- a/jira2markdown/markup/lists.py +++ b/jira2markdown/markup/lists.py @@ -3,7 +3,6 @@ from pyparsing import ( Char, Combine, - LineEnd, LineStart, Literal, MatchFirst, @@ -20,6 +19,7 @@ from jira2markdown.markup.advanced import Panel from jira2markdown.markup.base import AbstractMarkup from jira2markdown.markup.text_effects import BlockQuote, Color +from jira2markdown.tokens import UniversalLineEnd class ListIndentState: @@ -90,7 +90,7 @@ def action(self, tokens: ParseResults) -> str: @property def expr(self) -> ParserElement: WHITESPACE = Regex(r"[ \t]+", flags=re.UNICODE) - EOL = LineEnd() + EOL = UniversalLineEnd() LIST_BREAK = EOL + Optional(WHITESPACE) + EOL | StringEnd() IGNORE = BlockQuote(**self.init_kwargs).expr | Panel(**self.init_kwargs).expr | Color(**self.init_kwargs).expr ROW = ( diff --git a/jira2markdown/markup/tables.py b/jira2markdown/markup/tables.py index 7785d13..70f93e7 100644 --- a/jira2markdown/markup/tables.py +++ b/jira2markdown/markup/tables.py @@ -2,7 +2,6 @@ from pyparsing import ( Group, - LineEnd, LineStart, Literal, OneOrMore, @@ -19,6 +18,7 @@ from jira2markdown.markup.base import AbstractMarkup from jira2markdown.markup.images import Image from jira2markdown.markup.links import Link, MailTo, Mention +from jira2markdown.tokens import UniversalLineEnd class Table(AbstractMarkup): @@ -52,7 +52,7 @@ def action(self, tokens: ParseResults) -> str: @property def expr(self) -> ParserElement: - NL = LineEnd().suppress() + NL = UniversalLineEnd().suppress() SEP = (Literal("||") | Literal("|")).suppress() ROW_BREAK = NL + SEP | NL + NL | StringEnd() IGNORE = ( @@ -67,7 +67,7 @@ def expr(self) -> ParserElement: stop_on=ROW_BREAK | NL + ~SEP, ) - EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + LineEnd() + EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + UniversalLineEnd() return ( (StringStart() ^ Optional(EMPTY_LINE, default="\n")) + OneOrMore(LineStart() + Group(ROW) + NL).set_parse_action(self.action) diff --git a/jira2markdown/markup/text_breaks.py b/jira2markdown/markup/text_breaks.py index b582175..0cc9555 100644 --- a/jira2markdown/markup/text_breaks.py +++ b/jira2markdown/markup/text_breaks.py @@ -1,6 +1,7 @@ -from pyparsing import Keyword, LineEnd, LineStart, Optional, ParserElement, White, WordEnd, WordStart, replaceWith +from pyparsing import Keyword, LineStart, Optional, ParserElement, White, WordEnd, WordStart, replaceWith from jira2markdown.markup.base import AbstractMarkup +from jira2markdown.tokens import UniversalLineEnd class LineBreak(AbstractMarkup): @@ -33,5 +34,5 @@ def expr(self) -> ParserElement: + (Optional(White()) + Keyword("----", ident_chars="-") + Optional(White())).set_parse_action( replaceWith("\n----"), ) - + LineEnd() + + UniversalLineEnd() ) diff --git a/jira2markdown/markup/text_effects.py b/jira2markdown/markup/text_effects.py index b712730..b2680d3 100644 --- a/jira2markdown/markup/text_effects.py +++ b/jira2markdown/markup/text_effects.py @@ -5,7 +5,6 @@ Char, Combine, FollowedBy, - LineEnd, LineStart, Literal, OneOrMore, @@ -30,6 +29,7 @@ from jira2markdown.markup.base import AbstractMarkup from jira2markdown.markup.images import Image from jira2markdown.markup.links import Attachment, Link, Mention +from jira2markdown.tokens import UniversalLineEnd class QuotedElement(AbstractMarkup): @@ -159,7 +159,7 @@ def action(self, tokens: ParseResults) -> str: @property def expr(self) -> ParserElement: - NL = LineEnd() + NL = UniversalLineEnd() EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + NL ROW = ( LineStart() diff --git a/jira2markdown/parser.py b/jira2markdown/parser.py index 8cc2a53..a2a9325 100644 --- a/jira2markdown/parser.py +++ b/jira2markdown/parser.py @@ -17,5 +17,4 @@ def convert(text: str, usernames: Optional[dict] = None, elements: Optional[Mark inline_markup << elements.expr(inline_markup, markup, usernames, filter(lambda e: e.is_inline_element, elements)) markup << elements.expr(inline_markup, markup, usernames, elements) - text = text.replace("\r\n", "\n") return markup.transform_string(text) diff --git a/jira2markdown/tokens.py b/jira2markdown/tokens.py new file mode 100644 index 0000000..b39f932 --- /dev/null +++ b/jira2markdown/tokens.py @@ -0,0 +1,20 @@ +from pyparsing import LineEnd, ParseException, ParseImplReturnType + + +class UniversalLineEnd(LineEnd): + def __init__(self): + super().__init__() + self.whiteChars.discard("\r\n") + + def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: + if loc < len(instring): + if instring.startswith("\r\n", loc): + return loc + 2, "\n" + elif instring[loc] in ("\n", "\r"): + return loc + 1, "\n" + else: + raise ParseException(instring, loc, self.errmsg, self) + elif loc == len(instring): + return loc + 1, [] + else: + raise ParseException(instring, loc, self.errmsg, self) From 863ec8023e4c88b92c8e7a4e337acc2d8ee234d7 Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Mon, 7 Oct 2024 17:56:44 +0300 Subject: [PATCH 4/5] fixup! fix(jira2markdown): process windows line breaks as a Unix line breaks --- jira2markdown/markup/advanced.py | 6 ++++-- jira2markdown/markup/lists.py | 13 +++++-------- jira2markdown/markup/tables.py | 22 ++++++++++++++-------- jira2markdown/markup/text_effects.py | 8 +++++--- jira2markdown/tokens.py | 2 +- tests/markup/test_headings.py | 6 ++++++ tests/markup/test_lists.py | 2 +- tests/markup/test_mixed_content.py | 4 ++-- tests/markup/test_tables.py | 10 +++++----- tests/markup/test_text_effects.py | 15 +++++++++++++++ 10 files changed, 58 insertions(+), 30 deletions(-) diff --git a/jira2markdown/markup/advanced.py b/jira2markdown/markup/advanced.py index 951fd91..8401c81 100644 --- a/jira2markdown/markup/advanced.py +++ b/jira2markdown/markup/advanced.py @@ -58,8 +58,10 @@ def action(self, tokens: ParseResults) -> str: else: prefix = "" - text = self.markup.transform_string("\n".join([line.lstrip() for line in tokens.text.strip().splitlines()])) - return prefix + "\n".join([f"> {line}" for line in text.splitlines()]) + text = self.markup.transform_string( + "".join([line.lstrip() for line in tokens.text.strip().splitlines(keepends=True)]) + ) + return prefix + "".join([f"> {line}" for line in text.splitlines(keepends=True)]) @property def expr(self) -> ParserElement: diff --git a/jira2markdown/markup/lists.py b/jira2markdown/markup/lists.py index 3bb6127..7f4acd9 100644 --- a/jira2markdown/markup/lists.py +++ b/jira2markdown/markup/lists.py @@ -63,7 +63,7 @@ def __init__(self, nested_token: str, nested_indent: int, tokens: str, indent: i self.indent_state = ListIndentState() def action(self, tokens: ParseResults) -> str: - result = [] + result = "" for line in tokens: bullets, text = line.split(" ", maxsplit=1) @@ -77,15 +77,12 @@ def action(self, tokens: ParseResults) -> str: line_padding = " " * count item_padding = " " * (count - self.indent) + self.bullet + " " - text = self.markup.transform_string(text).splitlines() or [""] + text = self.markup.transform_string(text).splitlines(keepends=True) or [""] - result.append( - "\n".join([item_padding + line if i == 0 else line_padding + line for i, line in enumerate(text)]), - ) + result += "".join([item_padding + line if i == 0 else line_padding + line for i, line in enumerate(text)]) self.indent_state.reset() - text_end = "\n" if (tokens[-1][-1] == "\n") else "" - return "\n".join(result) + text_end + return result @property def expr(self) -> ParserElement: @@ -118,7 +115,7 @@ def action(self, tokens: ParseResults) -> str: result = super().action(tokens) first_line = (result.splitlines() or [""])[0].strip() - # Text with dashed below it turns into a heading. To prevent this + # Text with dashes below it turns into a heading. To prevent this # add a line break before an empty list. if first_line == "-": return "\n" + result diff --git a/jira2markdown/markup/tables.py b/jira2markdown/markup/tables.py index 70f93e7..c5983be 100644 --- a/jira2markdown/markup/tables.py +++ b/jira2markdown/markup/tables.py @@ -25,8 +25,10 @@ class Table(AbstractMarkup): is_inline_element = False def action(self, tokens: ParseResults) -> str: - lines = [line for line in tokens if len(line) > 0] - max_columns_count = max(len(row) for row in tokens) + eol = tokens[0].eol or "\n" + stripped_tokens = [row[:-1] if row[-1] == eol else row for row in tokens] + max_columns_count = max(len(row) for row in stripped_tokens) + lines = [row for row in stripped_tokens if len(row) > 0] # Converts multiline text to one line, # because markdown doesn't support multiline text in table cells @@ -34,7 +36,7 @@ def action(self, tokens: ParseResults) -> str: "|" + "|".join( map( - lambda cell: cell.replace("\n", "
"), + lambda cell: cell.replace(eol, "
"), map(self.markup.transform_string, row), ), ) @@ -48,7 +50,7 @@ def action(self, tokens: ParseResults) -> str: # Insert header delimiter after the first row output.insert(1, "|" + "---|" * max(max_columns_count, 1)) - return "\n".join(output) + "\n" + return eol.join(output) + eol @property def expr(self) -> ParserElement: @@ -62,14 +64,18 @@ def expr(self) -> ParserElement: | Mention(**self.init_kwargs).expr ) - ROW = SEP + ZeroOrMore( - SkipTo(SEP | ROW_BREAK, ignore=IGNORE) + Optional(SEP), - stop_on=ROW_BREAK | NL + ~SEP, + ROW = ( + SEP + + ZeroOrMore( + SkipTo(SEP | ROW_BREAK, ignore=IGNORE) + Optional(SEP), + stop_on=ROW_BREAK | NL + ~SEP, + ) + + UniversalLineEnd().set_results_name("eol") ) EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + UniversalLineEnd() return ( (StringStart() ^ Optional(EMPTY_LINE, default="\n")) - + OneOrMore(LineStart() + Group(ROW) + NL).set_parse_action(self.action) + + OneOrMore(LineStart() + Group(ROW)).set_parse_action(self.action) + (StringEnd() | Optional(EMPTY_LINE, default="\n")) ) diff --git a/jira2markdown/markup/text_effects.py b/jira2markdown/markup/text_effects.py index b2680d3..77af5ec 100644 --- a/jira2markdown/markup/text_effects.py +++ b/jira2markdown/markup/text_effects.py @@ -53,7 +53,7 @@ def expr(self) -> ParserElement: ELEMENT = Combine( TOKEN + (~White() & ~Char(self.TOKEN)) - + SkipTo(TOKEN, ignore=IGNORE, fail_on="\n") + + SkipTo(TOKEN, ignore=IGNORE, fail_on=UniversalLineEnd()) + TOKEN + FollowedBy(NON_ALPHANUMS | StringEnd()), ) @@ -173,8 +173,10 @@ def expr(self) -> ParserElement: class BlockQuote(AbstractMarkup): def action(self, tokens: ParseResults) -> str: - text = self.markup.transform_string("\n".join([line.lstrip() for line in tokens[0].strip().splitlines()])) - return "\n".join([f"> {line}" for line in text.splitlines()]) + text = self.markup.transform_string( + "".join([line.lstrip() for line in tokens[0].strip().splitlines(keepends=True)]) + ) + return "".join([f"> {line}" for line in text.splitlines(keepends=True)]) @property def expr(self) -> ParserElement: diff --git a/jira2markdown/tokens.py b/jira2markdown/tokens.py index b39f932..3bffef9 100644 --- a/jira2markdown/tokens.py +++ b/jira2markdown/tokens.py @@ -9,7 +9,7 @@ def __init__(self): def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType: if loc < len(instring): if instring.startswith("\r\n", loc): - return loc + 2, "\n" + return loc + 2, "\r\n" elif instring[loc] in ("\n", "\r"): return loc + 1, "\n" else: diff --git a/tests/markup/test_headings.py b/tests/markup/test_headings.py index ae418c1..e687925 100644 --- a/tests/markup/test_headings.py +++ b/tests/markup/test_headings.py @@ -12,3 +12,9 @@ def test_header_levels(self): def test_match_start_conditions(self): assert convert(" h2. Title") == " ## Title" assert convert(" A h2. Title") == " A h2. Title" + + def test_windows_line_breaks(self): + assert ( + convert("Line before heading\r\nh1. Title text\r\nLine after heading") + == "Line before heading\r\n# Title text\r\nLine after heading" + ) diff --git a/tests/markup/test_lists.py b/tests/markup/test_lists.py index 7a79ff6..f607b58 100644 --- a/tests/markup/test_lists.py +++ b/tests/markup/test_lists.py @@ -318,5 +318,5 @@ def test_list_indent(self): def test_windows_line_breaks(self): assert ( convert("Line before list\r\n * Bulleted item 1\r\n * Bulleted item 2\r\n\r\nLine after list") - == "Line before list\n- Bulleted item 1\n- Bulleted item 2\n\nLine after list" + == "Line before list\r\n- Bulleted item 1\r\n- Bulleted item 2\r\n\r\nLine after list" ) diff --git a/tests/markup/test_mixed_content.py b/tests/markup/test_mixed_content.py index 4b0f874..1eeecf7 100644 --- a/tests/markup/test_mixed_content.py +++ b/tests/markup/test_mixed_content.py @@ -182,8 +182,8 @@ def render_expected(self, expected, text): return expected[0] % text first_line, next_line = expected - return "\n".join( - [first_line % line if i == 0 else next_line % line for i, line in enumerate(text.splitlines())] + return "".join( + [first_line % line if i == 0 else next_line % line for i, line in enumerate(text.splitlines(keepends=True))] ) def test_headings(self, token, test_input, expected): diff --git a/tests/markup/test_tables.py b/tests/markup/test_tables.py index 331486b..768d94e 100644 --- a/tests/markup/test_tables.py +++ b/tests/markup/test_tables.py @@ -148,10 +148,10 @@ def test_windows_line_breaks(self): "|cell 2-1|cell 2-2|cell 2-3|\r\n\r\n" "text after table" ) == ( - "text before table:\n\n" - "|header 1|header 2|header 3|\n" - "|---|---|---|\n" - "|cell 1-1|cell 1-2|cell 1-3|\n" - "|cell 2-1|cell 2-2|cell 2-3|\n\n" + "text before table:\r\n\r\n" + "|header 1|header 2|header 3|\r\n" + "|---|---|---|\r\n" + "|cell 1-1|cell 1-2|cell 1-3|\r\n" + "|cell 2-1|cell 2-2|cell 2-3|\r\n\r\n" "text after table" ) diff --git a/tests/markup/test_text_effects.py b/tests/markup/test_text_effects.py index 32eb48b..df6a290 100644 --- a/tests/markup/test_text_effects.py +++ b/tests/markup/test_text_effects.py @@ -23,6 +23,9 @@ def test_match_end_conditions(self): def test_multiline(self): assert convert("*multiline\nbold*") == "\\*multiline\nbold\\*" + def test_multiline_windows_line_breaks(self): + assert convert("*multiline\r\nbold*") == "\\*multiline\r\nbold\\*" + def test_single_token(self): assert convert("single *char") == r"single \*char" @@ -229,6 +232,12 @@ def test_adjacent_text(self): """ ) + def test_windows_line_breaks(self): + assert ( + convert("Preceding line\r\nbq. First quote\r\nbq. Second quote\r\nNext line") + == "Preceding line\r\n> First quote\r\n> Second quote\r\n\nNext line" + ) + class TestBlockQuote: def test_basic_conversion(self): @@ -247,6 +256,12 @@ def test_basic_conversion(self): """ ) + def test_windows_line_breaks(self): + assert ( + convert("\r\n{quote}\r\nhere is quotable\r\ncontent to be quoted\r\n{quote}\r\n") + == "\r\n> here is quotable\r\n> content to be quoted\r\n" + ) + class TestMonospaced: def test_basic_conversion(self): From 05ce8bb8c5195bc8b76e99be17f07529e2a9794e Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Mon, 7 Oct 2024 18:00:13 +0300 Subject: [PATCH 5/5] fixup! fix(jira2markdown): process windows line breaks as a Unix line breaks --- jira2markdown/markup/advanced.py | 2 +- jira2markdown/markup/text_effects.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jira2markdown/markup/advanced.py b/jira2markdown/markup/advanced.py index 8401c81..cdcd8a3 100644 --- a/jira2markdown/markup/advanced.py +++ b/jira2markdown/markup/advanced.py @@ -59,7 +59,7 @@ def action(self, tokens: ParseResults) -> str: prefix = "" text = self.markup.transform_string( - "".join([line.lstrip() for line in tokens.text.strip().splitlines(keepends=True)]) + "".join([line.lstrip() for line in tokens.text.strip().splitlines(keepends=True)]), ) return prefix + "".join([f"> {line}" for line in text.splitlines(keepends=True)]) diff --git a/jira2markdown/markup/text_effects.py b/jira2markdown/markup/text_effects.py index 77af5ec..72f85e7 100644 --- a/jira2markdown/markup/text_effects.py +++ b/jira2markdown/markup/text_effects.py @@ -174,7 +174,7 @@ def expr(self) -> ParserElement: class BlockQuote(AbstractMarkup): def action(self, tokens: ParseResults) -> str: text = self.markup.transform_string( - "".join([line.lstrip() for line in tokens[0].strip().splitlines(keepends=True)]) + "".join([line.lstrip() for line in tokens[0].strip().splitlines(keepends=True)]), ) return "".join([f"> {line}" for line in text.splitlines(keepends=True)])