From 63b4ca1a423e406b46aeee245862ddda3e5844e8 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 6 Feb 2024 21:41:09 +0000 Subject: [PATCH 1/7] Fix bug and tests --- src/uwtools/config/atparse_to_jinja2.py | 8 +- .../tests/config/test_atparse_to_jinja2.py | 83 +++++++++++++------ 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index ba0bc3f2d..64cc23e0e 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -25,12 +25,12 @@ def convert( def lines() -> Generator[str, Any, Any]: with readable(input_file) as f_in: - for line in f_in.read().strip().split("\n"): - yield _replace(line.strip()) + for line in f_in.read().split("\n"): + yield _replace(line) def write(f_out: IO) -> None: for line in lines(): - print(_replace(line.strip()), file=f_out) + print(_replace(line), file=f_out) if dry_run: for line in lines(): @@ -54,5 +54,5 @@ def _replace(atline: str) -> str: # Set maxsplits to 1 so only first ] is captured, which should be the # bracket closing @[. after_atparse = atline.split("@[", 1)[1].split("]", 1)[1] - atline = "".join([before_atparse, "{{", within_atparse, "}}", after_atparse]) + atline = "".join([before_atparse, "{{ ", within_atparse, " }}", after_atparse]) return atline diff --git a/src/uwtools/tests/config/test_atparse_to_jinja2.py b/src/uwtools/tests/config/test_atparse_to_jinja2.py index 49a4ecc1c..c7d1e149d 100644 --- a/src/uwtools/tests/config/test_atparse_to_jinja2.py +++ b/src/uwtools/tests/config/test_atparse_to_jinja2.py @@ -18,83 +18,114 @@ @fixture -def atparselines(): - return ["@[greeting] to the @[subject]", "@[flowers] are @[color]"] +def txt_atparse(): + return """ +@[greeting] to the @[subject] +@[flowers] are @[color] +""".strip() @fixture -def atparsefile(atparselines, tmp_path): - path = tmp_path / "atparse.txt" - with open(path, "w", encoding="utf-8") as f: - for line in atparselines: - print(line, file=f) - return path +def txt_jinja2(): + return """ +{{ greeting }} to the {{ subject }} +{{ flowers }} are {{ color }} +""".strip() @fixture -def jinja2txt(): - return "{{greeting}} to the {{subject}}\n{{flowers}} are {{color}}\n" +def atparsefile(txt_atparse, tmp_path): + path = tmp_path / "atparse.txt" + with open(path, "w", encoding="utf-8") as f: + print(txt_atparse, file=f) + return path # Test functions -def test_convert_input_file_to_output_file(atparsefile, capsys, jinja2txt, tmp_path): +def test_convert_input_file_to_output_file(atparsefile, capsys, txt_jinja2, tmp_path): outfile = tmp_path / "outfile" atparse_to_jinja2.convert(input_file=atparsefile, output_file=outfile) with open(outfile, "r", encoding="utf-8") as f: - assert f.read() == jinja2txt + assert f.read().strip() == txt_jinja2 streams = capsys.readouterr() assert not streams.err assert not streams.out -def test_convert_input_file_to_logging(atparsefile, caplog, capsys, jinja2txt, tmp_path): +def test_convert_input_file_to_logging(atparsefile, caplog, capsys, txt_jinja2, tmp_path): log.setLevel(logging.INFO) outfile = tmp_path / "outfile" atparse_to_jinja2.convert(input_file=atparsefile, dry_run=True) streams = capsys.readouterr() - assert "\n".join(record.message for record in caplog.records) == jinja2txt.strip() + assert "\n".join(record.message for record in caplog.records).strip() == txt_jinja2 assert not streams.out assert not outfile.is_file() -def test_convert_input_file_to_stdout(atparsefile, capsys, jinja2txt): +def test_convert_input_file_to_stdout(atparsefile, capsys, txt_jinja2): atparse_to_jinja2.convert(input_file=atparsefile) streams = capsys.readouterr() assert not streams.err - assert streams.out == jinja2txt + assert streams.out.strip() == txt_jinja2 + + +def test_convert_preserve_whitespace(tmp_path): + atparse = """ +@[first_entry] + @[second_entry] + @[third_entry] + @[fourth_entry] + + @[fifth_entry] +""".strip() + infile = tmp_path / "atparse" + with open(infile, "w", encoding="utf-8") as f: + f.write(atparse) + outfile = tmp_path / "jinja2" + atparse_to_jinja2.convert(input_file=infile, output_file=outfile) + expected = """ +{{ first_entry }} + {{ second_entry }} + {{ third_entry }} + {{ fourth_entry }} + + {{ fifth_entry }} +""".strip() + with open(outfile, "r", encoding="utf-8") as f: + assert f.read().strip() == expected -def test_convert_stdin_to_file(atparselines, capsys, jinja2txt, tmp_path): +def test_convert_stdin_to_file(txt_atparse, capsys, txt_jinja2, tmp_path): outfile = tmp_path / "outfile" _stdinproxy.cache_clear() - with patch.object(sys, "stdin", new=StringIO("\n".join(atparselines))): + with patch.object(sys, "stdin", new=StringIO(txt_atparse)): atparse_to_jinja2.convert(output_file=outfile) with open(outfile, "r", encoding="utf-8") as f: - assert f.read() == jinja2txt + assert f.read().strip() == txt_jinja2 streams = capsys.readouterr() assert not streams.err assert not streams.out -def test_convert_stdin_to_logging(atparselines, caplog, jinja2txt, tmp_path): +def test_convert_stdin_to_logging(txt_atparse, caplog, txt_jinja2, tmp_path): log.setLevel(logging.INFO) outfile = tmp_path / "outfile" _stdinproxy.cache_clear() - with patch.object(sys, "stdin", new=StringIO("\n".join(atparselines))): + with patch.object(sys, "stdin", new=StringIO(txt_atparse)): atparse_to_jinja2.convert(output_file=outfile, dry_run=True) - assert "\n".join(record.message for record in caplog.records) == jinja2txt.strip() + assert "\n".join(record.message for record in caplog.records) == txt_jinja2.strip() assert not outfile.is_file() -def test_convert_stdin_to_stdout(atparselines, capsys, jinja2txt): +def test_convert_stdin_to_stdout(txt_atparse, capsys, txt_jinja2): _stdinproxy.cache_clear() - with patch.object(sys, "stdin", new=StringIO("\n".join(atparselines))): + with patch.object(sys, "stdin", new=StringIO(txt_atparse)): atparse_to_jinja2.convert() streams = capsys.readouterr() assert not streams.err - assert streams.out == jinja2txt + assert streams.out.strip() == txt_jinja2 def test__replace(): @@ -103,4 +134,4 @@ def test__replace(): assert atparse_to_jinja2._replace(line_without) == line_without # A line with atparse syntax should be returned updated to Jinja2 syntax: line_with = "@[greeting] to the @[subject]" - assert atparse_to_jinja2._replace(line_with) == "{{greeting}} to the {{subject}}" + assert atparse_to_jinja2._replace(line_with) == "{{ greeting }} to the {{ subject }}" From c55824ff9332ca193a378f910391057306b14db3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 6 Feb 2024 21:47:37 +0000 Subject: [PATCH 2/7] Simplify --- src/uwtools/config/atparse_to_jinja2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index 64cc23e0e..f2f7e60a9 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -29,8 +29,7 @@ def lines() -> Generator[str, Any, Any]: yield _replace(line) def write(f_out: IO) -> None: - for line in lines(): - print(_replace(line), file=f_out) + f_out.write("\n".join(lines())) if dry_run: for line in lines(): From 171daca46b68b2a04ed884a67061854414e149c3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 6 Feb 2024 21:54:49 +0000 Subject: [PATCH 3/7] Fix docs and --dry-run whitespace --- docs/sections/user_guide/cli/mode_template.rst | 6 +++--- src/uwtools/config/atparse_to_jinja2.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sections/user_guide/cli/mode_template.rst b/docs/sections/user_guide/cli/mode_template.rst index acd0f84fe..82bc72493 100644 --- a/docs/sections/user_guide/cli/mode_template.rst +++ b/docs/sections/user_guide/cli/mode_template.rst @@ -273,7 +273,7 @@ The examples in this section use atparse-formatted template file ``atparse.txt`` .. code-block:: text $ uw template translate --input-file atparse.txt - {{greeting}}, {{recipient}}! + {{ greeting }}, {{ recipient }}! Shell redirection via ``|``, ``>``, et al. may also be used to stream output to a file, another process, etc. @@ -287,11 +287,11 @@ The examples in this section use atparse-formatted template file ``atparse.txt`` .. code-block:: jinja - {{greeting}}, {{recipient}}! + {{ greeting }}, {{ recipient }}! * With the ``--dry-run`` flag specified, nothing is written to ``stdout`` (or to a file if ``--output-file`` is specified), but a report of what would have been written is logged to ``stderr``: .. code-block:: text $ uw template translate --input-file atparse.txt --dry-run - [2024-01-03T16:41:13] INFO {{greeting}}, {{recipient}}! + [2024-02-06T21:53:43] INFO {{ greeting }}, {{ recipient }}! diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index f2f7e60a9..ee57f698e 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -32,7 +32,7 @@ def write(f_out: IO) -> None: f_out.write("\n".join(lines())) if dry_run: - for line in lines(): + for line in "\n".join(lines()).strip().split("\n"): log.info(line) else: with writable(output_file) as f: From adea12a4acfb7c5922b913ffcd5937b603369271 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 6 Feb 2024 22:05:52 +0000 Subject: [PATCH 4/7] Simplify --- src/uwtools/config/atparse_to_jinja2.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index ee57f698e..d2d9011e3 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -3,7 +3,6 @@ """ import re -from typing import IO, Any, Generator from uwtools.logging import log from uwtools.utils.file import OptionalPath, readable, writable @@ -23,20 +22,15 @@ def convert( :param dry_run: Run in dry-run mode? """ - def lines() -> Generator[str, Any, Any]: - with readable(input_file) as f_in: - for line in f_in.read().split("\n"): - yield _replace(line) - - def write(f_out: IO) -> None: - f_out.write("\n".join(lines())) - + with readable(input_file) as f: + lines = f.read().split("\n") + jinja2 = "\n".join(_replace(line) for line in lines).rstrip() if dry_run: - for line in "\n".join(lines()).strip().split("\n"): + for line in jinja2.split("\n"): log.info(line) else: with writable(output_file) as f: - write(f) + print(jinja2, file=f) def _replace(atline: str) -> str: From 176f46fef820b5bee25d7aa31fd0217d593ed839 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 6 Feb 2024 22:19:26 +0000 Subject: [PATCH 5/7] Better unit test --- src/uwtools/config/atparse_to_jinja2.py | 2 +- src/uwtools/tests/config/test_atparse_to_jinja2.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index d2d9011e3..b083f47b2 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -24,7 +24,7 @@ def convert( with readable(input_file) as f: lines = f.read().split("\n") - jinja2 = "\n".join(_replace(line) for line in lines).rstrip() + jinja2 = re.sub(r"\n$", "", "\n".join(_replace(line) for line in lines), count=1) if dry_run: for line in jinja2.split("\n"): log.info(line) diff --git a/src/uwtools/tests/config/test_atparse_to_jinja2.py b/src/uwtools/tests/config/test_atparse_to_jinja2.py index c7d1e149d..cb57d14f6 100644 --- a/src/uwtools/tests/config/test_atparse_to_jinja2.py +++ b/src/uwtools/tests/config/test_atparse_to_jinja2.py @@ -73,28 +73,32 @@ def test_convert_input_file_to_stdout(atparsefile, capsys, txt_jinja2): def test_convert_preserve_whitespace(tmp_path): atparse = """ + @[first_entry] @[second_entry] @[third_entry] @[fourth_entry] @[fifth_entry] -""".strip() + +""" infile = tmp_path / "atparse" with open(infile, "w", encoding="utf-8") as f: f.write(atparse) outfile = tmp_path / "jinja2" atparse_to_jinja2.convert(input_file=infile, output_file=outfile) expected = """ + {{ first_entry }} {{ second_entry }} {{ third_entry }} {{ fourth_entry }} {{ fifth_entry }} -""".strip() + +""" with open(outfile, "r", encoding="utf-8") as f: - assert f.read().strip() == expected + assert f.read() == expected def test_convert_stdin_to_file(txt_atparse, capsys, txt_jinja2, tmp_path): From 8fcf604bf9b165ac034a2f6d59100e23b3afec70 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 7 Feb 2024 15:59:33 +0000 Subject: [PATCH 6/7] Improve test --- src/uwtools/tests/config/test_atparse_to_jinja2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/tests/config/test_atparse_to_jinja2.py b/src/uwtools/tests/config/test_atparse_to_jinja2.py index cb57d14f6..ad62db047 100644 --- a/src/uwtools/tests/config/test_atparse_to_jinja2.py +++ b/src/uwtools/tests/config/test_atparse_to_jinja2.py @@ -79,7 +79,7 @@ def test_convert_preserve_whitespace(tmp_path): @[third_entry] @[fourth_entry] - @[fifth_entry] + @[fifth_entry] @[sixth_entry] """ infile = tmp_path / "atparse" @@ -94,7 +94,7 @@ def test_convert_preserve_whitespace(tmp_path): {{ third_entry }} {{ fourth_entry }} - {{ fifth_entry }} + {{ fifth_entry }} {{ sixth_entry }} """ with open(outfile, "r", encoding="utf-8") as f: From 35d61435da5b984f0122dd304ef4b8a56bba760a Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 7 Feb 2024 16:11:42 +0000 Subject: [PATCH 7/7] Use readlines() --- src/uwtools/config/atparse_to_jinja2.py | 8 ++++---- src/uwtools/tests/config/test_atparse_to_jinja2.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/uwtools/config/atparse_to_jinja2.py b/src/uwtools/config/atparse_to_jinja2.py index b083f47b2..2b1bd3aa3 100644 --- a/src/uwtools/config/atparse_to_jinja2.py +++ b/src/uwtools/config/atparse_to_jinja2.py @@ -23,14 +23,14 @@ def convert( """ with readable(input_file) as f: - lines = f.read().split("\n") - jinja2 = re.sub(r"\n$", "", "\n".join(_replace(line) for line in lines), count=1) + lines = f.readlines() + jinja2 = "".join(_replace(line) for line in lines) if dry_run: - for line in jinja2.split("\n"): + for line in jinja2.removesuffix("\n").split("\n"): log.info(line) else: with writable(output_file) as f: - print(jinja2, file=f) + f.write(jinja2) def _replace(atline: str) -> str: diff --git a/src/uwtools/tests/config/test_atparse_to_jinja2.py b/src/uwtools/tests/config/test_atparse_to_jinja2.py index ad62db047..4c6df7745 100644 --- a/src/uwtools/tests/config/test_atparse_to_jinja2.py +++ b/src/uwtools/tests/config/test_atparse_to_jinja2.py @@ -119,7 +119,7 @@ def test_convert_stdin_to_logging(txt_atparse, caplog, txt_jinja2, tmp_path): _stdinproxy.cache_clear() with patch.object(sys, "stdin", new=StringIO(txt_atparse)): atparse_to_jinja2.convert(output_file=outfile, dry_run=True) - assert "\n".join(record.message for record in caplog.records) == txt_jinja2.strip() + assert "\n".join(record.message for record in caplog.records) == txt_jinja2 assert not outfile.is_file()