From 0014cd68f7a3ae1e8395fc3b19ae5fd678feccc4 Mon Sep 17 00:00:00 2001 From: Rob Taylor Date: Sun, 29 Sep 2024 17:26:20 -0400 Subject: [PATCH 1/2] Add ability to generate a markdown index --- .coveragerc | 7 +- doorstop/cli/commands.py | 3 + doorstop/cli/main.py | 9 ++- doorstop/cli/tests/test_all.py | 28 ++++++++ doorstop/core/publishers/markdown.py | 71 ++++++++++++++++++- .../tests/test_publisher_markdown.py | 53 +++++++++++++- doorstop/core/tests/test_publisher.py | 20 +++++- 7 files changed, 182 insertions(+), 9 deletions(-) diff --git a/.coveragerc b/.coveragerc index 9f403cb07..e7d5e97ff 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,8 @@ omit = */tests/* */__main__.py +dynamic_context = test_function + [report] omit = @@ -20,4 +22,7 @@ exclude_lines = raise NotImplementedError except DistributionNotFound -skip_covered = true +skip_covered = false + +[html] +show_contexts = True diff --git a/doorstop/cli/commands.py b/doorstop/cli/commands.py index 1df175318..5d26d89ac 100644 --- a/doorstop/cli/commands.py +++ b/doorstop/cli/commands.py @@ -561,6 +561,9 @@ def run_publish(args, cwd, error, catch=True): if args.width: kwargs["width"] = args.width + if args.index: + kwargs["index"] = True + # Write to output file(s) if args.path: path = os.path.abspath(os.path.join(cwd, args.path)) diff --git a/doorstop/cli/main.py b/doorstop/cli/main.py index 5b92b7c1a..3a337f5b0 100644 --- a/doorstop/cli/main.py +++ b/doorstop/cli/main.py @@ -76,7 +76,9 @@ def main(args=None): # pylint: disable=R0915 # Build main parser parser = argparse.ArgumentParser( - prog=CLI, description=DESCRIPTION, **shared # type: ignore + prog=CLI, + description=DESCRIPTION, + **shared, # type: ignore ) parser.add_argument( "-F", @@ -536,6 +538,11 @@ def _publish(subs, shared): help="do not include levels on heading and non-heading or non-heading items", ) sub.add_argument("--template", help="template file", default=None) + sub.add_argument( + "--index", + help="Generate top level index (when producing markdown).", + action="store_true", + ) if __name__ == "__main__": diff --git a/doorstop/cli/tests/test_all.py b/doorstop/cli/tests/test_all.py index 62f265712..7b6d046a6 100644 --- a/doorstop/cli/tests/test_all.py +++ b/doorstop/cli/tests/test_all.py @@ -810,10 +810,38 @@ def test_publish_tree_text(self): self.assertTrue(os.path.isdir(path)) self.assertFalse(os.path.isfile(os.path.join(path, "index.html"))) + def test_publish_tree_md(self): + """Verify 'doorstop publish' can create a Markdown directory.""" + path = os.path.join(self.temp, "all") + self.assertIs(None, main(["publish", "all", path, "--markdown", "--index"])) + self.assertTrue(os.path.isdir(path)) + self.assertTrue(os.path.isfile(os.path.join(path, "index.md"))) + def test_publish_tree_no_path(self): """Verify 'doorstop publish' returns an error with no path.""" self.assertRaises(SystemExit, main, ["publish", "all"]) + def test_publish_tree_markdown_with_index(self): + """Verify 'doorstop publish' can create Markdown output for a tree, + with an index.""" + path = os.path.join(self.temp, "all") + self.assertIs(None, main(["publish", "all", path, "--markdown", "--index"])) + self.assertTrue(os.path.isdir(path)) + self.assertTrue(os.path.isfile(os.path.join(path, "index.md"))) + + def test_publish_markdown_tree_no_path(self): + """Verify 'doorstop publish' returns an error with no path.""" + self.assertRaises( + SystemExit, + main, + [ + "publish", + "-m", + "--index", + "all", + ], + ) + class TestPublishCommand(TempTestCase): """Tests 'doorstop publish' options toc and template""" diff --git a/doorstop/core/publishers/markdown.py b/doorstop/core/publishers/markdown.py index 89f2c40c7..7b69c671d 100644 --- a/doorstop/core/publishers/markdown.py +++ b/doorstop/core/publishers/markdown.py @@ -2,20 +2,85 @@ """Functions to publish documents and items.""" +import os from re import sub from doorstop import common, settings -from doorstop.core.publishers.base import BasePublisher, format_level +from doorstop.core.publishers.base import ( + BasePublisher, + extract_prefix, + format_level, + get_document_attributes, +) from doorstop.core.types import is_item, iter_items log = common.logger(__name__) +INDEX = "index.md" class MarkdownPublisher(BasePublisher): """Markdown publisher.""" - def create_index(self, directory, index=None, extensions=(".md",), tree=None): - """No index for Markdown.""" + def create_index(self, directory, index=INDEX, extensions=(".md",), tree=None): + """Create an markdown index of all files in a directory. + + :param directory: directory for index + :param index: filename for index + :param extensions: file extensions to include + :param tree: optional tree to determine index structure + + """ + # Get paths for the index index + filenames = [] + for filename in os.listdir(directory): + if filename.endswith(extensions) and filename != INDEX: + filenames.append(os.path.join(filename)) + + # Create the index + if filenames: + path = os.path.join(directory, index) + log.info("creating an {}...".format(index)) + lines = self.lines_index(sorted(filenames), tree=tree) + common.write_text(" # Requirements index", path) + common.write_text("\n".join(lines), path) + else: + log.warning("no files for {}".format(index)) + + def _index_tree(self, tree, depth): + """Recursively generate markdown index. + + :param tree: optional tree to determine index structure + :param depth: depth recursed into tree + """ + + depth = depth + 1 + + title = get_document_attributes(tree.document)["title"] + prefix = extract_prefix(tree.document) + filename = f"{prefix}.md" + + # Tree structure + yield " " * (depth * 2 - 1) + f"* [{prefix}]({filename}) - {title}" + # yield self.table_of_contents(linkify=True, obj=tree.document, depth=depth, heading=False) + for child in tree.children: + yield from self._index_tree(tree=child, depth=depth) + + def lines_index(self, filenames, tree=None): + """Yield lines of Markdown for index.md. + + :param filenames: list of filenames to add to the index + :param tree: optional tree to determine index structure + """ + if tree: + yield from self._index_tree(tree, depth=0) + + # Additional files + if filenames: + yield "" + yield "### Published Documents:" + for filename in filenames: + name = os.path.splitext(filename)[0] + yield " * [{n}]({f})".format(f=filename, n=name) def create_matrix(self, directory): """No traceability matrix for Markdown.""" diff --git a/doorstop/core/publishers/tests/test_publisher_markdown.py b/doorstop/core/publishers/tests/test_publisher_markdown.py index 88e637ff5..ff852475a 100644 --- a/doorstop/core/publishers/tests/test_publisher_markdown.py +++ b/doorstop/core/publishers/tests/test_publisher_markdown.py @@ -5,15 +5,15 @@ # pylint: disable=unused-argument,protected-access import os -import stat import unittest from secrets import token_hex from shutil import rmtree -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch from doorstop.core import publisher from doorstop.core.publishers.tests.helpers import YAML_CUSTOM_ATTRIBUTES, getLines from doorstop.core.tests import ( + EMPTY, FILES, ROOT, MockDataMixIn, @@ -22,6 +22,7 @@ MockItemAndVCS, ) from doorstop.core.tests.helpers import on_error_with_retry +from doorstop.core.types import UID class TestModule(MockDataMixIn, unittest.TestCase): @@ -263,3 +264,51 @@ def test_toc(self): md_publisher = publisher.check(".md", self.document) toc = md_publisher.table_of_contents(linkify=True, obj=self.document) self.assertEqual(expected, toc) + + def test_index(self): + """Verify an Markdown index can be created.""" + # Arrange + path = os.path.join(FILES, "index.md") + md_publisher = publisher.check(".md") + # Act + md_publisher.create_index(FILES) + # Assert + self.assertTrue(os.path.isfile(path)) + + def test_index_no_files(self): + """Verify an Markdown index is only created when files exist.""" + path = os.path.join(EMPTY, "index.md") + md_publisher = publisher.check(".md") + # Act + md_publisher.create_index(EMPTY) + # Assert + self.assertFalse(os.path.isfile(path)) + + def test_index_tree(self): + """Verify an Markdown index can be created with a tree.""" + path = os.path.join(FILES, "index2.md") + mock_tree = MagicMock() + mock_tree.documents = [] + for prefix in ("SYS", "HLR", "LLR", "HLT", "LLT"): + mock_document = MagicMock() + mock_document.prefix = prefix + mock_tree.documents.append(mock_document) + mock_tree.draw = lambda: "(mock tree structure)" + mock_item = Mock() + mock_item.uid = "KNOWN-001" + mock_item.document = Mock() + mock_item.document.prefix = "KNOWN" + mock_item.header = None + mock_item_unknown = Mock(spec=["uid"]) + mock_item_unknown.uid = "UNKNOWN-002" + mock_trace = [ + (None, mock_item, None, None, None), + (None, None, None, mock_item_unknown, None), + (None, None, None, None, None), + ] + mock_tree.get_traceability = lambda: mock_trace + md_publisher = publisher.check(".md") + # Act + md_publisher.create_index(FILES, index="index2.md", tree=mock_tree) + # Assert + self.assertTrue(os.path.isfile(path)) diff --git a/doorstop/core/tests/test_publisher.py b/doorstop/core/tests/test_publisher.py index 064993d75..a8d2f3176 100644 --- a/doorstop/core/tests/test_publisher.py +++ b/doorstop/core/tests/test_publisher.py @@ -128,6 +128,22 @@ def test_index_none_for_md(self): # Assert self.assertEqual(result, False) + def test_index_true_md(self): + """Verify that index = true forces true.""" + tmp_publisher = publisher.check(".md", self.mock_tree) + tmp_publisher.setup(None, True, None) + do_index = tmp_publisher.getIndex() + # Assert + self.assertEqual(do_index, True) + + def test_index_false_md(self): + """Verify that index = false forces false.""" + tmp_publisher = publisher.check(".md", self.mock_tree) + tmp_publisher.setup(None, False, None) + do_index = tmp_publisher.getIndex() + # Assert + self.assertEqual(do_index, False) + def test_index_none_for_txt(self): """Verify that index = None works correctly.""" tmp_publisher = publisher.check(".txt", self.mock_tree) @@ -136,7 +152,7 @@ def test_index_none_for_txt(self): # Assert self.assertEqual(result, False) - def test_index_true(self): + def test_index_true_html(self): """Verify that index = true forces true.""" tmp_publisher = publisher.check(".html", self.mock_tree) tmp_publisher.setup(None, True, None) @@ -144,7 +160,7 @@ def test_index_true(self): # Assert self.assertEqual(do_index, True) - def test_index_false(self): + def test_index_false_html(self): """Verify that index = false forces false.""" tmp_publisher = publisher.check(".html", self.mock_tree) tmp_publisher.setup(None, False, None) From 2947607dadefc42bba1b607648c89008ad38b8ed Mon Sep 17 00:00:00 2001 From: Rob Taylor Date: Sun, 29 Sep 2024 20:53:08 -0400 Subject: [PATCH 2/2] Generate requirements markdown with inclusion in docs --- .readthedocs.yaml | 17 ++++++++++++----- Makefile | 2 +- docs/gen_reqs.py | 15 +++++++++++++++ mkdocs.yml | 4 ++++ 4 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 docs/gen_reqs.py diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0807bef7b..155c28719 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,13 +1,20 @@ version: 2 build: - os: ubuntu-22.04 + os: "ubuntu-22.04" tools: python: "3.9" + jobs: + post_create_environment: + # Install poetry + # https://python-poetry.org/docs/#installing-manually + - pip install poetry + post_install: + # Install dependencies with 'docs' dependency group + # https://python-poetry.org/docs/managing-dependencies/#dependency-groups + # VIRTUAL_ENV needs to be set manually for now. + # See https://github.com/readthedocs/readthedocs.org/pull/11152/ + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install mkdocs: configuration: mkdocs.yml - -python: - install: - - requirements: docs/requirements.txt diff --git a/Makefile b/Makefile index e09db7e4e..c8c74f89c 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ docs/gen/*.tex: $(YAML) .PHONY: reqs-md reqs-md: install docs/gen/*.md docs/gen/*.md: $(YAML) - $(DOORSTOP) publish all docs/gen --markdown + $(DOORSTOP) publish all docs/gen --markdown --index .PHONY: reqs-pdf reqs-pdf: reqs-latex diff --git a/docs/gen_reqs.py b/docs/gen_reqs.py new file mode 100644 index 000000000..24f4184ca --- /dev/null +++ b/docs/gen_reqs.py @@ -0,0 +1,15 @@ +import os + +from doorstop.core import publisher, builder +from doorstop.cli import utilities + + +def on_pre_build(config): + cwd = os.getcwd() + path = os.path.abspath(os.path.join(cwd, "docs/gen")) + tree = builder.build(cwd=cwd) + + published_path = publisher.publish(tree, path, ".md", index=True) + + if published_path: + utilities.show("published: {}".format(published_path)) diff --git a/mkdocs.yml b/mkdocs.yml index 750a9d3a0..0c21d5017 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,9 @@ theme: readthedocs markdown_extensions: - admonition +use_directory_urls: false +hooks: + - docs/gen_reqs.py nav: - Home: index.md @@ -25,6 +28,7 @@ nav: - Desktop Interface: gui/overview.md - Web Interface: web.md - Scripting Interface: api/scripting.md +- Doorstop's requirements: gen/index.md - Reference: - Tree: reference/tree.md - Document: reference/document.md