Merge pull request #661 from robtaylor/markdown-index

Add markdown index support

.coveragerc

@@ -7,6 +7,8 @@ omit =
  */tests/*
  */__main__.py
 
+dynamic_context = test_function
+
 [report]
 
 omit =
@@ -20,4 +22,7 @@ exclude_lines =
  raise NotImplementedError
  except DistributionNotFound
 
-skip_covered = true
+skip_covered = false
+
+[html]
+show_contexts = True

.readthedocs.yaml

@@ -1,13 +1,20 @@
 version: 2
 
 build:
- os: ubuntu-22.04
+ os: "ubuntu-22.04"
  tools:
  python: "3.9"
+ jobs:
+ post_create_environment:
+ # Install poetry
+ # https://python-poetry.org/docs/#installing-manually
+ - pip install poetry
+ post_install:
+ # Install dependencies with 'docs' dependency group
+ # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+ # VIRTUAL_ENV needs to be set manually for now.
+ # See https://github.com/readthedocs/readthedocs.org/pull/11152/
+ - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install
 
 mkdocs:
  configuration: mkdocs.yml
-
-python:
- install:
- - requirements: docs/requirements.txt

Makefile

@@ -177,7 +177,7 @@ docs/gen/*.tex: $(YAML)
 .PHONY: reqs-md
 reqs-md: install docs/gen/*.md
 docs/gen/*.md: $(YAML)
- $(DOORSTOP) publish all docs/gen --markdown
+ $(DOORSTOP) publish all docs/gen --markdown --index
 
 .PHONY: reqs-pdf
 reqs-pdf: reqs-latex

docs/gen_reqs.py

@@ -0,0 +1,15 @@
+import os
+
+from doorstop.core import publisher, builder
+from doorstop.cli import utilities
+
+
+def on_pre_build(config):
+ cwd = os.getcwd()
+ path = os.path.abspath(os.path.join(cwd, "docs/gen"))
+ tree = builder.build(cwd=cwd)
+
+ published_path = publisher.publish(tree, path, ".md", index=True)
+
+ if published_path:
+ utilities.show("published: {}".format(published_path))

doorstop/cli/commands.py

@@ -564,6 +564,9 @@ def run_publish(args, cwd, error, catch=True):
  if args.width:
  kwargs["width"] = args.width
 
+ if args.index:
+ kwargs["index"] = True
+
  # Write to output file(s)
  if args.path:
  path = os.path.abspath(os.path.join(cwd, args.path))

doorstop/cli/main.py

@@ -76,7 +76,9 @@ def main(args=None): # pylint: disable=R0915
 
  # Build main parser
  parser = argparse.ArgumentParser(
- prog=CLI, description=DESCRIPTION, **shared # type: ignore
+ prog=CLI,
+ description=DESCRIPTION,
+ **shared, # type: ignore
  )
  parser.add_argument(
  "-F",
@@ -536,6 +538,11 @@ def _publish(subs, shared):
  help="do not include levels on heading and non-heading or non-heading items",
  )
  sub.add_argument("--template", help="template file", default=None)
+ sub.add_argument(
+ "--index",
+ help="Generate top level index (when producing markdown).",
+ action="store_true",
+ )
 
 
 if __name__ == "__main__":

doorstop/cli/tests/test_all.py

@@ -810,10 +810,38 @@ def test_publish_tree_text(self):
  self.assertTrue(os.path.isdir(path))
  self.assertFalse(os.path.isfile(os.path.join(path, "index.html")))
 
+ def test_publish_tree_md(self):
+ """Verify 'doorstop publish' can create a Markdown directory."""
+ path = os.path.join(self.temp, "all")
+ self.assertIs(None, main(["publish", "all", path, "--markdown", "--index"]))
+ self.assertTrue(os.path.isdir(path))
+ self.assertTrue(os.path.isfile(os.path.join(path, "index.md")))
+
  def test_publish_tree_no_path(self):
  """Verify 'doorstop publish' returns an error with no path."""
  self.assertRaises(SystemExit, main, ["publish", "all"])
 
+ def test_publish_tree_markdown_with_index(self):
+ """Verify 'doorstop publish' can create Markdown output for a tree,
+ with an index."""
+ path = os.path.join(self.temp, "all")
+ self.assertIs(None, main(["publish", "all", path, "--markdown", "--index"]))
+ self.assertTrue(os.path.isdir(path))
+ self.assertTrue(os.path.isfile(os.path.join(path, "index.md")))
+
+ def test_publish_markdown_tree_no_path(self):
+ """Verify 'doorstop publish' returns an error with no path."""
+ self.assertRaises(
+ SystemExit,
+ main,
+ [
+ "publish",
+ "-m",
+ "--index",
+ "all",
+ ],
+ )
+
 
 class TestPublishCommand(TempTestCase):
  """Tests 'doorstop publish' options toc and template"""

doorstop/core/publishers/markdown.py

@@ -2,20 +2,85 @@
 
 """Functions to publish documents and items."""
 
+import os
 from re import sub
 
 from doorstop import common, settings
-from doorstop.core.publishers.base import BasePublisher, format_level
+from doorstop.core.publishers.base import (
+ BasePublisher,
+ extract_prefix,
+ format_level,
+ get_document_attributes,
+)
 from doorstop.core.types import is_item, iter_items
 
 log = common.logger(__name__)
+INDEX = "index.md"
 
 
 class MarkdownPublisher(BasePublisher):
  """Markdown publisher."""
 
- def create_index(self, directory, index=None, extensions=(".md",), tree=None):
- """No index for Markdown."""
+ def create_index(self, directory, index=INDEX, extensions=(".md",), tree=None):
+ """Create an markdown index of all files in a directory.
+
+ :param directory: directory for index
+ :param index: filename for index
+ :param extensions: file extensions to include
+ :param tree: optional tree to determine index structure
+
+ """
+ # Get paths for the index index
+ filenames = []
+ for filename in os.listdir(directory):
+ if filename.endswith(extensions) and filename != INDEX:
+ filenames.append(os.path.join(filename))
+
+ # Create the index
+ if filenames:
+ path = os.path.join(directory, index)
+ log.info("creating an {}...".format(index))
+ lines = self.lines_index(sorted(filenames), tree=tree)
+ common.write_text(" # Requirements index", path)
+ common.write_text("\n".join(lines), path)
+ else:
+ log.warning("no files for {}".format(index))
+
+ def _index_tree(self, tree, depth):
+ """Recursively generate markdown index.
+
+ :param tree: optional tree to determine index structure
+ :param depth: depth recursed into tree
+ """
+
+ depth = depth + 1
+
+ title = get_document_attributes(tree.document)["title"]
+ prefix = extract_prefix(tree.document)
+ filename = f"{prefix}.md"
+
+ # Tree structure
+ yield " " * (depth * 2 - 1) + f"* [{prefix}]({filename}) - {title}"
+ # yield self.table_of_contents(linkify=True, obj=tree.document, depth=depth, heading=False)
+ for child in tree.children:
+ yield from self._index_tree(tree=child, depth=depth)
+
+ def lines_index(self, filenames, tree=None):
+ """Yield lines of Markdown for index.md.
+
+ :param filenames: list of filenames to add to the index
+ :param tree: optional tree to determine index structure
+ """
+ if tree:
+ yield from self._index_tree(tree, depth=0)
+
+ # Additional files
+ if filenames:
+ yield ""
+ yield "### Published Documents:"
+ for filename in filenames:
+ name = os.path.splitext(filename)[0]
+ yield " * [{n}]({f})".format(f=filename, n=name)
 
  def create_matrix(self, directory):
  """No traceability matrix for Markdown."""

doorstop/core/publishers/tests/test_publisher_markdown.py

@@ -5,15 +5,15 @@
 # pylint: disable=unused-argument,protected-access
 
 import os
-import stat
 import unittest
 from secrets import token_hex
 from shutil import rmtree
-from unittest.mock import Mock, patch
+from unittest.mock import MagicMock, Mock, patch
 
 from doorstop.core import publisher
 from doorstop.core.publishers.tests.helpers import YAML_CUSTOM_ATTRIBUTES, getLines
 from doorstop.core.tests import (
+ EMPTY,
  FILES,
  ROOT,
  MockDataMixIn,
@@ -22,6 +22,7 @@
  MockItemAndVCS,
 )
 from doorstop.core.tests.helpers import on_error_with_retry
+from doorstop.core.types import UID
 
 
 class TestModule(MockDataMixIn, unittest.TestCase):
@@ -263,3 +264,51 @@ def test_toc(self):
  md_publisher = publisher.check(".md", self.document)
  toc = md_publisher.table_of_contents(linkify=True, obj=self.document)
  self.assertEqual(expected, toc)
+
+ def test_index(self):
+ """Verify an Markdown index can be created."""
+ # Arrange
+ path = os.path.join(FILES, "index.md")
+ md_publisher = publisher.check(".md")
+ # Act
+ md_publisher.create_index(FILES)
+ # Assert
+ self.assertTrue(os.path.isfile(path))
+
+ def test_index_no_files(self):
+ """Verify an Markdown index is only created when files exist."""
+ path = os.path.join(EMPTY, "index.md")
+ md_publisher = publisher.check(".md")
+ # Act
+ md_publisher.create_index(EMPTY)
+ # Assert
+ self.assertFalse(os.path.isfile(path))
+
+ def test_index_tree(self):
+ """Verify an Markdown index can be created with a tree."""
+ path = os.path.join(FILES, "index2.md")
+ mock_tree = MagicMock()
+ mock_tree.documents = []
+ for prefix in ("SYS", "HLR", "LLR", "HLT", "LLT"):
+ mock_document = MagicMock()
+ mock_document.prefix = prefix
+ mock_tree.documents.append(mock_document)
+ mock_tree.draw = lambda: "(mock tree structure)"
+ mock_item = Mock()
+ mock_item.uid = "KNOWN-001"
+ mock_item.document = Mock()
+ mock_item.document.prefix = "KNOWN"
+ mock_item.header = None
+ mock_item_unknown = Mock(spec=["uid"])
+ mock_item_unknown.uid = "UNKNOWN-002"
+ mock_trace = [
+ (None, mock_item, None, None, None),
+ (None, None, None, mock_item_unknown, None),
+ (None, None, None, None, None),
+ ]
+ mock_tree.get_traceability = lambda: mock_trace
+ md_publisher = publisher.check(".md")
+ # Act
+ md_publisher.create_index(FILES, index="index2.md", tree=mock_tree)
+ # Assert
+ self.assertTrue(os.path.isfile(path))

doorstop/core/tests/test_publisher.py

@@ -128,6 +128,22 @@ def test_index_none_for_md(self):
  # Assert
  self.assertEqual(result, False)
 
+ def test_index_true_md(self):
+ """Verify that index = true forces true."""
+ tmp_publisher = publisher.check(".md", self.mock_tree)
+ tmp_publisher.setup(None, True, None)
+ do_index = tmp_publisher.getIndex()
+ # Assert
+ self.assertEqual(do_index, True)
+
+ def test_index_false_md(self):
+ """Verify that index = false forces false."""
+ tmp_publisher = publisher.check(".md", self.mock_tree)
+ tmp_publisher.setup(None, False, None)
+ do_index = tmp_publisher.getIndex()
+ # Assert
+ self.assertEqual(do_index, False)
+
  def test_index_none_for_txt(self):
  """Verify that index = None works correctly."""
  tmp_publisher = publisher.check(".txt", self.mock_tree)
@@ -136,15 +152,15 @@ def test_index_none_for_txt(self):
  # Assert
  self.assertEqual(result, False)
 
- def test_index_true(self):
+ def test_index_true_html(self):
  """Verify that index = true forces true."""
  tmp_publisher = publisher.check(".html", self.mock_tree)
  tmp_publisher.setup(None, True, None)
  do_index = tmp_publisher.getIndex()
  # Assert
  self.assertEqual(do_index, True)
 
- def test_index_false(self):
+ def test_index_false_html(self):
  """Verify that index = false forces false."""
  tmp_publisher = publisher.check(".html", self.mock_tree)
  tmp_publisher.setup(None, False, None)

mkdocs.yml

@@ -9,6 +9,9 @@ theme: readthedocs
 
 markdown_extensions:
  - admonition
+use_directory_urls: false
+hooks: 
+ - docs/gen_reqs.py
 
 nav:
 - Home: index.md
@@ -25,6 +28,7 @@ nav:
 - Desktop Interface: gui/overview.md
 - Web Interface: web.md
 - Scripting Interface: api/scripting.md
+- Doorstop's requirements: gen/index.md
 - Reference:
  - Tree: reference/tree.md
  - Document: reference/document.md