From 8f2c763609787766c63b856008e5fd3e1c2c338c Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Wed, 21 Jun 2023 13:21:42 -0600 Subject: [PATCH 01/19] add ability to add contributor information to html pages (#1254) --- README.md | 8 +++ dev_tools/docs/nxdl.py | 40 +++++------ dev_tools/ext/__init__.py | 0 dev_tools/ext/contrib_ext.py | 26 +++++++ dev_tools/utils/github.py | 89 ++++++++++++++++++++++++ manual/source/_templates/sourcelink.html | 46 ++++++++++++ manual/source/conf.py | 18 ++--- 7 files changed, 197 insertions(+), 30 deletions(-) mode change 100644 => 100755 README.md mode change 100644 => 100755 dev_tools/docs/nxdl.py create mode 100755 dev_tools/ext/__init__.py create mode 100755 dev_tools/ext/contrib_ext.py create mode 100755 dev_tools/utils/github.py create mode 100755 manual/source/_templates/sourcelink.html mode change 100644 => 100755 manual/source/conf.py diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 6bb56e3ea..76112c770 --- a/README.md +++ b/README.md @@ -25,6 +25,14 @@ Open the HTML manual in a web brower for visual verification firefox build/manual/build/html/index.html +### HTML pages with contributor information + +To build the html pages that contains contributor information on the sidebar set a github access token to an environment variable called GH_TOKEN. + +Note: If set this will increase the build time of the documentation by approximately a factor of 4. + + setenv GH_TOKEN + ## Repository content These are the components that define the structure of NeXus data files diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py old mode 100644 new mode 100755 index 8da3ebbc0..9f3c9c0ec --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -1,5 +1,7 @@ import os import re +import datetime + from collections import OrderedDict from html import parser as HTMLParser from pathlib import Path @@ -13,6 +15,7 @@ from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL from ..utils.types import PathLike +from ..utils.github import get_file_contributors_via_api from .anchor_list import AnchorRegistry @@ -80,6 +83,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print( f".. auto-generated by {__name__} from the NXDL source {source} -- DO NOT EDIT" ) + self._print("") self._print(".. index::") self._print(f" ! {nxclass_name} ({self._listing_category})") @@ -100,6 +104,21 @@ def _parse_nxdl_file(self, nxdl_file: Path): else: extends = f":ref:`{extends}`" + # add the contributors as variables to the rst file that will + nxdl_root = get_nxdl_root() + rel_path = str(nxdl_file.relative_to(nxdl_root)) + rel_html = str(rel_path).replace(os.sep, "/") + contribs_dct = get_file_contributors_via_api("definitions", rel_html) + if contribs_dct is not None: + self._print("") + self._print("..") + self._print(" Contributors List") + i = 0 + for date_str in list(contribs_dct.keys()): + self._print("") + self._print(f" .. |contrib_name| replace:: {contribs_dct[date_str]['name']}|{contribs_dct[date_str]['committer']['committer']['login']}|{contribs_dct[date_str]['committer']['committer']['avatar_url']}|{date_str.split('T')[0]}") + i += 1 + self._print("") self._print("**Status**:\n") self._print(f" {self._listing_category.strip()}, extends {extends}") @@ -127,27 +146,6 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print(f": {doc}", end="") self._print("\n") - # print group references - self._print("**Groups cited**:") - node_list = root.xpath("//nx:group", namespaces=ns) - groups = [] - for node in node_list: - g = node.get("type") - if g.startswith("NX") and g not in groups: - groups.append(g) - if len(groups) == 0: - self._print(" none\n") - else: - out = [(f":ref:`{g}`") for g in groups] - txt = ", ".join(sorted(out)) - self._print(f" {txt}\n") - out = [ - ("%s (base class); used in %s" % (g, self._listing_category)) - for g in groups - ] - txt = ", ".join(out) - self._print(f".. index:: {txt}\n") - # print full tree self._print("**Structure**:\n") for subnode in root.xpath("nx:attribute", namespaces=ns): diff --git a/dev_tools/ext/__init__.py b/dev_tools/ext/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/dev_tools/ext/contrib_ext.py b/dev_tools/ext/contrib_ext.py new file mode 100755 index 000000000..451271cf1 --- /dev/null +++ b/dev_tools/ext/contrib_ext.py @@ -0,0 +1,26 @@ +import re + +# a custom sphinx extension that is connected to the source-read hook for rst files, +# the purpose is to read all of the contributor information from the rst file and +# place it in a string variable that will be used in the sourcelink.html jinja template +# that has been over ridden and sits in the _templates directory to produce the +# contributor information on the for right sidebar of the html pages + +def extract_contributor_vars(app, docname, source): + # Read the RST file content + content = source[0] + + # Extract variables using regular expressions + variables = re.findall(r'\|(\w+)\| replace::\s(.+)', content) + + # Create a dictionary to store the extracted variables + # this will create a list of single strings each of which contain the info about the contributor + extracted_variables = [var[1] for var in variables] + if "variables" not in app.config.html_context.keys(): + app.config.html_context['variables'] = {} + + # Add the extracted variables to the Jinja environment + app.config.html_context['variables'][docname] = extracted_variables + +def setup(app): + app.connect("source-read", extract_contributor_vars) \ No newline at end of file diff --git a/dev_tools/utils/github.py b/dev_tools/utils/github.py new file mode 100755 index 000000000..1ca9c2f1a --- /dev/null +++ b/dev_tools/utils/github.py @@ -0,0 +1,89 @@ + +import os +import requests + +def format_author_name(nm): + """ + make sure all words in name start with a capital + """ + s = "" + nms = nm.split(" ") + for n in nms: + s += n[0].upper() + n[1:] + " " + s = s[:-1] + return(s) + +def get_github_profile_name(email): + """ + given an email addr return the github login name + """ + email = email.replace(' ', '') + idx1 = email.find("@") + return(email[:idx1]) + +def get_file_contributors_via_api(repo_name, file_path): + """ + This function takes the repo name (ex:"definitions") and relative path to the nxdl + file (ex: "applications/NXmx.nxdl.xml") and using the github api it retrieves a dictionary + of committers for that file in descending date order. + + In order to increase the capacity (rate) of use of the github API an access token is used if it exists + as an environment variable called GH_TOKEN, in the ci yaml file this is expected to be assigned from the secret + object like this + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + + With the access token the rate is 5000 times per hour and without it is 60 + + returns a sorted dict of unique contributors to a file, or None if no GH_TOKEN has been defined in os.environ + """ + have_token = False + if "GH_TOKEN" in os.environ.keys(): + access_token = os.environ["GH_TOKEN"] + if len(access_token) > 0: + have_token = True + else: + # because the environment does not contain GH_TOKEN, assume the user wants to build the + # docs without contributor info + return(None) + + contrib_skip_list = ["GitHub"] + url = f"https://api.github.com/repos/nexusformat/{repo_name}/commits" + params = {"path": file_path} + headers = {} + if have_token: + # Set the headers with the access token + headers = { + 'Authorization': f'token {access_token}', + 'Accept': 'application/vnd.github.v3+json' + } + + response = requests.get(url, params=params, headers=headers) + commits = response.json() + if response.status_code == 403: + # the max rate per hour has been reached + raise Exception(f"{commits['message']},{commits['documentation_url']}") + + contributor_names = set() + contribs_dct = {} + _email_lst = [] + for commit_dct in commits: + if commit_dct["committer"] is not None: + contributor = commit_dct["commit"]["committer"]["name"] + if contributor in contrib_skip_list: + continue + contributor_names.add(contributor) + if commit_dct["commit"]["committer"]["email"] not in _email_lst: + _email = commit_dct["commit"]["committer"]["email"] + _email_lst.append(_email) + contribs_dct[commit_dct["commit"]['committer']["date"]] = { + "name": format_author_name(commit_dct["commit"]["committer"]["name"]), "committer": commit_dct} + + #sort them so they are in descending order from newest to oldest + sorted_keys = sorted(contribs_dct.keys(), reverse=True) + sorted_dict = {key: contribs_dct[key] for key in sorted_keys} + + return (sorted_dict) + + + diff --git a/manual/source/_templates/sourcelink.html b/manual/source/_templates/sourcelink.html new file mode 100755 index 000000000..6ab27a694 --- /dev/null +++ b/manual/source/_templates/sourcelink.html @@ -0,0 +1,46 @@ +{# + basic/sourcelink.html + ~~~~~~~~~~~~~~~~~~~~~ + 'classes/applications/NXstxm' + + Sphinx sidebar template: "show source" link. + + :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +#} +{%- if show_source and has_source and sourcename %} +
+

{{ _('This Page') }}

+

Have a Question? get help

+
    + + {% set nx_class_nm = sourcename|replace(".rst.txt","") %} + {% if variables[nx_class_nm]|length > 0 %} +

    Contributors

    + {% else %} + {% endif %} +
    + {% for vars_string in variables[nx_class_nm] %} + {% set var_list = vars_string.split('|') %} + {% set tooltip_string = var_list[0] ~ " " ~ var_list[3] %} + + GitHub Avatar + {% endfor %} +
+
+{%- endif %} diff --git a/manual/source/conf.py b/manual/source/conf.py old mode 100644 new mode 100755 index 51b35e4bb..8e0875dbb --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -8,14 +8,13 @@ import sys, os, datetime + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - +# add the abs path to the custom extension for collecting the contributor variables from the rst files +sys.path.insert(0, os.path.abspath('../../../dev_tools/ext')) # -- Project information ----------------------------------------------------- @@ -47,7 +46,8 @@ 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', 'sphinx.ext.todo', - 'sphinx_tabs.tabs' + 'sphinx_tabs.tabs', + 'contrib_ext' ] # Show `.. todo` directives in the output @@ -80,10 +80,10 @@ html_sidebars = { '**': [ - 'localtoc.html', - 'relations.html', - 'sourcelink.html', - 'searchbox.html', + 'localtoc.html', + 'relations.html', + 'sourcelink.html', + 'searchbox.html', 'google_search.html' ], } From 1b0ef77e129ab8ec6ee2dc594ca7ed3470734f0e Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Wed, 21 Jun 2023 14:41:35 -0600 Subject: [PATCH 02/19] replaced the accidently deleted Groups Cited section of parser --- dev_tools/docs/nxdl.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 9f3c9c0ec..614007bce 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -146,6 +146,27 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print(f": {doc}", end="") self._print("\n") + # print group references + self._print("**Groups cited**:") + node_list = root.xpath("//nx:group", namespaces=ns) + groups = [] + for node in node_list: + g = node.get("type") + if g.startswith("NX") and g not in groups: + groups.append(g) + if len(groups) == 0: + self._print(" none\n") + else: + out = [(f":ref:`{g}`") for g in groups] + txt = ", ".join(sorted(out)) + self._print(f" {txt}\n") + out = [ + ("%s (base class); used in %s" % (g, self._listing_category)) + for g in groups + ] + txt = ", ".join(out) + self._print(f".. index:: {txt}\n") + # print full tree self._print("**Structure**:\n") for subnode in root.xpath("nx:attribute", namespaces=ns): From c3384f81c72250fc8ea080408da732d8ed0189ab Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Thu, 22 Jun 2023 10:52:51 -0600 Subject: [PATCH 03/19] cleaned up, formatted with black --- dev_tools/docs/nxdl.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 614007bce..80c8dec0b 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -113,11 +113,14 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print("") self._print("..") self._print(" Contributors List") - i = 0 - for date_str in list(contribs_dct.keys()): + for date_str, contrib_dct in contribs_dct.items(): + date_str = date_str.split("T")[0] + name = contrib_dct['name'] + gh_login_nm = contrib_dct['committer']['commit']['committer']['name'] + gh_avatar_url = contrib_dct['committer']['committer']['avatar_url'] self._print("") - self._print(f" .. |contrib_name| replace:: {contribs_dct[date_str]['name']}|{contribs_dct[date_str]['committer']['committer']['login']}|{contribs_dct[date_str]['committer']['committer']['avatar_url']}|{date_str.split('T')[0]}") - i += 1 + s = "|".join([name, gh_login_nm, gh_avatar_url, date_str]) + self._print(f" .. |contrib_info| replace:: {s}") self._print("") self._print("**Status**:\n") From f0d77e07feade3f008205dd195bd0c28df401d52 Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Thu, 22 Jun 2023 10:54:52 -0600 Subject: [PATCH 04/19] compiled re instead of passing string expression each time --- dev_tools/ext/contrib_ext.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev_tools/ext/contrib_ext.py b/dev_tools/ext/contrib_ext.py index 451271cf1..ab0756f83 100755 --- a/dev_tools/ext/contrib_ext.py +++ b/dev_tools/ext/contrib_ext.py @@ -6,21 +6,23 @@ # that has been over ridden and sits in the _templates directory to produce the # contributor information on the for right sidebar of the html pages +variables_re = re.compile(r"\|(\w+)\| replace::\s(.+)") def extract_contributor_vars(app, docname, source): # Read the RST file content content = source[0] # Extract variables using regular expressions - variables = re.findall(r'\|(\w+)\| replace::\s(.+)', content) + variables = variables_re.findall(content) # Create a dictionary to store the extracted variables # this will create a list of single strings each of which contain the info about the contributor extracted_variables = [var[1] for var in variables] if "variables" not in app.config.html_context.keys(): - app.config.html_context['variables'] = {} + app.config.html_context["variables"] = {} # Add the extracted variables to the Jinja environment - app.config.html_context['variables'][docname] = extracted_variables + app.config.html_context["variables"][docname] = extracted_variables + def setup(app): - app.connect("source-read", extract_contributor_vars) \ No newline at end of file + app.connect("source-read", extract_contributor_vars) From df9eb829b34cb060e45f922f0295d5afe53880ab Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Thu, 22 Jun 2023 10:57:31 -0600 Subject: [PATCH 05/19] remove useless string in comment --- manual/source/_templates/sourcelink.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manual/source/_templates/sourcelink.html b/manual/source/_templates/sourcelink.html index 6ab27a694..791bdb334 100755 --- a/manual/source/_templates/sourcelink.html +++ b/manual/source/_templates/sourcelink.html @@ -1,7 +1,6 @@ {# basic/sourcelink.html ~~~~~~~~~~~~~~~~~~~~~ - 'classes/applications/NXstxm' Sphinx sidebar template: "show source" link. @@ -11,7 +10,7 @@ {%- if show_source and has_source and sourcename %}

{{ _('This Page') }}

-

Have a Question? get help

+

Have a Question? Get help