From 675d10e23b03f416b57d1e50f581dd38540cc380 Mon Sep 17 00:00:00 2001 From: calpt <36051308+calpt@users.noreply.github.com> Date: Mon, 31 Jan 2022 22:52:24 +0100 Subject: [PATCH] Multiversion docs --- .github/workflows/adapter_docs_build.yml | 9 +-- adapter_docs/Makefile | 4 ++ adapter_docs/_static/custom.css | 4 ++ adapter_docs/_templates/versions.html | 33 ++++++++++ adapter_docs/conf.py | 61 ++++++++++++------- adapter_docs/scripts/post_build.py | 16 +++++ setup.py | 2 + src/transformers/adapters/model_mixin.py | 3 +- src/transformers/dependency_versions_table.py | 1 + 9 files changed, 107 insertions(+), 26 deletions(-) create mode 100644 adapter_docs/_templates/versions.html create mode 100644 adapter_docs/scripts/post_build.py diff --git a/.github/workflows/adapter_docs_build.yml b/.github/workflows/adapter_docs_build.yml index 5103396d6a..882de8e155 100644 --- a/.github/workflows/adapter_docs_build.yml +++ b/.github/workflows/adapter_docs_build.yml @@ -2,8 +2,8 @@ name: Build Adapter Docs on: push: - tags: - - 'adapters*' + branches: [ master ] + paths: [ 'adapter_docs/**' ] workflow_dispatch: jobs: @@ -13,16 +13,17 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive + fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: 3.6 - name: Install run: | pip install setuptools==57.4.0 - pip install .[tf,torch,docs] + pip install -e .[torch,docs] - name: Build run: | - cd adapter_docs && make html && cd .. + cd adapter_docs && make html-multi-version && cd .. - name: Deploy uses: peaceiris/actions-gh-pages@v3 with: diff --git a/adapter_docs/Makefile b/adapter_docs/Makefile index d4bb2cbb9e..d83273c4ad 100644 --- a/adapter_docs/Makefile +++ b/adapter_docs/Makefile @@ -14,6 +14,10 @@ help: .PHONY: help Makefile +html-multi-version: + sphinx-multiversion "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + python scripts/post_build.py "$(BUILDDIR)/html" + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/adapter_docs/_static/custom.css b/adapter_docs/_static/custom.css index b009a1c026..be8a5fa6d4 100644 --- a/adapter_docs/_static/custom.css +++ b/adapter_docs/_static/custom.css @@ -24,3 +24,7 @@ a { .rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal { color: #39B3C6; } +/* Sidebar scroll space for version switcher */ +.wy-side-scroll { + padding-bottom: 1em; +} diff --git a/adapter_docs/_templates/versions.html b/adapter_docs/_templates/versions.html new file mode 100644 index 0000000000..92c3a3dae2 --- /dev/null +++ b/adapter_docs/_templates/versions.html @@ -0,0 +1,33 @@ + +{%- if current_version %} +
+ + Versions + v: {{ current_version.name | replace('adapters', 'v') }} + + +
+ {%- if versions.tags %} +
+
Tags
+ {%- for item in versions.tags %} + {%- if current_version.name == 'master' -%} + {%- set item_url = item.url | replace('../', './', 1) -%} + {%- else -%} + {%- set item_url = item.url -%} + {%- endif -%} +
{{ item.name | replace('adapters', 'v') }}
+ {%- endfor %} +
+ {%- endif %} + {%- if versions.branches %} +
+
Branches
+ {%- for item in versions.branches %} +
{{ item.name }}
+ {%- endfor %} +
+ {%- endif %} +
+
+{%- endif %} \ No newline at end of file diff --git a/adapter_docs/conf.py b/adapter_docs/conf.py index 8f0d5f9985..85543b34ba 100644 --- a/adapter_docs/conf.py +++ b/adapter_docs/conf.py @@ -3,6 +3,11 @@ # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +import os +import sys + +from recommonmark.transform import AutoStructify + # -- Path setup -------------------------------------------------------------- @@ -10,17 +15,20 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) -from recommonmark.transform import AutoStructify +rootdir = os.path.join(os.getenv("SPHINX_MULTIVERSION_SOURCEDIR", default="."), "../src") +sys.path.insert(0, rootdir) # -- Project information ----------------------------------------------------- -project = 'adapter-transformers' -copyright = '2020, Adapter-Hub Team' -author = 'Adapter-Hub Team' +project = "adapter-transformers" +copyright = "2020-2022, Adapter-Hub Team" +author = "Adapter-Hub Team" + +docs_versions = [ + "adapters1.1.1", + "adapters2.2.0", +] # -- General configuration --------------------------------------------------- @@ -29,19 +37,20 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'recommonmark', - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx_copybutton', + "recommonmark", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx_copybutton", + "sphinx_multiversion", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'README.md'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "README.md"] # -- Options for HTML output ------------------------------------------------- @@ -49,20 +58,30 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] + +html_logo = "logo.png" +html_favicon = "favicon.png" + + +# -- Options for sphinx-multiversion ------------------------------------------ + +# Whitelist pattern for tags (set to None to ignore all tags) +smv_tag_whitelist = r"({})".format("|".join([v.replace(".", r"\.") for v in docs_versions])) + +# Whitelist pattern for branches (set to None to ignore all branches) +smv_branch_whitelist = r"^master$" -html_logo = 'logo.png' -html_favicon = 'favicon.png' +# Whitelist pattern for remotes (set to None to use local branches only) +smv_remote_whitelist = None def setup(app): - app.add_config_value('recommonmark_config', { - 'enable_eval_rst': True - }, True) + app.add_config_value("recommonmark_config", {"enable_eval_rst": True}, True) app.add_transform(AutoStructify) - app.add_css_file('custom.css') + app.add_css_file("custom.css") diff --git a/adapter_docs/scripts/post_build.py b/adapter_docs/scripts/post_build.py new file mode 100644 index 0000000000..f247531fe5 --- /dev/null +++ b/adapter_docs/scripts/post_build.py @@ -0,0 +1,16 @@ +import os +import shutil +import sys + + +BUILD_DIR = sys.argv[1] + +for folder in os.listdir(BUILD_DIR): + path = os.path.join(BUILD_DIR, folder) + if folder == "master": + file_names = os.listdir(path) + for file_name in file_names: + shutil.move(os.path.join(path, file_name), BUILD_DIR) + os.rmdir(path) + else: + shutil.move(path, path.replace("adapters", "v")) diff --git a/setup.py b/setup.py index 73982f0d98..e313dfa64e 100644 --- a/setup.py +++ b/setup.py @@ -145,6 +145,7 @@ "sphinx==3.2.1", "sphinxext-opengraph==0.4.1", "sphinx-intl", + "sphinx-multiversion", "starlette", "tensorflow-cpu>=2.3", "tensorflow>=2.3", @@ -298,6 +299,7 @@ def run(self): "sphinx-copybutton", "sphinxext-opengraph", "sphinx-intl", + "sphinx-multiversion", ) # "docs" needs "all" to resolve all the references extras["docs"] = extras["all"] + extras["docs_specific"] diff --git a/src/transformers/adapters/model_mixin.py b/src/transformers/adapters/model_mixin.py index 658049ebd1..08b3423dcd 100644 --- a/src/transformers/adapters/model_mixin.py +++ b/src/transformers/adapters/model_mixin.py @@ -8,7 +8,6 @@ import torch from torch import nn -from ..models.auto.tokenization_auto import AutoTokenizer from .composition import AdapterCompositionBlock, Fuse, Stack, parse_composition from .configuration import AdapterConfig, AdapterFusionConfig, ModelAdaptersConfig, get_adapter_config_hash from .hub_mixin import PushAdapterToHubMixin @@ -555,6 +554,8 @@ def load_embeddings(self, path: str, name: str): Returns: a tokenizer if it ws saved with the embedding otherwise None """ + from ..models.auto.tokenization_auto import AutoTokenizer + if name in self.loaded_embeddings: raise ValueError("An embedding with the name {} already exists".format(name)) tokenizer = None diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index f5f8aeb0aa..e6d6d3fcad 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -62,6 +62,7 @@ "sphinx": "sphinx==3.2.1", "sphinxext-opengraph": "sphinxext-opengraph==0.4.1", "sphinx-intl": "sphinx-intl", + "sphinx-multiversion": "sphinx-multiversion", "starlette": "starlette", "tensorflow-cpu": "tensorflow-cpu>=2.3", "tensorflow": "tensorflow>=2.3",