From 9a5c3f19a2e826ee4731f4cfcd7d72d1e81fc68d Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 9 Sep 2020 22:02:11 +0100 Subject: [PATCH 1/4] :books: update documentation on loading jinja2 extension --- docs/level-12-use-template-engine-extensions/README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/level-12-use-template-engine-extensions/README.rst b/docs/level-12-use-template-engine-extensions/README.rst index 49435d86..6de882f1 100644 --- a/docs/level-12-use-template-engine-extensions/README.rst +++ b/docs/level-12-use-template-engine-extensions/README.rst @@ -23,7 +23,8 @@ Please also note that the following extensions are included by default: .. note:: - if you intend to use extensions for one off usage, please use '-e' cli option + if you intend to use extensions for one off usage, please use '-e' cli option. + for example: `moban -e jinja2=your_custom_jinja2_extension `_ Evaluation -------------------------------------------------------------------------------- From 6c17bdd10e82c0299771cf87138be484834c4b1b Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 10 Sep 2020 18:51:36 +0100 Subject: [PATCH 2/4] Use existing module as jinja2 filter test globals (#401) * :sparkles: the prototype to load existing filters, tests, globals from other ready made libraries, i.e. ansible. such a feature will remove the need of moban-ansible, moban-xxxx. * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :bug: remove bugs * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :bug: add moban-ansible dependencies * :books: update change log * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :microscope: regression tests plus documentation * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :lipstick: update coding style per latest black v20.8b1 * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :lipstick: update changelog yaml * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :lipstick: make yamllint happy * :green_heart: make unit test pass in py3.8 Co-authored-by: chfw --- .github/workflows/moban-update.yml | 2 +- .moban.cd/changelog.yml | 7 + .moban.cd/moban.yml | 4 +- .moban.d/moban_readme.jj2 | 26 +- CHANGELOG.rst | 8 + README.rst | 273 +++--------------- docs/conf.py | 2 +- .../.moban.yml | 3 + .../README.rst | 63 +++- .../a.template | 8 + moban/_version.py | 2 +- moban/main.py | 8 +- moban/plugins/jinja2/engine.py | 43 ++- setup.py | 2 +- .../test_command_line_options.py | 29 ++ tests/requirements.txt | 8 +- tests/test_docs.py | 3 + 17 files changed, 240 insertions(+), 251 deletions(-) diff --git a/.github/workflows/moban-update.yml b/.github/workflows/moban-update.yml index 52a82f48..1c6d5224 100644 --- a/.github/workflows/moban-update.yml +++ b/.github/workflows/moban-update.yml @@ -14,7 +14,7 @@ jobs: python-version: '3.7' - name: check changes run: | - pip install moban gitfs2 pypifs moban-jinja2-github + pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible make update git status git diff --exit-code diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 11b1ba32..65395c3c 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,13 @@ name: moban organisation: moremoban releases: + - changes: + - action: Fixed + details: + - Use any functions, any data structure of any python packages + as jinja2 filters, tests, globals + date: 04.09.2020 + version: 0.8.2 - changes: - action: Fixed details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 40595e10..fde9c3b4 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,8 +4,8 @@ organisation: moremoban author: chfw contact: wangc_2011@hotmail.com license: MIT -version: 0.8.1 -current_version: 0.8.1 +version: 0.8.2 +current_version: 0.8.2 release: 0.8.1 branch: master master: index diff --git a/.moban.d/moban_readme.jj2 b/.moban.d/moban_readme.jj2 index c4b186ef..6672769d 100644 --- a/.moban.d/moban_readme.jj2 +++ b/.moban.d/moban_readme.jj2 @@ -139,7 +139,31 @@ moban allows the injection of user preferred jinja2 extensions: $ moban -e jj2=jinja2_time.TimeExtension ... -Can I write my own jinja2 test, filter and/or globals + +Well, can I nick some existing functions as filters, tests? Or create a global from another library? +----------------------------------------------------------------------------------------------------- + +Sure, you can use the same '-e' syntax: + +.. code-block:: bash + + $ moban -e jinja2=filter:module.path.filter_function \ + jinja2=test:module.path.test_function \ + jinja2=global:identifier=module.path.variable + +In this case, you would have to include the external library in your own requirements.txt + +Here is an example: + +.. code-block:: bash + + $ moban -e jinja2=filter:moban.externals.file_system.url_join \ + jinja2=test:moban.externals.file_system.exists \ + jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION \ + -t "{{ 'a'|url_join('b')}} {{'b' is exists}}" + + +Can I write my own jinja2 test, filter and/or globals? ----------------------------------------------------------- moban allows the freedom of craftsmanship. Please refer to the docs for more diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6679df7a..6cb59fe2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.8.2 - 04.09.2020 +-------------------------------------------------------------------------------- + +**Fixed** + +#. Use any functions, any data structure of any python packages as jinja2 + filters, tests, globals + 0.8.1 - 04.09.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 367c8a31..6672769d 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ versions lower than 0.7.0 if you are still using python 2. Quick start ================================================================================ - +{% raw %} .. code-block:: bash $ export HELLO="world" @@ -72,6 +72,7 @@ Or simply $ HELLO="world" moban "{{HELLO}}" +{% endraw %} A bit formal example: @@ -88,12 +89,13 @@ Given data.yml as: and my.template as: - +{% raw %} .. code-block:: bash {{hello}} +{% endraw %} Please note that data.yml will take precedence over environment variables. @@ -137,7 +139,31 @@ moban allows the injection of user preferred jinja2 extensions: $ moban -e jj2=jinja2_time.TimeExtension ... -Can I write my own jinja2 test, filter and/or globals + +Well, can I nick some existing functions as filters, tests? Or create a global from another library? +----------------------------------------------------------------------------------------------------- + +Sure, you can use the same '-e' syntax: + +.. code-block:: bash + + $ moban -e jinja2=filter:module.path.filter_function \ + jinja2=test:module.path.test_function \ + jinja2=global:identifier=module.path.variable + +In this case, you would have to include the external library in your own requirements.txt + +Here is an example: + +.. code-block:: bash + + $ moban -e jinja2=filter:moban.externals.file_system.url_join \ + jinja2=test:moban.externals.file_system.exists \ + jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION \ + -t "{{ 'a'|url_join('b')}} {{'b' is exists}}" + + +Can I write my own jinja2 test, filter and/or globals? ----------------------------------------------------------- moban allows the freedom of craftsmanship. Please refer to the docs for more @@ -162,12 +188,13 @@ details. Here is an example: And you can use it within your jinja2 template, `mytest.jj2`: - +{% raw %} .. code-block:: python {{ 'abc' | base64encode }} +{% endraw %} Assume that the custom example was saved in `custom-jj2-plugin` @@ -182,22 +209,7 @@ Slim template syntax for jinja2 with `moban-slim `_ installed, -Given a data.json file with the following content - -.. code-block:: - - { - "person": { - "firstname": "Smith", - "lastname": "Jones", - }, - } - -.. code-block:: bash - - - $ moban --template-type slim -c data.json "{{person.firstname}} {{person.lastname}}" - Smith Jones +{% include "slim_example.rst.jj2" %} Handlebars.js template ---------------------------- @@ -205,80 +217,7 @@ Handlebars.js template With `moban-handlebars `_ installed, -Given a data.json file with the following content - -.. code-block:: - - { - "person": { - "firstname": "Yehuda", - "lastname": "Katz", - }, - } - - -.. code-block:: bash - - - $ moban --template-type handlebars -c data.json "{{person.firstname}} {{person.lastname}}" - Yehuda Katz - -For `handlebars.js` users, yes, the example was copied from handlebarjs.com. The -aim is to show off what we can do. - -Let's continue with a bit more fancy feature: - - - -.. code-block:: bash - - $ moban --template-type handlebars -c data.json "{{#with person}}{{firstname}} {{lastname}} {{/with}}" - - -Moban's way of `pybar3 usage `_: - -Let's save the following file a `script.py` under `helper_and_partial` folder: - -.. code-block:: python - - from moban_handlebars.api import Helper, register_partial - - register_partial('header', '

People

') - - - @Helper('list') - def _list(this, options, items): - result = [u'
    '] - for thing in items: - result.append(u'
  • ') - result.extend(options['fn'](thing)) - result.append(u'
    • ') - result.append(u'
') - return result - -And given `data.json` reads as the following: - -.. code-block:: - - { - "people":[ - {"name": "Bill", "age": 100}, - {"name": "Bob", "age": 90}, - {"name": "Mark", "age": 25} - ] - } - -Let's invoke handlebar template: - - -.. code-block:: bash - - $ moban --template-type hbs -pd helper_and_partial -c data.json "{{>header}}{{#list people}}{{name}} {{age}}{{/list}}" - Handlebars-ing {{>header}... to moban.output - Handlebarsed 1 file. - $ cat moban.output -

People

  • Bill 100
  • Bob 90
  • Mark 25
- +{% include "handlebars_example.rst.jj2" %} Velocity template ---------------------------- @@ -286,46 +225,7 @@ Velocity template With `moban-velocity `_ installed, -Given the following data.json: - -.. code-block:: - - {"people": - [ - {"name": "Bill", "age": 100}, - {"name": "Bob", "age": 90}, - {"name": "Mark", "age": 25} - ] - } - -And given the following velocity.template: - -.. code-block:: - - Old people: - #foreach ($person in $people) - #if($person.age > 70) - $person.name - #end - #end - - Third person is $people[2].name - -**moban** can do the template: - -.. code-block:: bash - - $ moban --template-type velocity -c data.json -t velocity.template - Old people: - - Bill - - Bob - - - Third person is Mark - - +{% include "velocity_example.rst.jj2" %} Can I write my own template engine? -------------------------------------- @@ -361,63 +261,7 @@ TOML data format `moban-anyconfig `_ should be installed first. -Given the following toml file, sample.toml: - -.. code-block:: - - title = "TOML Example" - [owner] - name = "Tom Preston-Werner" - - -You can do: - - -.. code-block:: bash - - $ moban -c sample.toml "{{owner.name}} made {{title}}" - Tom Preston-Werner made TOML Example - -Not limited to toml, you can supply moban with the following data formats: - -.. csv-table:: Always supported formats, quoting from python-anyconfig - :header: "Format", "Type", "Requirement" - :widths: 15, 10, 40 - - JSON, json, ``json`` (standard lib) or ``simplejson`` - Ini-like, ini, ``configparser`` (standard lib) - Pickle, pickle, ``pickle`` (standard lib) - XML, xml, ``ElementTree`` (standard lib) - Java properties, properties, None (native implementation with standard lib) - B-sh, shellvars, None (native implementation with standard lib) - -For any of the following data formats, you elect to install by yourself. - -.. csv-table:: Supported formats by pluggable backend modules - :header: "Format", "Type", "Required backend" - :widths: 15, 10, 40 - - Amazon Ion, ion, ``anyconfig-ion-backend`` - BSON, bson, ``anyconfig-bson-backend`` - CBOR, cbor, ``anyconfig-cbor-backend`` or ``anyconfig-cbor2-backend`` - ConifgObj, configobj, ``anyconfig-configobj-backend`` - MessagePack, msgpack, ``anyconfig-msgpack-backend`` - -Or you could choose to install all: - -.. code-block:: bash - - $ pip install moban-anyconfig[all-backends] - -**Why not to use python-anyconfig itself, but yet another package?** - -moban gives you a promise of any location which `python-anyconfig` does not support. - -**Why do it mean 'any location'?** - -Thanks to `pyfilesystem 2 `_, -moban is able to read data back from `git repo `_, `pypi `_ package, `http(s) `_, zip, -tar, ftp, `s3 `_ or `you name it `_. +{% include "anyconfig_example.rst.jj2" %} Templates and configuration files over HTTP(S) @@ -425,17 +269,7 @@ Templates and configuration files over HTTP(S) `httpfs `_ should be installed first. -With httpfs, `moban`_ can access any files over http(s) as its -template or data file: - -.. code-block:: bash - - $ moban -t 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/_version.py.jj2'\ - -c 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/data.yml'\ - -o _version.py - - -.. _moban: https://github.com/moremoban/moban +{% include "httpfs_example.rst.jj2" %} In an edge case, if github repo's public url is given, this github repo shall not have sub repos. This library will fail to @@ -447,20 +281,7 @@ Templates and configuration files in a git repo `gitfs2 `_ is optional since v0.7.0 but was installed by default since v0.6.1 -You can do the following with moban: - -.. code-block:: bash - - $ moban -t 'git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2' \ - -c 'git://github.com/moremoban/pypi-mobans.git!/config/data.yml' \ - -o _version.py - Info: Found repo in /Users/jaska/Library/Caches/gitfs2/repos/pypi-mobans - Templating git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2 to _version.py - Templated 1 file. - $ cat _version.py - __version__ = "0.1.1rc3" - __author__ = "C.W." - +{% include "gitfs2_example.rst.jj2" %} Templates and configuration files in a python package ================================================================================ @@ -468,22 +289,7 @@ Templates and configuration files in a python package `pypifs `_ is optional since v0.7.0 but was installed by default since v0.6.1 -You can do the following with moban: - -.. code-block:: bash - - $ moban -t 'pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2' \ - -c 'pypi://pypi-mobans-pkg/resources/config/data.yml' \ - -o _version.py - Collecting pypi-mobans-pkg - .... - Installing collected packages: pypi-mobans-pkg - Successfully installed pypi-mobans-pkg-0.0.7 - Templating pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2 to _version.py - Templated 1 file. - $ cat _version.py - __version__ = "0.1.1rc3" - __author__ = "C.W." +{% include "pypi_example.rst.jj2" %} Work with S3 and other cloud based file systems ================================================================================ @@ -495,7 +301,7 @@ Please install `fs-s3fs `_:: Then you can access your files in s3 bucket: - +{% raw %} .. code-block:: bash @@ -505,6 +311,7 @@ Then you can access your files in s3 bucket: $ cat moban.output world +{% endraw %} Where the configuration sits in a s3 bucket, the output is a file in a zip. The content of s3data.yaml is: diff --git a/docs/conf.py b/docs/conf.py index 5403c914..1ce87d1d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ copyright = '2017-2020 Onni Software Ltd.' author = 'chfw' # The short X.Y version -version = '0.8.1' +version = '0.8.2' # The full version, including alpha/beta/rc tags release = '0.8.1' diff --git a/docs/level-12-use-template-engine-extensions/.moban.yml b/docs/level-12-use-template-engine-extensions/.moban.yml index 63b4bc64..0c3fd342 100644 --- a/docs/level-12-use-template-engine-extensions/.moban.yml +++ b/docs/level-12-use-template-engine-extensions/.moban.yml @@ -4,3 +4,6 @@ targets: extensions: jinja2: - jinja2.ext.with_ + - filter:moban.externals.file_system.url_join + - test:moban.externals.file_system.exists + - global:description=moban.constants.PROGRAM_DESCRIPTION diff --git a/docs/level-12-use-template-engine-extensions/README.rst b/docs/level-12-use-template-engine-extensions/README.rst index 6de882f1..9b63f23e 100644 --- a/docs/level-12-use-template-engine-extensions/README.rst +++ b/docs/level-12-use-template-engine-extensions/README.rst @@ -1,6 +1,12 @@ Level 12: use template engine extensions ================================================================================ +There are three possible ways to provide extensions for your template engine. +Let's take jinja2 as an example. + +1. Ready made extensions +----------------------------- + jinja2 comes with a lot of extensions. In order not to be the blocker in the middle, **extensions** is allowed in moban file to initialize jinja2 engine with desired extensions. Two extensions, expression-statement and loop-controls @@ -21,10 +27,61 @@ For example:: Please also note that the following extensions are included by default: `jinja2.ext.do`, `jinja2.ext.loopcontrols` -.. note:: - if you intend to use extensions for one off usage, please use '-e' cli option. - for example: `moban -e jinja2=your_custom_jinja2_extension `_ +**Command line** + +if you intend to use extensions for one off usage, please use '-e' cli option. +for example: `moban -e jinja2=your_custom_jinja2_extension `_ + + +2. Ad-hoc declaration +----------------------------- + +Let's say you are fond of some existing functions, for example, ansible's combine +filter. With moban, you can immediately include it for your template via the following +syntax: + +.. code-block:: + + extensions: + jinja2: + - filter:module.path.filter_function + - test:module.path.test_function + - global:identifier=module.path.variable + +For example:: + + extensions: + jinja2: + - filter:ansible.plugins.filter.core.combine + - test:moban.externals.file_system.exists + +**Command line** + +.. code-block:: bash + + $ moban -e jinja2=filter:module.path.filter_function jinja2=test:module.path.test_function jinja2=global:identifier=module.path.variable + +you can do this:: + + $ moban -e jinja2=filter:module.path.filter_function \ + jinja2=test:module.path.test_function \ + jinja2=global:identifier=module.path.variable + + +3. Make your own extensions +-------------------------------- + +You can choose to write an extension for the template type of your choice. +For example, you can write a reusable extension for jinja2. moban will be +able to load it as it is. + +If you decide that you only want to write them for moban but for your own +use, you can follow `Level 7: Custom jinja filters, tests and globals` and +write your own. When you would like to make yours avaiable for all moban +users, you can follow `moban-jinja2-github `_ and +`moban-ansible `_ + Evaluation -------------------------------------------------------------------------------- diff --git a/docs/level-12-use-template-engine-extensions/a.template b/docs/level-12-use-template-engine-extensions/a.template index da43df97..2d229231 100644 --- a/docs/level-12-use-template-engine-extensions/a.template +++ b/docs/level-12-use-template-engine-extensions/a.template @@ -1,3 +1,11 @@ {% for _ in range(1,5) %} {{ hello }} {% endfor %} +{% if 'b.template' is exists %} +b.template exists +{% endif %} +{% if 'alien' is exists %} +we are not the only intelligent life in the universe +{% endif %} +{{ 'a' | url_join('b')}} +{{ description }} diff --git a/moban/_version.py b/moban/_version.py index ef44f972..ad5224e3 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.8.1" +__version__ = "0.8.2" __author__ = "chfw" diff --git a/moban/main.py b/moban/main.py index be2ef2d4..3d64aa47 100644 --- a/moban/main.py +++ b/moban/main.py @@ -8,6 +8,7 @@ :license: MIT License, see LICENSE for more details """ +import re import sys import logging import argparse @@ -299,8 +300,11 @@ def handle_custom_extensions(list_of_definitions): user_extensions = defaultdict(set) if list_of_definitions: for definition in list_of_definitions: - key, value = definition.split("=") - user_extensions[key].add(value) + result = re.match("(.*?)=(.*)", definition) + if result: + key, value = result.group(1), result.group(2) + user_extensions[key].add(value) + ENGINES.register_extensions(user_extensions) diff --git a/moban/plugins/jinja2/engine.py b/moban/plugins/jinja2/engine.py index e827b78e..6550b4a4 100644 --- a/moban/plugins/jinja2/engine.py +++ b/moban/plugins/jinja2/engine.py @@ -86,15 +86,23 @@ def __init__(self, template_fs, options: Dict = None): ) self.template_loader = template_loader + filters = tests = _globals = {} if options: if "extensions" in options: - extensions = options.pop("extensions") + raw_extensions = options.pop("extensions") + ( + filters, + tests, + _globals, + extensions, + ) = parse_extensioins(raw_extensions) if is_extension_list_valid(extensions): # because it is modified here env_params["extensions"] += extensions import_module_of_extension(extensions) env_params.update(options) self.jj2_environment = Environment(**env_params) + for filter_name, filter_function in FILTERS.get_all(): self.jj2_environment.filters[filter_name] = filter_function @@ -104,6 +112,11 @@ def __init__(self, template_fs, options: Dict = None): for global_name, dict_obj in GLOBALS.get_all(): self.jj2_environment.globals[global_name] = dict_obj + # can override others too + self.jj2_environment.filters.update(filters) + self.jj2_environment.tests.update(tests) + self.jj2_environment.globals.update(_globals) + def get_template(self, template_file): """ :param str template_file: the template file name that appeared in moban @@ -175,3 +188,31 @@ def import_module_of_extension(extensions): def strip_off_trailing_new_lines(content): return re.sub(r"(\n\s+)+$", r"\n", content) + + +def parse_extensioins(extensions): + from jinja2.utils import import_string + + filters = {} + tests = {} + _globals = {} + jinja2_extensions = [] + + for extension in extensions: + if extension.startswith("test:"): + test_function_string = extension.replace("test:", "").strip() + test_function = import_string(test_function_string) + tests[test_function.__name__] = test_function + elif extension.startswith("filter:"): + filter_function_string = extension.replace("filter:", "").strip() + filter_function = import_string(filter_function_string) + filters[filter_function.__name__] = filter_function + elif extension.startswith("global:"): + a_global = extension.replace("global:", "").strip() + identifier, global_string = a_global.split("=") + the_global = import_string(global_string) + _globals[identifier] = the_global + else: + jinja2_extensions.append(extension) + + return filters, tests, _globals, jinja2_extensions diff --git a/setup.py b/setup.py index 99d27d1d..3dfe18d0 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ NAME = "moban" AUTHOR = "chfw" -VERSION = "0.8.1" +VERSION = "0.8.2" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 9250e009..6056b6ff 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -588,3 +588,32 @@ def test_render_file_stdout(): main() eq_(fake_stdout.getvalue(), "world\n") + + +def test_custom_jinja2_filters_tests(): + config_file = "config.yaml" + with open(config_file, "w") as f: + f.write("hello: world") + template_file = "t.jj2" + with open(template_file, "w") as f: + f.write("{{hello}}") + test_args = [ + "moban", + "-e", + "jinja2=filter:moban.externals.file_system.url_join", + "jinja2=test:moban.externals.file_system.exists", + "jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION", + "-t", + "{{'a'|url_join('b')}} {{'b' is exists}}{{ description }}", + ] + with patch.object(sys, "argv", test_args): + with patch("sys.stdout", new_callable=StringIO) as fake_stdout: + from moban.main import main + + expected_output = ( + "a/b False" + + "Static text generator using " + + "any template, any data and any location.\n" + ) + main() + eq_(fake_stdout.getvalue(), expected_output) diff --git a/tests/requirements.txt b/tests/requirements.txt index e1b88a91..505db119 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -4,13 +4,11 @@ coverage mock yamllint flake8 -black;python_version>="3.6" -isort;python_version>="3.6" +black +isort moban-handlebars pypi-mobans-pkg==0.0.12 -# arrow 0.14.0 doesnt support Python 3.4 -arrow<0.14.0;python_version=="3.4" -arrow;python_version!="3.4" +arrow jinja2_time pypifs gitfs2 diff --git a/tests/test_docs.py b/tests/test_docs.py index 92053e20..1b7f30fb 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -168,6 +168,9 @@ def test_level_12(self): world world world + b.template exists + a/b + Static text generator using any template, any data and any location. """ expected_b = """ 142 From 8b09eafea854ebf5b1c966d71b88936c2117b61e Mon Sep 17 00:00:00 2001 From: jaska Date: Thu, 10 Sep 2020 20:31:10 +0100 Subject: [PATCH 3/4] Bug fix (#402) * :bug: fix readme and :books: document trouble shooting guide * :books: update migration guide * This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst * :lipstick: report exception from lower lane Co-authored-by: chfw --- .moban.d/moban_readme.jj2 | 12 +- README.rst | 258 ++++++++++++++++++++++++++++---- docs/index.rst | 1 + docs/migration-notes.rst | 17 +++ docs/trouble-shooting-guide.rst | 9 ++ moban/core/moban_factory.py | 3 +- moban/plugins/jinja2/engine.py | 1 + 7 files changed, 265 insertions(+), 36 deletions(-) create mode 100644 docs/trouble-shooting-guide.rst diff --git a/.moban.d/moban_readme.jj2 b/.moban.d/moban_readme.jj2 index 6672769d..0fa78444 100644 --- a/.moban.d/moban_readme.jj2 +++ b/.moban.d/moban_readme.jj2 @@ -40,15 +40,6 @@ package. `moban.plugins.jinja2.filters.github` is moved to moban-jinja2-github package Please install them for backward compatibility. -From 2020 onwards, minimum requirement is Python 3.6 - - -For existing moban users, python 2 support has been dropped. Please stay with -versions lower than 0.7.0 if you are still using python 2. - - - - Quick start ================================================================================ @@ -155,13 +146,14 @@ In this case, you would have to include the external library in your own require Here is an example: +{% raw %} .. code-block:: bash $ moban -e jinja2=filter:moban.externals.file_system.url_join \ jinja2=test:moban.externals.file_system.exists \ jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION \ -t "{{ 'a'|url_join('b')}} {{'b' is exists}}" - +{% endraw %} Can I write my own jinja2 test, filter and/or globals? ----------------------------------------------------------- diff --git a/README.rst b/README.rst index 6672769d..734c9c05 100644 --- a/README.rst +++ b/README.rst @@ -40,19 +40,10 @@ package. `moban.plugins.jinja2.filters.github` is moved to moban-jinja2-github package Please install them for backward compatibility. -From 2020 onwards, minimum requirement is Python 3.6 - - -For existing moban users, python 2 support has been dropped. Please stay with -versions lower than 0.7.0 if you are still using python 2. - - - - Quick start ================================================================================ -{% raw %} + .. code-block:: bash $ export HELLO="world" @@ -72,7 +63,6 @@ Or simply $ HELLO="world" moban "{{HELLO}}" -{% endraw %} A bit formal example: @@ -89,13 +79,12 @@ Given data.yml as: and my.template as: -{% raw %} + .. code-block:: bash {{hello}} -{% endraw %} Please note that data.yml will take precedence over environment variables. @@ -155,6 +144,7 @@ In this case, you would have to include the external library in your own require Here is an example: + .. code-block:: bash $ moban -e jinja2=filter:moban.externals.file_system.url_join \ @@ -162,7 +152,6 @@ Here is an example: jinja2=global:description=moban.constants.PROGRAM_DESCRIPTION \ -t "{{ 'a'|url_join('b')}} {{'b' is exists}}" - Can I write my own jinja2 test, filter and/or globals? ----------------------------------------------------------- @@ -188,13 +177,12 @@ details. Here is an example: And you can use it within your jinja2 template, `mytest.jj2`: -{% raw %} + .. code-block:: python {{ 'abc' | base64encode }} -{% endraw %} Assume that the custom example was saved in `custom-jj2-plugin` @@ -209,7 +197,22 @@ Slim template syntax for jinja2 with `moban-slim `_ installed, -{% include "slim_example.rst.jj2" %} +Given a data.json file with the following content + +.. code-block:: + + { + "person": { + "firstname": "Smith", + "lastname": "Jones", + }, + } + +.. code-block:: bash + + + $ moban --template-type slim -c data.json "{{person.firstname}} {{person.lastname}}" + Smith Jones Handlebars.js template ---------------------------- @@ -217,7 +220,80 @@ Handlebars.js template With `moban-handlebars `_ installed, -{% include "handlebars_example.rst.jj2" %} +Given a data.json file with the following content + +.. code-block:: + + { + "person": { + "firstname": "Yehuda", + "lastname": "Katz", + }, + } + + +.. code-block:: bash + + + $ moban --template-type handlebars -c data.json "{{person.firstname}} {{person.lastname}}" + Yehuda Katz + +For `handlebars.js` users, yes, the example was copied from handlebarjs.com. The +aim is to show off what we can do. + +Let's continue with a bit more fancy feature: + + + +.. code-block:: bash + + $ moban --template-type handlebars -c data.json "{{#with person}}{{firstname}} {{lastname}} {{/with}}" + + +Moban's way of `pybar3 usage `_: + +Let's save the following file a `script.py` under `helper_and_partial` folder: + +.. code-block:: python + + from moban_handlebars.api import Helper, register_partial + + register_partial('header', '

People

') + + + @Helper('list') + def _list(this, options, items): + result = [u'
    '] + for thing in items: + result.append(u'
  • ') + result.extend(options['fn'](thing)) + result.append(u'
    • ') + result.append(u'
') + return result + +And given `data.json` reads as the following: + +.. code-block:: + + { + "people":[ + {"name": "Bill", "age": 100}, + {"name": "Bob", "age": 90}, + {"name": "Mark", "age": 25} + ] + } + +Let's invoke handlebar template: + + +.. code-block:: bash + + $ moban --template-type hbs -pd helper_and_partial -c data.json "{{>header}}{{#list people}}{{name}} {{age}}{{/list}}" + Handlebars-ing {{>header}... to moban.output + Handlebarsed 1 file. + $ cat moban.output +

People

  • Bill 100
  • Bob 90
  • Mark 25
+ Velocity template ---------------------------- @@ -225,7 +301,46 @@ Velocity template With `moban-velocity `_ installed, -{% include "velocity_example.rst.jj2" %} +Given the following data.json: + +.. code-block:: + + {"people": + [ + {"name": "Bill", "age": 100}, + {"name": "Bob", "age": 90}, + {"name": "Mark", "age": 25} + ] + } + +And given the following velocity.template: + +.. code-block:: + + Old people: + #foreach ($person in $people) + #if($person.age > 70) + $person.name + #end + #end + + Third person is $people[2].name + +**moban** can do the template: + +.. code-block:: bash + + $ moban --template-type velocity -c data.json -t velocity.template + Old people: + + Bill + + Bob + + + Third person is Mark + + Can I write my own template engine? -------------------------------------- @@ -261,7 +376,63 @@ TOML data format `moban-anyconfig `_ should be installed first. -{% include "anyconfig_example.rst.jj2" %} +Given the following toml file, sample.toml: + +.. code-block:: + + title = "TOML Example" + [owner] + name = "Tom Preston-Werner" + + +You can do: + + +.. code-block:: bash + + $ moban -c sample.toml "{{owner.name}} made {{title}}" + Tom Preston-Werner made TOML Example + +Not limited to toml, you can supply moban with the following data formats: + +.. csv-table:: Always supported formats, quoting from python-anyconfig + :header: "Format", "Type", "Requirement" + :widths: 15, 10, 40 + + JSON, json, ``json`` (standard lib) or ``simplejson`` + Ini-like, ini, ``configparser`` (standard lib) + Pickle, pickle, ``pickle`` (standard lib) + XML, xml, ``ElementTree`` (standard lib) + Java properties, properties, None (native implementation with standard lib) + B-sh, shellvars, None (native implementation with standard lib) + +For any of the following data formats, you elect to install by yourself. + +.. csv-table:: Supported formats by pluggable backend modules + :header: "Format", "Type", "Required backend" + :widths: 15, 10, 40 + + Amazon Ion, ion, ``anyconfig-ion-backend`` + BSON, bson, ``anyconfig-bson-backend`` + CBOR, cbor, ``anyconfig-cbor-backend`` or ``anyconfig-cbor2-backend`` + ConifgObj, configobj, ``anyconfig-configobj-backend`` + MessagePack, msgpack, ``anyconfig-msgpack-backend`` + +Or you could choose to install all: + +.. code-block:: bash + + $ pip install moban-anyconfig[all-backends] + +**Why not to use python-anyconfig itself, but yet another package?** + +moban gives you a promise of any location which `python-anyconfig` does not support. + +**Why do it mean 'any location'?** + +Thanks to `pyfilesystem 2 `_, +moban is able to read data back from `git repo `_, `pypi `_ package, `http(s) `_, zip, +tar, ftp, `s3 `_ or `you name it `_. Templates and configuration files over HTTP(S) @@ -269,7 +440,17 @@ Templates and configuration files over HTTP(S) `httpfs `_ should be installed first. -{% include "httpfs_example.rst.jj2" %} +With httpfs, `moban`_ can access any files over http(s) as its +template or data file: + +.. code-block:: bash + + $ moban -t 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/_version.py.jj2'\ + -c 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/data.yml'\ + -o _version.py + + +.. _moban: https://github.com/moremoban/moban In an edge case, if github repo's public url is given, this github repo shall not have sub repos. This library will fail to @@ -281,7 +462,20 @@ Templates and configuration files in a git repo `gitfs2 `_ is optional since v0.7.0 but was installed by default since v0.6.1 -{% include "gitfs2_example.rst.jj2" %} +You can do the following with moban: + +.. code-block:: bash + + $ moban -t 'git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2' \ + -c 'git://github.com/moremoban/pypi-mobans.git!/config/data.yml' \ + -o _version.py + Info: Found repo in /Users/jaska/Library/Caches/gitfs2/repos/pypi-mobans + Templating git://github.com/moremoban/pypi-mobans.git!/templates/_version.py.jj2 to _version.py + Templated 1 file. + $ cat _version.py + __version__ = "0.1.1rc3" + __author__ = "C.W." + Templates and configuration files in a python package ================================================================================ @@ -289,7 +483,22 @@ Templates and configuration files in a python package `pypifs `_ is optional since v0.7.0 but was installed by default since v0.6.1 -{% include "pypi_example.rst.jj2" %} +You can do the following with moban: + +.. code-block:: bash + + $ moban -t 'pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2' \ + -c 'pypi://pypi-mobans-pkg/resources/config/data.yml' \ + -o _version.py + Collecting pypi-mobans-pkg + .... + Installing collected packages: pypi-mobans-pkg + Successfully installed pypi-mobans-pkg-0.0.7 + Templating pypi://pypi-mobans-pkg/resources/templates/_version.py.jj2 to _version.py + Templated 1 file. + $ cat _version.py + __version__ = "0.1.1rc3" + __author__ = "C.W." Work with S3 and other cloud based file systems ================================================================================ @@ -301,7 +510,7 @@ Please install `fs-s3fs `_:: Then you can access your files in s3 bucket: -{% raw %} + .. code-block:: bash @@ -311,7 +520,6 @@ Then you can access your files in s3 bucket: $ cat moban.output world -{% endraw %} Where the configuration sits in a s3 bucket, the output is a file in a zip. The content of s3data.yaml is: diff --git a/docs/index.rst b/docs/index.rst index 64a5f623..777b1b9d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,6 +59,7 @@ Migration Notes .. toctree:: + trouble-shooting-guide migration-note diff --git a/docs/migration-notes.rst b/docs/migration-notes.rst index 306d675c..1b704715 100644 --- a/docs/migration-notes.rst +++ b/docs/migration-notes.rst @@ -1,3 +1,20 @@ +Migrate to 0.8.x +================================================================================ + +In version 0.8.0, `moban.plugins.jinja2.tests.files` is moved to moban-ansible +package. `moban.plugins.jinja2.filters.github` is moved to moban-jinja2-github +package Please install them for backward compatibility. + + +Migrate to 0.7.x +================================================================================ + +From 2020 onwards, minimum requirement is Python 3.6 + + +For existing moban users, python 2 support has been dropped. Please stay with +versions lower than 0.7.0 if you are still using python 2. + Migrate to 0.6.x ================================================================================ diff --git a/docs/trouble-shooting-guide.rst b/docs/trouble-shooting-guide.rst new file mode 100644 index 00000000..bd8a970b --- /dev/null +++ b/docs/trouble-shooting-guide.rst @@ -0,0 +1,9 @@ +Trouble shooting guide +========================== + +1. Why a file was not templated but copied instead? + + It has been coded so that template engine can choose to pass on the template if it failed to handle. Moban will take over + and use default 'copy' action. + + In order to find out what went wrong, you can use '-vvv' to enable all logs to assist you. diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index 7cbd6ec3..da713376 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -237,7 +237,8 @@ def _render_with_finding_template_first(self, template_file_index): ) self.templated_count += 1 self.file_count += 1 - except exceptions.PassOn: + except exceptions.PassOn as e: + LOG.info(e) for (data_file, output) in data_output_pairs: self.fall_out_targets.append( TemplateTarget( diff --git a/moban/plugins/jinja2/engine.py b/moban/plugins/jinja2/engine.py index 6550b4a4..57188aff 100644 --- a/moban/plugins/jinja2/engine.py +++ b/moban/plugins/jinja2/engine.py @@ -143,6 +143,7 @@ def get_template(self, template_file): except fs.errors.ResourceNotFound: return self.jj2_environment.from_string(template_file) except (UnicodeDecodeError, TemplateSyntaxError) as e: + LOG.error(e) raise exceptions.PassOn(str(e)) def get_template_from_string(self, string): From c852a35d69e16b0e0e4723b195a4925a2e90c40c Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 10 Sep 2020 20:32:09 +0100 Subject: [PATCH 4/4] :egg: :ferris_wheel: release 0.8.2 --- .moban.cd/moban.yml | 2 +- docs/conf.py | 2 +- setup.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index fde9c3b4..121b8776 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -6,7 +6,7 @@ contact: wangc_2011@hotmail.com license: MIT version: 0.8.2 current_version: 0.8.2 -release: 0.8.1 +release: 0.8.2 branch: master master: index command_line_interface: "moban" diff --git a/docs/conf.py b/docs/conf.py index 1ce87d1d..e773433d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,7 @@ # The short X.Y version version = '0.8.2' # The full version, including alpha/beta/rc tags -release = '0.8.1' +release = '0.8.2' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 3dfe18d0..19bdb78b 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ "General purpose static text generator" ) URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.8.1.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.8.2.tar.gz" % URL FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -97,8 +97,8 @@ } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.8.1 " + - "Find 0.8.1 in changelog for more details") +GS_COMMAND = ("gs moban v0.8.2 " + + "Find 0.8.2 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = (