From 2d2831d4c89322b3448022d59bea8df78a4974e3 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 11 Mar 2021 00:00:53 +0100 Subject: [PATCH 01/40] setup: Start of a heresy. Renamed to Juiced Jinja2 command-line tool (jj2cli). The upstream [1] appears to have fallen in disrepair. In the meantime, this fork has implemented additional functionality and alternative (and arguably more flexible) approaches to some use cases. At this point, a merge with the upstream is unlikely to happen and it will become even more likely with each future commit. Starting with this commit, this fork will be developed independently and under a new name to reflect the additional capabilities (and incompatibilities). [1] https://github.com/kolypto/j2cli --- j2cli/__init__.py | 13 ---- j2cli/extras/__init__.py | 1 - setup.py | 72 ++++++++++++++--------- src/jj2cli/__init__.py | 13 ++++ {j2cli => src/jj2cli}/cli.py | 12 ++-- {j2cli => src/jj2cli}/context.py | 0 {j2cli/extras => src/jj2cli}/customize.py | 0 {j2cli/extras => src/jj2cli}/filters.py | 0 8 files changed, 63 insertions(+), 48 deletions(-) delete mode 100644 j2cli/__init__.py delete mode 100644 j2cli/extras/__init__.py create mode 100644 src/jj2cli/__init__.py rename {j2cli => src/jj2cli}/cli.py (95%) rename {j2cli => src/jj2cli}/context.py (100%) rename {j2cli/extras => src/jj2cli}/customize.py (100%) rename {j2cli/extras => src/jj2cli}/filters.py (100%) diff --git a/j2cli/__init__.py b/j2cli/__init__.py deleted file mode 100644 index bc39e44..0000000 --- a/j2cli/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env python - -""" j2cli main file """ -import pkg_resources - -__author__ = "Mark Vartanyan" -__email__ = "kolypto@gmail.com" -__version__ = pkg_resources.get_distribution('j2cli').version - -from j2cli.cli import main - -if __name__ == '__main__': - main() diff --git a/j2cli/extras/__init__.py b/j2cli/extras/__init__.py deleted file mode 100644 index 8b2e53c..0000000 --- a/j2cli/extras/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import filters diff --git a/setup.py b/setup.py index 5733ee5..1737415 100755 --- a/setup.py +++ b/setup.py @@ -1,58 +1,74 @@ #!/usr/bin/env python -""" j2cli - Jinja2 Command-Line Tool -================================ +""" # jj2cli - Juiced Jinja2 command-line tool -`j2cli` is a command-line tool for templating in shell-scripts, -leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. +`jj2cli` (previously `j2cli`) is a command-line tool for templating in +shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) +library. Features: -* Jinja2 templating -* INI, YAML, JSON data sources supported -* Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) +* Jinja2 templating with support +* Support for data sources in various formats (ini, yaml, json, env) +* Mixing and matching data sources +* Template dependency analysis -Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) +Inspired by [kolypto/j2cli](https://github.com/kolypto/j2cli) and +[mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli). """ from setuptools import setup, find_packages -import sys +from sys import version_info as PYVER -# PyYAML 3.11 was the last to support Python 2.6 -# This code limits pyyaml version for older pythons -pyyaml_version = 'pyyaml >= 3.10' # fresh -if sys.version_info[:2] == (2, 6): - pyyaml_version = 'pyyaml<=3.11' + +### Compatibility packages. +packages_compat = [] +# Jinja2 +if PYVER < (2, 7) or (3, 0) <= PYVER < (3, 5): + packages_compat.append('jinja2 ~= 2.10.0') +else: + packages_compat.append('jinja2 ~= 2.11.0') + +### Packages for optional functionality. +packages_extra = [] +# yaml support +if PYVER < (2, 7) or (2, 7) < PYVER < (3, 4): + # XXX: Python2.6 + packages_extra.append(('yaml', 'pyyaml <= 3.11')) +else: + packages_extra.append(('yaml', 'pyyaml > 5.4')) setup( - name='j2cli', - version='0.3.12b', - author='Mark Vartanyan', - author_email='kolypto@gmail.com', + name='jj2cli', + version='0.4.0', + author='Manolis Stamatogiannakis', + author_email='mstamat@gmail.com', - url='https://github.com/kolypto/j2cli', + url='https://github.com/m000/j2cli', # XXX: fix before release license='BSD', - description='Command-line interface to Jinja2 for templating in shell scripts.', + description='Juiced Jinja2 command-line tool.', long_description=__doc__, # can't do open('README.md').read() because we're describing self long_description_content_type='text/markdown', keywords=['Jinja2', 'templating', 'command-line', 'CLI'], - packages=find_packages(), + packages=find_packages('src'), + package_dir={'': 'src'}, + #py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + scripts=[], entry_points={ 'console_scripts': [ - 'j2 = j2cli:main', + 'j2 = jj2cli:render', # temporarily keep the old entry point + 'jj2 = jj2cli:render', ] }, install_requires=[ - 'jinja2 >= 2.7.2', - 'six >= 1.10', + 'six >= 1.13', + packages_compat, ], - extras_require={ - 'yaml': [pyyaml_version,] - }, - include_package_data=True, + extras_require=dict(packages_extra), zip_safe=False, test_suite='nose.collector', diff --git a/src/jj2cli/__init__.py b/src/jj2cli/__init__.py new file mode 100644 index 0000000..acfebcb --- /dev/null +++ b/src/jj2cli/__init__.py @@ -0,0 +1,13 @@ +#! /usr/bin/env python + +""" j2cli main file """ +import pkg_resources + +__author__ = "Manolis Stamatogiannakis" +__email__ = "mstamat@gmail.com" +__version__ = pkg_resources.get_distribution('jj2cli').version + +from jj2cli.cli import main + +if __name__ == '__main__': + main() diff --git a/j2cli/cli.py b/src/jj2cli/cli.py similarity index 95% rename from j2cli/cli.py rename to src/jj2cli/cli.py index f900627..df18981 100644 --- a/j2cli/cli.py +++ b/src/jj2cli/cli.py @@ -5,14 +5,14 @@ import jinja2 import jinja2.loaders -from . import __version__ import imp, inspect +from . import __version__ +from . import filters from .context import FORMATS from .context import parse_data_spec, read_context_data2, dict_update_deep -from .extras import filters -from .extras.customize import CustomizationModule +from .customize import CustomizationModule # available log levels, adjusted with -v at command line LOGLEVELS = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] @@ -107,12 +107,12 @@ def render_command(argv): """ formats_names = list(FORMATS.keys()) parser = argparse.ArgumentParser( - description='Command-line interface to Jinja2 for templating in shell scripts.', + description='Renders Jinja2 templates from the command line.', epilog='', formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument('-V', '--version', action='version', - version='j2cli {0}, Jinja2 {1}'.format(__version__, jinja2.__version__)) + version='jj2cli {0}, Jinja2 {1}'.format(__version__, jinja2.__version__)) parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase verbosity.') parser.add_argument('-f', '--fallback-format', default='ini', choices=formats_names, @@ -123,7 +123,7 @@ def render_command(argv): parser.add_argument('--tests', nargs='+', default=[], metavar='python-file', dest='tests', help='Load top-level functions from the specified file(s) as Jinja2 tests.') parser.add_argument('--customize', default=None, metavar='python-file', dest='customize', - help='Load custom j2cli behavior from a Python file.') + help='Load custom jj2cli behavior from a Python file.') parser.add_argument('--no-compact', action='store_true', dest='no_compact', help='Do not compact space around Jinja2 blocks.') parser.add_argument('-U', '--undefined', action='store_true', dest='undefined', diff --git a/j2cli/context.py b/src/jj2cli/context.py similarity index 100% rename from j2cli/context.py rename to src/jj2cli/context.py diff --git a/j2cli/extras/customize.py b/src/jj2cli/customize.py similarity index 100% rename from j2cli/extras/customize.py rename to src/jj2cli/customize.py diff --git a/j2cli/extras/filters.py b/src/jj2cli/filters.py similarity index 100% rename from j2cli/extras/filters.py rename to src/jj2cli/filters.py From b24390aed51689f21e584195cd7f18213d4a879b Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 28 Jul 2019 16:52:04 +0200 Subject: [PATCH 02/40] context processing: Renamed read_context_data2() to read_context_data(). Removed the obsolete implementation of read_context_data(). --- src/jj2cli/cli.py | 4 ++-- src/jj2cli/context.py | 35 +---------------------------------- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index df18981..d61a2d8 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -11,7 +11,7 @@ from . import __version__ from . import filters from .context import FORMATS -from .context import parse_data_spec, read_context_data2, dict_update_deep +from .context import parse_data_spec, read_context_data, dict_update_deep from .customize import CustomizationModule # available log levels, adjusted with -v at command line @@ -152,7 +152,7 @@ def render_command(argv): customize = CustomizationModule(None) # Read data based on specs - data = [read_context_data2(*dspec) for dspec in dspecs] + data = [read_context_data(*dspec) for dspec in dspecs] # Squash data into a single context context = reduce(dict_update_deep, data, {}) diff --git a/src/jj2cli/context.py b/src/jj2cli/context.py index 40760c5..fdf8d37 100644 --- a/src/jj2cli/context.py +++ b/src/jj2cli/context.py @@ -254,7 +254,7 @@ def parse_data_spec(dspec, fallback_format='ini'): ### return ############################################ return (source, ctx_dst, fmt) -def read_context_data2(source, ctx_dst, fmt): +def read_context_data(source, ctx_dst, fmt): """ Read context data into a dictionary :param source: Source file to read from. Use '-' for stdin, None to read environment (requires fmt == 'env'.) @@ -300,36 +300,3 @@ def read_context_data2(source, ctx_dst, fmt): else: return {ctx_dst: context} -def read_context_data(format, f, environ, import_env=None): - """ Read context data into a dictionary - :param format: Data format - :type format: str - :param f: Data file stream, or None (for env) - :type f: file|None - :param import_env: Variable name, if any, that will contain environment variables of the template. - :type import_env: bool|None - :return: Dictionary with the context data - :rtype: dict - """ - - # Special case: environment variables - if format == 'env' and f is None: - return _parse_env(environ) - - # Read data string stream - data_string = f.read() - - # Parse it - if format not in FORMATS: - raise ValueError('{0} format unavailable'.format(format)) - context = FORMATS[format](data_string) - - # Import environment - if import_env is not None: - if import_env == '': - context.update(environ) - else: - context[import_env] = environ - - # Done - return context From 3d582e587fa4cacf6b3ad6badf3f973bfc842fbd Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 21 Nov 2019 17:31:52 +0100 Subject: [PATCH 03/40] cli options: Added -I/--ignore-missing flag for ignoring missing data files. --- src/jj2cli/cli.py | 4 +++- src/jj2cli/context.py | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index d61a2d8..eab9a21 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -128,6 +128,8 @@ def render_command(argv): help='Do not compact space around Jinja2 blocks.') parser.add_argument('-U', '--undefined', action='store_true', dest='undefined', help='Allow undefined variables to be used in templates (no error will be raised.)') + parser.add_argument('-I', '--ignore-missing', action='store_true', + help='Ignore any missing data files.') parser.add_argument('-o', metavar='outfile', dest='output_file', help="Output to a file instead of stdout.") parser.add_argument('template', help='Template file to process.') parser.add_argument('data', nargs='+', default=[], @@ -152,7 +154,7 @@ def render_command(argv): customize = CustomizationModule(None) # Read data based on specs - data = [read_context_data(*dspec) for dspec in dspecs] + data = [read_context_data(*dspec, args.ignore_missing) for dspec in dspecs] # Squash data into a single context context = reduce(dict_update_deep, data, {}) diff --git a/src/jj2cli/context.py b/src/jj2cli/context.py index fdf8d37..144f351 100644 --- a/src/jj2cli/context.py +++ b/src/jj2cli/context.py @@ -254,7 +254,7 @@ def parse_data_spec(dspec, fallback_format='ini'): ### return ############################################ return (source, ctx_dst, fmt) -def read_context_data(source, ctx_dst, fmt): +def read_context_data(source, ctx_dst, fmt, ignore_missing=False): """ Read context data into a dictionary :param source: Source file to read from. Use '-' for stdin, None to read environment (requires fmt == 'env'.) @@ -264,6 +264,9 @@ def read_context_data(source, ctx_dst, fmt): :type ctx_dst: str|None :param fmt: Data format of the loaded data. :type fmt: str + :param ignore_missing: Flag whether missing files should be ignored and return + an empty context rather than raising an error. + :type ignore_missing: bool|False :return: Dictionary with the context data. :rtype: dict """ @@ -275,8 +278,15 @@ def read_context_data(source, ctx_dst, fmt): data = sys.stdin.read() elif source is not None: # read data from file - with open(source, 'r') as sourcef: - data = sourcef.read() + try: + with open(source, 'r') as sourcef: + data = sourcef.read() + except FileNotFoundError as e: + if ignore_missing: + logging.warning('skipping missing input data file "%s"', source) + return {} + else: + raise e else: data = None From a92a49dbc422e186c480bad02749cd0b8a6333dc Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 21 Nov 2019 01:46:45 +0100 Subject: [PATCH 04/40] context processing: Fix for env format parsing. Convert return data to dict. Down the line, the data are iterated using six.iteritems(), which does not support iterating over a filter. --- src/jj2cli/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jj2cli/context.py b/src/jj2cli/context.py index 144f351..4491993 100644 --- a/src/jj2cli/context.py +++ b/src/jj2cli/context.py @@ -128,7 +128,7 @@ def _parse_env(data_string): """ # Parse if isinstance(data_string, six.string_types): - data = filter( + data = dict(filter( lambda l: len(l) == 2 , ( list(map( @@ -136,7 +136,7 @@ def _parse_env(data_string): line.split('=', 1) )) for line in data_string.split("\n")) - ) + )) else: data = data_string From 983b1636d5e25b03d56a041b4331bacd2ecdd3a9 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 21 Nov 2019 15:45:39 +0100 Subject: [PATCH 05/40] filters: Exported a bunch of Python functions as filters. --- setup.cfg | 3 +++ setup.py | 4 +++- src/jj2cli/cli.py | 5 +---- src/jj2cli/filters.py | 33 ++++++++++++++++++++++++++++++--- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index ed8a958..d1dc52c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,6 @@ universal = 1 [metadata] license_file = LICENSE + +[easy_install] + diff --git a/setup.py b/setup.py index 1737415..87a1a9f 100755 --- a/setup.py +++ b/setup.py @@ -27,6 +27,9 @@ packages_compat.append('jinja2 ~= 2.10.0') else: packages_compat.append('jinja2 ~= 2.11.0') +# Misc. +if PYVER < (3, 0): + packages_compat.append('shutilwhich ~= 1.1') ### Packages for optional functionality. packages_extra = [] @@ -63,7 +66,6 @@ 'jj2 = jj2cli:render', ] }, - install_requires=[ 'six >= 1.13', packages_compat, diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index eab9a21..414d8c4 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -167,10 +167,7 @@ def render_command(argv): customize.j2_environment(renderer._env) # Filters, Tests - renderer.register_filters({ - 'docker_link': filters.docker_link, - 'env': filters.env, - }) + renderer.register_filters(filters.EXTRA_FILTERS) for fname in args.filters: renderer.import_filters(fname) for fname in args.tests: diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index 2890ec4..17d98da 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -1,12 +1,26 @@ -""" Custom Jinja2 filters """ +""" Additional Jinja2 filters """ import os - -from jinja2 import is_undefined import re +import sys +from jinja2 import is_undefined + +if sys.version_info >= (3,0): + from shutil import which +elif sys.version_info >= (2,5): + from shutilwhich import which +else: + assert False, "Unsupported Python version: %s" % sys.version_info +if sys.version_info >= (3,3): + from shlex import quote as sh_quote +elif sys.version_info >= (2,7): + from pipes import quote as sh_quote +else: + assert False, "Unsupported Python version: %s" % sys.version_info def docker_link(value, format='{addr}:{port}'): """ Given a Docker Link environment variable value, format it into something else. + XXX: The name of the filter is not very informative. This is actually a partial URI parser. This first parses a Docker Link value like this: @@ -77,3 +91,16 @@ def env(varname, default=None): else: # Raise KeyError when not provided return os.environ[varname] + + +# Filters to be loaded +EXTRA_FILTERS = { + 'sh_quote': sh_quote, + 'sh_which': which, + 'sh_expand': lambda s: os.path.expandvars(os.path.expanduser(s)), + 'sh_expanduser': os.path.expanduser, + 'sh_expandvars': os.path.expandvars, + 'docker_link': docker_link, + 'env': env, +} + From 86d8508fd4e1cb5101816151c0b9284e53c3684b Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 19 Dec 2019 13:37:49 +0100 Subject: [PATCH 06/40] cli options: Fixed bug with verbosity level wrapping-around. --- src/jj2cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index 414d8c4..0bdc0fd 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -139,7 +139,7 @@ def render_command(argv): 'Parts of the specification that are not needed can be ommitted. ' 'See examples at the end of the help.') args = parser.parse_args(argv[1:]) - logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[args.verbose % len(LOGLEVELS)]) + logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[min(args.verbose, len(LOGLEVELS)-1)]) logging.debug("Parsed arguments: %s", args) # Parse data specifications From 84b4edb059e3bbbd37d107e71bba0fbcce1b499a Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 19 Dec 2019 13:54:00 +0100 Subject: [PATCH 07/40] cli options: Restored previous behaviour for --undefined option. Allows selecting any of the available Jinja2 behaviours (strict, normal, debug) for handling missing variables. --- src/jj2cli/cli.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index 0bdc0fd..b1e6a89 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -20,6 +20,13 @@ # format to use for logging LOGFORMAT = '%(levelname)s: %(message)s' +# map keywords to to Jinja2 error handlers +UNDEFINED = { + 'strict': jinja2.StrictUndefined, # raises errors for undefined variables + 'normal': jinja2.Undefined, # can be printed/iterated - error on other operations + 'debug': jinja2.DebugUndefined, # return the debug info when printed +} + class FilePathLoader(jinja2.BaseLoader): """ Custom Jinja2 template loader which just loads a single template file """ @@ -52,10 +59,10 @@ class Jinja2TemplateRenderer(object): 'jinja2.ext.loopcontrols', ) - def __init__(self, cwd, allow_undefined, no_compact=False, j2_env_params={}): + def __init__(self, cwd, undefined='strict', no_compact=False, j2_env_params={}): # Custom env params j2_env_params.setdefault('keep_trailing_newline', True) - j2_env_params.setdefault('undefined', jinja2.Undefined if allow_undefined else jinja2.StrictUndefined) + j2_env_params.setdefault('undefined', UNDEFINED[undefined]) j2_env_params.setdefault('trim_blocks', not no_compact) j2_env_params.setdefault('lstrip_blocks', not no_compact) j2_env_params.setdefault('extensions', self.ENABLED_EXTENSIONS) @@ -126,11 +133,13 @@ def render_command(argv): help='Load custom jj2cli behavior from a Python file.') parser.add_argument('--no-compact', action='store_true', dest='no_compact', help='Do not compact space around Jinja2 blocks.') - parser.add_argument('-U', '--undefined', action='store_true', dest='undefined', - help='Allow undefined variables to be used in templates (no error will be raised.)') + parser.add_argument('-U', '--undefined', default='strict', dest='undefined', + choices=UNDEFINED.keys(), + help='Set the Junja2 beahaviour for undefined variables.)') parser.add_argument('-I', '--ignore-missing', action='store_true', help='Ignore any missing data files.') - parser.add_argument('-o', metavar='outfile', dest='output_file', help="Output to a file instead of stdout.") + parser.add_argument('-o', metavar='outfile', dest='output_file', + help="Output to a file instead of stdout.") parser.add_argument('template', help='Template file to process.') parser.add_argument('data', nargs='+', default=[], help='Input data specification. Multiple sources in different formats can be specified. ' From 7f01adb17aa6a2efb9ce8c09f86b50d0c746cb3b Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Fri, 17 Apr 2020 13:03:44 +0200 Subject: [PATCH 08/40] filters: Added ifelse/onoff/yesno. --- src/jj2cli/filters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index 17d98da..12c8989 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -100,6 +100,9 @@ def env(varname, default=None): 'sh_expand': lambda s: os.path.expandvars(os.path.expanduser(s)), 'sh_expanduser': os.path.expanduser, 'sh_expandvars': os.path.expandvars, + 'ifelse': lambda t, truev, falsev: truev if t else falsev, + 'onoff': lambda t: 'on' if t else 'off', + 'yesno': lambda t: 'yes' if t else 'no', 'docker_link': docker_link, 'env': env, } From 3f9d682b4994816efbf99e57be386cdf0ea9203c Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Fri, 17 Apr 2020 14:16:45 +0200 Subject: [PATCH 09/40] filters: Added align_suffix filter. Useful for aligning trailing line comments in a block of text. --- src/jj2cli/filters.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index 12c8989..a0e9225 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -92,6 +92,32 @@ def env(varname, default=None): # Raise KeyError when not provided return os.environ[varname] +def align_suffix(text, delim, column=None, spaces_after_delim=1): + """ Align the suffixes of lines in text, starting from the specified delim. + """ + s='' + + if column is None or column == 'auto': + column = max(map(lambda l: l.find(delim), text.splitlines())) + elif column == 'previous': + column = align_suffix.column_previous + + for l in map(lambda s: s.split(delim, 1), text.splitlines()): + if len(l) < 2: + # no delimiter occurs + s += l[0].rstrip() + os.linesep + elif l[0].strip() == '': + # no content before delimiter - leave as-is + s += l[0] + delim + l[1] + os.linesep + else: + # align + s += l[0].rstrip().ljust(column) + delim + spaces_after_delim*' ' + l[1].strip() + os.linesep + + align_suffix.column_previous = column + return s + +align_suffix.column_previous = None + # Filters to be loaded EXTRA_FILTERS = { @@ -105,5 +131,6 @@ def env(varname, default=None): 'yesno': lambda t: 'yes' if t else 'no', 'docker_link': docker_link, 'env': env, + 'align_suffix': align_suffix, } From ac8ffcb460a8b159077f61e0dc3aea5367ae926d Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Fri, 8 May 2020 15:56:41 +0200 Subject: [PATCH 10/40] filters: Added ctxlookup. Allows looking up a string in the context. --- src/jj2cli/filters.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index a0e9225..3f0b8a1 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -2,7 +2,7 @@ import os import re import sys -from jinja2 import is_undefined +from jinja2 import is_undefined, contextfilter if sys.version_info >= (3,0): from shutil import which @@ -119,6 +119,18 @@ def align_suffix(text, delim, column=None, spaces_after_delim=1): align_suffix.column_previous = None +@contextfilter +def ctxlookup(context, key): + """ Lookup the value of a key in the template context. + """ + v = context + try: + for k in key.split('.'): + v = v[k] + return v + except KeyError: + return context.environment.undefined(name=key) + # Filters to be loaded EXTRA_FILTERS = { 'sh_quote': sh_quote, @@ -132,5 +144,6 @@ def align_suffix(text, delim, column=None, spaces_after_delim=1): 'docker_link': docker_link, 'env': env, 'align_suffix': align_suffix, + 'ctxlookup': ctxlookup, } From 3879fbadba9758e9e9d6528cc21f2f637f31c576 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Fri, 8 May 2020 15:53:17 +0200 Subject: [PATCH 11/40] filters: Added sh_realpath, sh_opt, sh_optq. --- src/jj2cli/filters.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index 3f0b8a1..ed133f0 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -131,6 +131,20 @@ def ctxlookup(context, key): except KeyError: return context.environment.undefined(name=key) +def sh_opt(text, name, delim=" ", quote=False): + """ Format text as a command line option. + """ + if not text: + return '' + if quote: + text = sh_quote(text) + return '%s%s%s' % (name, delim, text) + +def sh_optq(text, name, delim=" "): + """ Quote text and format as a command line option. + """ + return sh_opt(text, name, delim, quote=True) + # Filters to be loaded EXTRA_FILTERS = { 'sh_quote': sh_quote, @@ -138,6 +152,9 @@ def ctxlookup(context, key): 'sh_expand': lambda s: os.path.expandvars(os.path.expanduser(s)), 'sh_expanduser': os.path.expanduser, 'sh_expandvars': os.path.expandvars, + 'sh_realpath': os.path.realpath, + 'sh_opt': sh_opt, + 'sh_optq': sh_optq, 'ifelse': lambda t, truev, falsev: truev if t else falsev, 'onoff': lambda t: 'on' if t else 'off', 'yesno': lambda t: 'yes' if t else 'no', From cc83255037c58d15954fada149899c8559e6ee29 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Mon, 8 Mar 2021 22:08:26 +0100 Subject: [PATCH 12/40] cli options: Grouped options in categories. More readable help output. --- src/jj2cli/cli.py | 69 ++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index b1e6a89..918554b 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -112,41 +112,56 @@ def render_command(argv): :return: Rendered template :rtype: basestring """ + version_info = (__version__, jinja2.__version__) formats_names = list(FORMATS.keys()) parser = argparse.ArgumentParser( description='Renders Jinja2 templates from the command line.', epilog='', formatter_class=argparse.ArgumentDefaultsHelpFormatter ) + p_input = parser.add_argument_group('input options') + p_output = parser.add_argument_group('output options') + p_custom = parser.add_argument_group('customization options') + + ### optional arguments ########################################## parser.add_argument('-V', '--version', action='version', - version='jj2cli {0}, Jinja2 {1}'.format(__version__, jinja2.__version__)) + version='jj2cli {0}, Jinja2 {1}'.format(*version_info)) parser.add_argument('-v', '--verbose', action='count', default=0, - help='Increase verbosity.') - parser.add_argument('-f', '--fallback-format', default='ini', choices=formats_names, - help='Specify fallback data format. ' - 'Used for data with no specified format and no appropriate extension.') - parser.add_argument('--filters', nargs='+', default=[], metavar='python-file', dest='filters', - help='Load top-level functions from the specified file(s) as Jinja2 filters.') - parser.add_argument('--tests', nargs='+', default=[], metavar='python-file', dest='tests', - help='Load top-level functions from the specified file(s) as Jinja2 tests.') - parser.add_argument('--customize', default=None, metavar='python-file', dest='customize', - help='Load custom jj2cli behavior from a Python file.') - parser.add_argument('--no-compact', action='store_true', dest='no_compact', - help='Do not compact space around Jinja2 blocks.') - parser.add_argument('-U', '--undefined', default='strict', dest='undefined', - choices=UNDEFINED.keys(), - help='Set the Junja2 beahaviour for undefined variables.)') - parser.add_argument('-I', '--ignore-missing', action='store_true', - help='Ignore any missing data files.') - parser.add_argument('-o', metavar='outfile', dest='output_file', - help="Output to a file instead of stdout.") - parser.add_argument('template', help='Template file to process.') - parser.add_argument('data', nargs='+', default=[], - help='Input data specification. Multiple sources in different formats can be specified. ' - 'The different sources will be squashed into a singled dict. ' - 'The format is ::. ' - 'Parts of the specification that are not needed can be ommitted. ' - 'See examples at the end of the help.') + help='Increase verbosity.') + ### input options ############################################### + p_input.add_argument('template', help='Template file to process.') + p_input.add_argument('data', nargs='+', default=[], + help='Input data specification. ' + 'Multiple sources in different formats can be specified. ' + 'The different sources will be squashed into a singled dict. ' + 'The format is ::. ' + 'Parts of the specification that are not needed can be ommitted. ' + 'See examples at the end of the help.') + p_input.add_argument('-U', '--undefined', default='strict', + dest='undefined', choices=UNDEFINED.keys(), + help='Set the Jinja2 beahaviour for undefined variables.)') + p_input.add_argument('-I', '--ignore-missing', action='store_true', + help='Ignore any missing data files.') + p_input.add_argument('-f', '--fallback-format', + default='ini', choices=formats_names, + help='Specify fallback data format. ' + 'Used for data with no specified format and no appropriate extension.') + ### output options ############################################## + p_output.add_argument('-o', metavar='outfile', dest='output_file', + help="Output to a file instead of stdout.") + p_output.add_argument('--no-compact', action='store_true', dest='no_compact', + help='Do not compact space around Jinja2 blocks.') + ### customization ############################################### + p_custom.add_argument('--filters', nargs='+', default=[], + metavar='python-file', dest='filters', + help='Load the top-level functions from the specified file(s) as Jinja2 filters.') + p_custom.add_argument('--tests', nargs='+', default=[], + metavar='python-file', dest='tests', + help='Load the top-level functions from the specified file(s) as Jinja2 tests.') + p_custom.add_argument('--customize', default=None, + metavar='python-file', dest='customize', + help='Load custom jj2cli behavior from a Python file.') + args = parser.parse_args(argv[1:]) logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[min(args.verbose, len(LOGLEVELS)-1)]) logging.debug("Parsed arguments: %s", args) From 925a119781617973b5faf842ad407ba76002cf3c Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Mon, 10 May 2021 21:33:26 +0200 Subject: [PATCH 13/40] cli options: More robust data spec string parsing. Change of data spec behaviour! The old code was making an attempt to work with any part of the data spec missing: - data_in:json would interpret the contents of data_in as json. - data_in:foo would interpret the contents of data_in as the fallback format, and attach them to "foo" in the context. This resulted in the code being complicated, and also not working as expected in some cases. For this, it was decided to simplify the data spec format as |[]:[[:::. ' - 'Parts of the specification that are not needed can be ommitted. ' + 'The format is [:[:]]. ' + 'Parts of the specification may be left empty. ' 'See examples at the end of the help.') p_input.add_argument('-U', '--undefined', default='strict', dest='undefined', choices=UNDEFINED.keys(), diff --git a/src/jj2cli/context.py b/src/jj2cli/context.py index 4491993..bd792b2 100644 --- a/src/jj2cli/context.py +++ b/src/jj2cli/context.py @@ -6,6 +6,7 @@ import logging import platform import collections +from pathlib import Path # Adjust for aliases removed in python 3.8 try: @@ -208,53 +209,47 @@ def dict_update_deep(d, u): def parse_data_spec(dspec, fallback_format='ini'): """ Parse a data file specification. - :param dspec: Data file specification in format [:][:]. + :param dspec: Data file specification in format [:[:ctx_dst]]. :type dspec: str :param fallback_format: Format to fallback to if no format is set/guessed. :type fallback_format: str - :return: (location, ctx_dest, format) + :return: (path, format, ctx_dst) :rtype: tuple """ - source = ctx_dst = fmt = None - - ### set fmt ########################################### - # manually specified format - if fmt is None: - left, delim, right = dspec.rpartition(':') - if left != '' and right in FORMATS_ALIASES: - source = left - fmt = FORMATS_ALIASES[right] - # guess format by extension - if fmt is None or right == '?': - left, delim, right = dspec.rpartition('.') - if left != '' and right in FORMATS_ALIASES: - source = dspec - fmt = FORMATS_ALIASES[right] - # use fallback format - if fmt is None: - source = dspec - fmt = FORMATS_ALIASES[fallback_format] + MAX_DSPEC_COMPONENTS = 3 + DSPEC_SEP = ':' + + # detect windows-style paths + # NB: In Windows filenames matching [a-z] require a directory prefix. + if DSPEC_SEP == ':' and platform.system() == 'Windows': + m = re.match(r'^[a-z]:[^:]+', dspec, re.I) + else: + m = None - ### set ctx_dst ####################################### - left, delim, right = source.rpartition(':') - if platform.system() == 'Windows' and re.match(r'^[a-z]$', left, re.I): - # windows path (e.g. 'c:\foo.json') -- ignore split - pass - elif left != '' and right != '': - # normal case (e.g. '/data/foo.json:dst') - source = left - ctx_dst = right - elif left != '' and right == '': - # empty ctx_dst (e.g. '/data/foo:1.json:) -- used when source contains ':' - source = left + # parse supplied components + if m is None: + dspec = dspec.rsplit(DSPEC_SEP, MAX_DSPEC_COMPONENTS-1) else: - # no ctx_dst specified - pass + dspec = dspec[m.span()[1] + 1:] + dspec = [m.group(0)] + dspec.rsplit(DSPEC_SEP, MAX_DSPEC_COMPONENTS-2) + + # pad missing components + dspec += (MAX_DSPEC_COMPONENTS - len(dspec))*[None] + + # post-process parsed components + path, fmt, ctx_dst = dspec + path = Path(path) if path not in ['', '-'] else None + if fmt in FORMATS_ALIASES: + fmt = FORMATS_ALIASES[fmt] + elif fmt in ['', '?', None] and path.suffix[1:] in FORMATS_ALIASES: + fmt = FORMATS_ALIASES[path.suffix[1:]] + else: + fmt = FORMATS_ALIASES[fallback_format] + ctx_dst = None if ctx_dst in ['', None] else None - ### return ############################################ - return (source, ctx_dst, fmt) + return (path, fmt, ctx_dst) -def read_context_data(source, ctx_dst, fmt, ignore_missing=False): +def read_context_data(source, fmt, ctx_dst, ignore_missing=False): """ Read context data into a dictionary :param source: Source file to read from. Use '-' for stdin, None to read environment (requires fmt == 'env'.) From 08d3b8d1333a61b6ca4eceafa910080d570ef740 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Wed, 12 May 2021 14:23:25 +0200 Subject: [PATCH 14/40] refactor 1/: cli.py -> render.py --- src/jj2cli/{cli.py => render.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/jj2cli/{cli.py => render.py} (100%) diff --git a/src/jj2cli/cli.py b/src/jj2cli/render.py similarity index 100% rename from src/jj2cli/cli.py rename to src/jj2cli/render.py From 41da67f63c272c35e705d1c11c553a9dc06848b5 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 5 Nov 2023 23:32:33 +0100 Subject: [PATCH 15/40] refactor 2/: Spun-off cli-parts from render.py. --- src/jj2cli/cli.py | 142 ++++++++++++++++++++++++++++++++++++++ src/jj2cli/render.py | 161 +++---------------------------------------- 2 files changed, 151 insertions(+), 152 deletions(-) create mode 100644 src/jj2cli/cli.py diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py new file mode 100644 index 0000000..1170ecc --- /dev/null +++ b/src/jj2cli/cli.py @@ -0,0 +1,142 @@ +import io +import os +import sys +import argparse +import logging +from functools import reduce + +import jinja2 +import jinja2.meta +import jinja2.loaders + +import imp + +from . import __version__ +from . import filters +from .context import FORMATS +from .context import parse_data_spec, read_context_data, dict_update_deep +from .customize import CustomizationModule +from .render import Jinja2TemplateRenderer + +# available log levels, adjusted with -v at command line +LOGLEVELS = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] + +# format to use for logging +LOGFORMAT = '%(levelname)s: %(message)s' + + +def render_command(argv): + """ Pure render command + :param argv: Command-line arguments + :type argv: list + :return: Rendered template + :rtype: basestring + """ + version_info = (__version__, jinja2.__version__) + formats_names = list(FORMATS.keys()) + parser = argparse.ArgumentParser( + description='Renders Jinja2 templates from the command line.', + epilog='', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + p_input = parser.add_argument_group('input options') + p_output = parser.add_argument_group('output options') + p_custom = parser.add_argument_group('customization options') + + ### optional arguments ########################################## + parser.add_argument('-V', '--version', action='version', + version='jj2cli {0}, Jinja2 {1}'.format(*version_info)) + parser.add_argument('-v', '--verbose', action='count', default=0, + help='Increase verbosity.') + ### input options ############################################### + p_input.add_argument('template', help='Template file to process.') + p_input.add_argument('data', nargs='+', default=[], + help='Input data specification. ' + 'Multiple sources in different formats can be specified. ' + 'The different sources will be squashed into a singled dict. ' + 'The format is [:[:]]. ' + 'Parts of the specification may be left empty. ' + 'See examples at the end of the help.') + p_input.add_argument('-U', '--undefined', default='strict', + dest='undefined', choices=Jinja2TemplateRenderer.UNDEFINED.keys(), + help='Set the Jinja2 beahaviour for undefined variables.)') + p_input.add_argument('-I', '--ignore-missing', action='store_true', + help='Ignore any missing data files.') + p_input.add_argument('-f', '--fallback-format', + default='ini', choices=formats_names, + help='Specify fallback data format. ' + 'Used for data with no specified format and no appropriate extension.') + ### output options ############################################## + p_output.add_argument('-o', metavar='outfile', dest='output_file', + help="Output to a file instead of stdout.") + p_output.add_argument('--no-compact', action='store_true', dest='no_compact', + help='Do not compact space around Jinja2 blocks.') + ### customization ############################################### + p_custom.add_argument('--filters', nargs='+', default=[], + metavar='python-file', dest='filters', + help='Load the top-level functions from the specified file(s) as Jinja2 filters.') + p_custom.add_argument('--tests', nargs='+', default=[], + metavar='python-file', dest='tests', + help='Load the top-level functions from the specified file(s) as Jinja2 tests.') + p_custom.add_argument('--customize', default=None, + metavar='python-file', dest='customize', + help='Load custom jj2cli behavior from a Python file.') + + args = parser.parse_args(argv[1:]) + logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[min(args.verbose, len(LOGLEVELS)-1)]) + logging.debug("PARSED_ARGS render_command: %s", args) + + # Parse data specifications + dspecs = [parse_data_spec(d, fallback_format=args.fallback_format) for d in args.data] + + # Customization + if args.customize is not None: + customize = CustomizationModule( + imp.load_source('customize-module', args.customize) + ) + else: + customize = CustomizationModule(None) + + # Read data based on specs + data = [read_context_data(*dspec, args.ignore_missing) for dspec in dspecs] + + # Squash data into a single context. + context = reduce(dict_update_deep, data, {}) + + # Apply final customizations. + context = customize.alter_context(context) + + # Renderer + renderer = Jinja2TemplateRenderer(os.getcwd(), args.undefined, args.no_compact, j2_env_params=customize.j2_environment_params()) + customize.j2_environment(renderer._env) + + # Filters, Tests + renderer.register_filters(filters.EXTRA_FILTERS) + for fname in args.filters: + renderer.import_filters(fname) + for fname in args.tests: + renderer.import_tests(fname) + + renderer.register_filters(customize.extra_filters()) + renderer.register_tests(customize.extra_tests()) + result = renderer.render(args.template, context) + + # -o + if args.output_file: + with io.open(args.output_file, 'wt', encoding='utf-8') as f: + f.write(result.decode('utf-8')) + f.close() + return b'' + + # Finish + return result + + +def main(): + """ CLI entry point for rendering templates. """ + try: + output = render_command(sys.argv) + except SystemExit: + return 1 + outstream = getattr(sys.stdout, 'buffer', sys.stdout) + outstream.write(output) diff --git a/src/jj2cli/render.py b/src/jj2cli/render.py index 53f6c5a..7881b05 100644 --- a/src/jj2cli/render.py +++ b/src/jj2cli/render.py @@ -1,18 +1,13 @@ -import io, os, sys -import argparse +import io +import os import logging -from functools import reduce import jinja2 import jinja2.loaders import imp, inspect -from . import __version__ from . import filters -from .context import FORMATS -from .context import parse_data_spec, read_context_data, dict_update_deep -from .customize import CustomizationModule # available log levels, adjusted with -v at command line LOGLEVELS = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] @@ -20,12 +15,6 @@ # format to use for logging LOGFORMAT = '%(levelname)s: %(message)s' -# map keywords to to Jinja2 error handlers -UNDEFINED = { - 'strict': jinja2.StrictUndefined, # raises errors for undefined variables - 'normal': jinja2.Undefined, # can be printed/iterated - error on other operations - 'debug': jinja2.DebugUndefined, # return the debug info when printed -} class FilePathLoader(jinja2.BaseLoader): """ Custom Jinja2 template loader which just loads a single template file """ @@ -59,10 +48,16 @@ class Jinja2TemplateRenderer(object): 'jinja2.ext.loopcontrols', ) + UNDEFINED = { + 'strict': jinja2.StrictUndefined, # raises errors for undefined variables + 'normal': jinja2.Undefined, # can be printed/iterated - error on other operations + 'debug': jinja2.DebugUndefined, # return the debug info when printed + } + def __init__(self, cwd, undefined='strict', no_compact=False, j2_env_params={}): # Custom env params j2_env_params.setdefault('keep_trailing_newline', True) - j2_env_params.setdefault('undefined', UNDEFINED[undefined]) + j2_env_params.setdefault('undefined', self.UNDEFINED[undefined]) j2_env_params.setdefault('trim_blocks', not no_compact) j2_env_params.setdefault('lstrip_blocks', not no_compact) j2_env_params.setdefault('extensions', self.ENABLED_EXTENSIONS) @@ -103,141 +98,3 @@ def render(self, template_path, context): .get_template(template_path) \ .render(context) \ .encode('utf-8') - - -def render_command(argv): - """ Pure render command - :param argv: Command-line arguments - :type argv: list - :return: Rendered template - :rtype: basestring - """ - version_info = (__version__, jinja2.__version__) - formats_names = list(FORMATS.keys()) - parser = argparse.ArgumentParser( - description='Renders Jinja2 templates from the command line.', - epilog='', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - p_input = parser.add_argument_group('input options') - p_output = parser.add_argument_group('output options') - p_custom = parser.add_argument_group('customization options') - - ### optional arguments ########################################## - parser.add_argument('-V', '--version', action='version', - version='jj2cli {0}, Jinja2 {1}'.format(*version_info)) - parser.add_argument('-v', '--verbose', action='count', default=0, - help='Increase verbosity.') - ### input options ############################################### - p_input.add_argument('template', help='Template file to process.') - p_input.add_argument('data', nargs='+', default=[], - help='Input data specification. ' - 'Multiple sources in different formats can be specified. ' - 'The different sources will be squashed into a singled dict. ' - 'The format is [:[:]]. ' - 'Parts of the specification may be left empty. ' - 'See examples at the end of the help.') - p_input.add_argument('-U', '--undefined', default='strict', - dest='undefined', choices=UNDEFINED.keys(), - help='Set the Jinja2 beahaviour for undefined variables.)') - p_input.add_argument('-I', '--ignore-missing', action='store_true', - help='Ignore any missing data files.') - p_input.add_argument('-f', '--fallback-format', - default='ini', choices=formats_names, - help='Specify fallback data format. ' - 'Used for data with no specified format and no appropriate extension.') - ### output options ############################################## - p_output.add_argument('-o', metavar='outfile', dest='output_file', - help="Output to a file instead of stdout.") - p_output.add_argument('--no-compact', action='store_true', dest='no_compact', - help='Do not compact space around Jinja2 blocks.') - ### customization ############################################### - p_custom.add_argument('--filters', nargs='+', default=[], - metavar='python-file', dest='filters', - help='Load the top-level functions from the specified file(s) as Jinja2 filters.') - p_custom.add_argument('--tests', nargs='+', default=[], - metavar='python-file', dest='tests', - help='Load the top-level functions from the specified file(s) as Jinja2 tests.') - p_custom.add_argument('--customize', default=None, - metavar='python-file', dest='customize', - help='Load custom jj2cli behavior from a Python file.') - - args = parser.parse_args(argv[1:]) - logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[min(args.verbose, len(LOGLEVELS)-1)]) - logging.debug("Parsed arguments: %s", args) - - # Parse data specifications - dspecs = [parse_data_spec(d, fallback_format=args.fallback_format) for d in args.data] - - # Customization - if args.customize is not None: - customize = CustomizationModule( - imp.load_source('customize-module', args.customize) - ) - else: - customize = CustomizationModule(None) - - # Read data based on specs - data = [read_context_data(*dspec, args.ignore_missing) for dspec in dspecs] - - # Squash data into a single context - context = reduce(dict_update_deep, data, {}) - - # Apply final customizations - context = customize.alter_context(context) - - # Renderer - renderer = Jinja2TemplateRenderer(os.getcwd(), args.undefined, args.no_compact, j2_env_params=customize.j2_environment_params()) - customize.j2_environment(renderer._env) - - # Filters, Tests - renderer.register_filters(filters.EXTRA_FILTERS) - for fname in args.filters: - renderer.import_filters(fname) - for fname in args.tests: - renderer.import_tests(fname) - - renderer.register_filters(customize.extra_filters()) - renderer.register_tests(customize.extra_tests()) - - # Render - try: - result = renderer.render(args.template, context) - except jinja2.exceptions.UndefinedError as e: - # When there's data at stdin, tell the user they should use '-' - try: - stdin_has_data = stdin is not None and not stdin.isatty() - if args.format == 'env' and args.data == None and stdin_has_data: - extra_info = ( - "\n\n" - "If you're trying to pipe a .env file, please run me with a '-' as the data file name:\n" - "$ {cmd} {argv} -".format(cmd=os.path.basename(sys.argv[0]), argv=' '.join(sys.argv[1:])) - ) - e.args = (e.args[0] + extra_info,) + e.args[1:] - except: - # The above code is so optional that any, ANY, error, is ignored - pass - - # Proceed - raise - - # -o - if args.output_file: - with io.open(args.output_file, 'wt', encoding='utf-8') as f: - f.write(result.decode('utf-8')) - f.close() - return b'' - - # Finish - return result - - - -def main(): - """ CLI Entry point """ - try: - output = render_command(sys.argv) - except SystemExit: - return 1 - outstream = getattr(sys.stdout, 'buffer', sys.stdout) - outstream.write(output) From 6aac2eb8d054d20de73fa6115cde87f46801ccf4 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 13 May 2021 01:02:59 +0200 Subject: [PATCH 16/40] refactor 3/: context.py -> parsers.py --- src/jj2cli/cli.py | 30 ++--- src/jj2cli/context.py | 307 ------------------------------------------ src/jj2cli/parsers.py | 211 +++++++++++++++++++++++++++++ src/jj2cli/render.py | 7 - 4 files changed, 223 insertions(+), 332 deletions(-) delete mode 100644 src/jj2cli/context.py create mode 100644 src/jj2cli/parsers.py diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index 1170ecc..c603448 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -1,20 +1,18 @@ +import argparse +import imp import io +import logging import os import sys -import argparse -import logging -from functools import reduce - import jinja2 -import jinja2.meta import jinja2.loaders +import jinja2.meta -import imp +from functools import reduce from . import __version__ from . import filters -from .context import FORMATS -from .context import parse_data_spec, read_context_data, dict_update_deep +from . import parsers from .customize import CustomizationModule from .render import Jinja2TemplateRenderer @@ -33,7 +31,6 @@ def render_command(argv): :rtype: basestring """ version_info = (__version__, jinja2.__version__) - formats_names = list(FORMATS.keys()) parser = argparse.ArgumentParser( description='Renders Jinja2 templates from the command line.', epilog='', @@ -50,7 +47,7 @@ def render_command(argv): help='Increase verbosity.') ### input options ############################################### p_input.add_argument('template', help='Template file to process.') - p_input.add_argument('data', nargs='+', default=[], + p_input.add_argument('data', nargs='+', default=[], type=parsers.InputDataType(), help='Input data specification. ' 'Multiple sources in different formats can be specified. ' 'The different sources will be squashed into a singled dict. ' @@ -58,12 +55,12 @@ def render_command(argv): 'Parts of the specification may be left empty. ' 'See examples at the end of the help.') p_input.add_argument('-U', '--undefined', default='strict', - dest='undefined', choices=Jinja2TemplateRenderer.UNDEFINED.keys(), + dest='undefined', choices=Jinja2TemplateRenderer.UNDEFINED, help='Set the Jinja2 beahaviour for undefined variables.)') p_input.add_argument('-I', '--ignore-missing', action='store_true', help='Ignore any missing data files.') p_input.add_argument('-f', '--fallback-format', - default='ini', choices=formats_names, + default='ini', choices=parsers.FORMATS, help='Specify fallback data format. ' 'Used for data with no specified format and no appropriate extension.') ### output options ############################################## @@ -86,9 +83,6 @@ def render_command(argv): logging.basicConfig(format=LOGFORMAT, level=LOGLEVELS[min(args.verbose, len(LOGLEVELS)-1)]) logging.debug("PARSED_ARGS render_command: %s", args) - # Parse data specifications - dspecs = [parse_data_spec(d, fallback_format=args.fallback_format) for d in args.data] - # Customization if args.customize is not None: customize = CustomizationModule( @@ -97,11 +91,11 @@ def render_command(argv): else: customize = CustomizationModule(None) - # Read data based on specs - data = [read_context_data(*dspec, args.ignore_missing) for dspec in dspecs] + # Read data based on specs. + data = [d.parse(ignore_missing=args.ignore_missing, fallback_format=args.fallback_format) for d in args.data] # Squash data into a single context. - context = reduce(dict_update_deep, data, {}) + context = reduce(parsers.dict_squash, data, {}) # Apply final customizations. context = customize.alter_context(context) diff --git a/src/jj2cli/context.py b/src/jj2cli/context.py deleted file mode 100644 index bd792b2..0000000 --- a/src/jj2cli/context.py +++ /dev/null @@ -1,307 +0,0 @@ -import six -import os -import sys -import six -import re -import logging -import platform -import collections -from pathlib import Path - -# Adjust for aliases removed in python 3.8 -try: - collectionsAbc = collections.abc -except AttributeError: - collectionsAbc = collections - -#region Parsers - -def _parse_ini(data_string): - """ INI data input format. - - data.ini: - - ``` - [nginx] - hostname=localhost - webroot=/var/www/project - logs=/var/log/nginx/ - ``` - - Usage: - - $ j2 config.j2 data.ini - $ cat data.ini | j2 --format=ini config.j2 - """ - from io import StringIO - - # Override - class MyConfigParser(ConfigParser.ConfigParser): - def as_dict(self): - """ Export as dict - :rtype: dict - """ - d = dict(self._sections) - for k in d: - d[k] = dict(self._defaults, **d[k]) - d[k].pop('__name__', None) - return d - - # Parse - ini = MyConfigParser() - ini.readfp(ini_file_io(data_string)) - - # Export - return ini.as_dict() - -def _parse_json(data_string): - """ JSON data input format - - data.json: - - ``` - { - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project", - "logs": "/var/log/nginx/" - } - } - ``` - - Usage: - - $ j2 config.j2 data.json - $ cat data.json | j2 --format=ini config.j2 - """ - return json.loads(data_string) - -def _parse_yaml(data_string): - """ YAML data input format. - - data.yaml: - - ``` - nginx: - hostname: localhost - webroot: /var/www/project - logs: /var/log/nginx - ``` - - Usage: - - $ j2 config.j2 data.yml - $ cat data.yml | j2 --format=yaml config.j2 - """ - # Loader - try: - # PyYAML 5.1 supports FullLoader - Loader = yaml.FullLoader - except AttributeError: - # Have to use SafeLoader for older versions - Loader = yaml.SafeLoader - # Done - return yaml.load(data_string, Loader=Loader) - -def _parse_env(data_string): - """ Data input from environment variables. - - Render directly from the current environment variable values: - - $ j2 config.j2 - - Or alternatively, read the values from a dotenv file: - - ``` - NGINX_HOSTNAME=localhost - NGINX_WEBROOT=/var/www/project - NGINX_LOGS=/var/log/nginx/ - ``` - - And render with: - - $ j2 config.j2 data.env - $ env | j2 --format=env config.j2 - - If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: - - $ j2 config.j2 - < data.env - """ - # Parse - if isinstance(data_string, six.string_types): - data = dict(filter( - lambda l: len(l) == 2 , - ( - list(map( - str.strip, - line.split('=', 1) - )) - for line in data_string.split("\n")) - )) - else: - data = data_string - - # Finish - return data - - -FORMATS = { - 'ini': _parse_ini, - 'json': _parse_json, - 'yaml': _parse_yaml, - 'env': _parse_env -} - -FORMATS_ALIASES = dict(zip(FORMATS.keys(), FORMATS.keys())) -FORMATS_ALIASES.update({ - 'yml': 'yaml', -}) - -#endregion - - - -#region Imports - -# JSON: simplejson | json -try: - import simplejson as json -except ImportError: - try: - import json - except ImportError: - del FORMATS['json'] - -# INI: Python 2 | Python 3 -try: - import ConfigParser - from io import BytesIO as ini_file_io -except ImportError: - import configparser as ConfigParser - from io import StringIO as ini_file_io - -# YAML -try: - import yaml -except ImportError: - del FORMATS['yaml'] - -#endregion - -def dict_update_deep(d, u): - """ Performs a deep update of d with data from u. - :param d: Dictionary to be updated. - :type dict: dict - :param u: Dictionary with updates to be applied. - :type dict: dict - :return: Updated version of d. - :rtype: dict - """ - for k, v in six.iteritems(u): - dv = d.get(k, {}) - if not isinstance(dv, collectionsAbc.Mapping): - d[k] = v - elif isinstance(v, collectionsAbc.Mapping): - d[k] = dict_update_deep(dv, v) - else: - d[k] = v - return d - -def parse_data_spec(dspec, fallback_format='ini'): - """ Parse a data file specification. - :param dspec: Data file specification in format [:[:ctx_dst]]. - :type dspec: str - :param fallback_format: Format to fallback to if no format is set/guessed. - :type fallback_format: str - :return: (path, format, ctx_dst) - :rtype: tuple - """ - MAX_DSPEC_COMPONENTS = 3 - DSPEC_SEP = ':' - - # detect windows-style paths - # NB: In Windows filenames matching [a-z] require a directory prefix. - if DSPEC_SEP == ':' and platform.system() == 'Windows': - m = re.match(r'^[a-z]:[^:]+', dspec, re.I) - else: - m = None - - # parse supplied components - if m is None: - dspec = dspec.rsplit(DSPEC_SEP, MAX_DSPEC_COMPONENTS-1) - else: - dspec = dspec[m.span()[1] + 1:] - dspec = [m.group(0)] + dspec.rsplit(DSPEC_SEP, MAX_DSPEC_COMPONENTS-2) - - # pad missing components - dspec += (MAX_DSPEC_COMPONENTS - len(dspec))*[None] - - # post-process parsed components - path, fmt, ctx_dst = dspec - path = Path(path) if path not in ['', '-'] else None - if fmt in FORMATS_ALIASES: - fmt = FORMATS_ALIASES[fmt] - elif fmt in ['', '?', None] and path.suffix[1:] in FORMATS_ALIASES: - fmt = FORMATS_ALIASES[path.suffix[1:]] - else: - fmt = FORMATS_ALIASES[fallback_format] - ctx_dst = None if ctx_dst in ['', None] else None - - return (path, fmt, ctx_dst) - -def read_context_data(source, fmt, ctx_dst, ignore_missing=False): - """ Read context data into a dictionary - :param source: Source file to read from. - Use '-' for stdin, None to read environment (requires fmt == 'env'.) - :type source: str|None - :param ctx_dst: Variable name that will contain the loaded data in the returned dict. - If None, data are loaded to the top-level of the dict. - :type ctx_dst: str|None - :param fmt: Data format of the loaded data. - :type fmt: str - :param ignore_missing: Flag whether missing files should be ignored and return - an empty context rather than raising an error. - :type ignore_missing: bool|False - :return: Dictionary with the context data. - :rtype: dict - """ - logging.debug("Reading data: source=%s, ctx_dst=%s, fmt=%s", source, ctx_dst, fmt) - - # Special case: environment variables - if source == '-': - # read data from stdin - data = sys.stdin.read() - elif source is not None: - # read data from file - try: - with open(source, 'r') as sourcef: - data = sourcef.read() - except FileNotFoundError as e: - if ignore_missing: - logging.warning('skipping missing input data file "%s"', source) - return {} - else: - raise e - else: - data = None - - if data is None and fmt == env: - # load environment to context dict - if sys.version_info[0] > 2: - context = os.environ.copy() - else: - # python2: encode environment variables as unicode - context = dict((k.decode('utf-8'), v.decode('utf-8')) for k, v in os.environ.items()) - elif data is not None: - # parse data to context dict - context = FORMATS[fmt](data) - else: - # this shouldn't have happened - logging.error("Can't read data in %s format from %s.", fmt, source) - sys.exit(1) - - if ctx_dst is None: - return context - else: - return {ctx_dst: context} - diff --git a/src/jj2cli/parsers.py b/src/jj2cli/parsers.py new file mode 100644 index 0000000..5f80cf8 --- /dev/null +++ b/src/jj2cli/parsers.py @@ -0,0 +1,211 @@ +import itertools +import json +import logging +import os +import platform +import re +import six +import sys + +from os import linesep +from pathlib import Path +from six.moves import collections_abc, configparser, filter, zip + +try: + import yaml + try: + yaml_loader = yaml.FullLoader + except AttributeError: + yaml_loader = yaml.SafeLoader +except ImportError: + yaml_loader = None + +# XXX: Chaining used instead of unpacking (*) for backwards compatibility. +FORMATS = ['env', 'ENV', 'ini', 'json', 'yaml'] +if yaml_loader is None: + FORMATS.remove('yaml') +FORMATS_ALIASES = dict(itertools.chain(zip(FORMATS, FORMATS), + filter(lambda t: t[1] in FORMATS, [('yml', 'yaml')]))) + +DATASPEC_SEP = ':' +DATASPEC_COMPONENTS_MAX = 3 + +class InputDataType: + """Factory for creating jj2cli input data types. + + Instances of InputDataType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + - encoding -- The file's encoding. Accepts the same values as the + builtin open() function. + - errors -- A string indicating how encoding and decoding errors are to + be handled. Accepts the same value as the builtin open() function. + """ + def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + self._mode = mode + self._bufsize = bufsize + self._encoding = encoding + self._errors = errors + + def __call__(self, dspec): + # detect windows-style paths + # NB: In Windows, filenames matching ^[a-z]$ always require a directory + # if the data spec has >1 components. + if DATASPEC_SEP == ':' and platform.system() == 'Windows': + m = re.match(r'^[a-z]:[^:]+', dspec, re.I) + else: + m = None + + # parse supplied components + if m is None: + # normal case + dspec = dspec.rsplit(DATASPEC_SEP, DATASPEC_COMPONENTS_MAX-1) + else: + # matched windows drive at the start of the data spec + dspec = dspec[m.span()[1] + 1:] + dspec = [m.group(0)] + dspec.rsplit(DATASPEC_SEP, DATASPEC_COMPONENTS_MAX-2) + + # pad missing components + dspec += (DATASPEC_COMPONENTS_MAX - len(dspec))*[None] + + # post-process parsed components + path, fmt, ctx_dst = dspec + path = Path(path) if path not in ['', '-'] else None + if fmt in FORMATS_ALIASES: + # forced format is case-sensitive + fmt = FORMATS_ALIASES[fmt] + elif fmt in ['', '?', None] and path is not None and path.suffix[1:] in FORMATS_ALIASES: + # file extensions are case-insensitive + fmt = FORMATS_ALIASES[path.suffix[1:].lower()] + else: + fmt = None + ctx_dst = None if ctx_dst in ['', None] else None + + # check for formats that don't use file input + if fmt == 'ENV' and path is not None: + logging.warning("Ignoring source for %s format: %s", fmt, path) + path = None + + # open stream and return InputData object + if path is None: + if fmt == 'ENV': + iostr = None + elif 'r' in self._mode: + iostr = sys.stdin + elif 'w' in self._mode: + # XXX: Is there a use-case we could use this? + iostr = sys.stdout + else: + raise ValueError("Invalid mode %r for std streams." % self._mode) + else: + try: + iostr = path.open(self._mode, self._bufsize, self._encoding, self._errors) + except FileNotFoundError as e: + # FileNotFoundError will be reraised later by InputData.parse(), + # depending on # whether the -I flag has been specified. + iostr = e + + return InputData(iostr, fmt, ctx_dst) + + +class InputData: + def __init__(self, iostr, fmt=None, ctx_dst=None): + self._iostr = iostr + self._fmt = fmt + self._ctx_dst = ctx_dst + + def __repr__(self): + ioinfo = (self._iostr + if self._iostr is None or isinstance(self._iostr, FileNotFoundError) + else '%s:%s:%s' % (self._iostr.name, self._iostr.mode, self._iostr.encoding)) + return '%s(%s, %s, %s)' % (type(self).__name__, ioinfo, self.fmt, self._ctx_dst) + + @property + def fmt(self): + return self._fmt + + @fmt.setter + def set_fmt(self, v): + if v in FORMATS: + self._fmt = v + else: + raise ValueError("Invalid format %s." % v) + + def parse(self, ignore_missing=False, fallback_format='ini'): + """Parses the data from the data stream of the object. + If ignore_missing is set, missing files will produce and empty dict. + If no format is set for the object, fallback_format is used. + """ + fmt = self._fmt if self._fmt is not None else fallback_format + if isinstance(self._iostr, FileNotFoundError): + if ignore_missing is True: + return {} + else: + raise self._iostr + return getattr(self, '_parse_%s' % fmt)() + + def _parse_ENV(self): + """Loads data from shell environment. + """ + return os.environ.copy() + + def _parse_env(self): + """Parses an env-like format. + XXX + """ + normalize = lambda t: (t[0].strip(), t[1].strip()) + return dict([ + normalize(ln.split('=', 1)) + for ln in self._iostr + if '=' in ln + ]) + + def _parse_ini(self): + """Parses windows-style ini files. + """ + class MyConfigParser(configparser.ConfigParser): + def as_dict(self): + """ Export as dict + :rtype: dict + """ + d = dict(self._sections) + for k in d: + d[k] = dict(self._defaults, **d[k]) + d[k].pop('__name__', None) + return d + ini = MyConfigParser() + ini.readfp(self._iostr) + return ini.as_dict() + + def _parse_json(self): + return json.load(self._iostr) + + def _parse_yaml(self): + if yaml_loader is None: + raise RuntimeError("YAML data parser invoked, but no YAML support is present.") + return yaml.load(self._iostr, Loader=yaml_loader) + + +def dict_squash(d, u): + """ Squashes contents of u on d. + :param d: Dictionary to be updated. + :type dict: dict + :param u: Dictionary with updates to be applied. + :type dict: dict + :return: Updated version of d. + :rtype: dict + """ + for k, v in six.iteritems(u): + dv = d.get(k, {}) + if not isinstance(dv, collections_abc.Mapping): + d[k] = v + elif isinstance(v, collections_abc.Mapping): + d[k] = dict_squash(dv, v) + else: + d[k] = v + return d diff --git a/src/jj2cli/render.py b/src/jj2cli/render.py index 7881b05..bcce5fe 100644 --- a/src/jj2cli/render.py +++ b/src/jj2cli/render.py @@ -9,13 +9,6 @@ from . import filters -# available log levels, adjusted with -v at command line -LOGLEVELS = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] - -# format to use for logging -LOGFORMAT = '%(levelname)s: %(message)s' - - class FilePathLoader(jinja2.BaseLoader): """ Custom Jinja2 template loader which just loads a single template file """ From 47e0e1e373028c0a83b80586f32d2cdbc65b7367 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 5 Nov 2023 23:48:16 +0100 Subject: [PATCH 17/40] refactor 4/: Collected dispersed defaults in a module. --- src/jj2cli/cli.py | 3 ++- src/jj2cli/defaults.py | 40 +++++++++++++++++++++++++++++++++++++ src/jj2cli/parsers.py | 45 +++++++++++++----------------------------- src/jj2cli/render.py | 9 ++------- 4 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 src/jj2cli/defaults.py diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index c603448..ebd99e7 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -14,6 +14,7 @@ from . import filters from . import parsers from .customize import CustomizationModule +from .defaults import CONTEXT_FORMATS from .render import Jinja2TemplateRenderer # available log levels, adjusted with -v at command line @@ -60,7 +61,7 @@ def render_command(argv): p_input.add_argument('-I', '--ignore-missing', action='store_true', help='Ignore any missing data files.') p_input.add_argument('-f', '--fallback-format', - default='ini', choices=parsers.FORMATS, + default='ini', choices=CONTEXT_FORMATS, help='Specify fallback data format. ' 'Used for data with no specified format and no appropriate extension.') ### output options ############################################## diff --git a/src/jj2cli/defaults.py b/src/jj2cli/defaults.py new file mode 100644 index 0000000..69ec48b --- /dev/null +++ b/src/jj2cli/defaults.py @@ -0,0 +1,40 @@ +import itertools + +# Jinja2 extensions loaded by jj2cli. +JINJA2_ENABLED_EXTENSIONS = ( + 'jinja2.ext.i18n', + 'jinja2.ext.do', + 'jinja2.ext.loopcontrols', +) + +# Set yaml_loader for parsers. +try: + import yaml + try: + _yaml_loader = yaml.FullLoader + except AttributeError: + _yaml_loader = yaml.SafeLoader + yaml_load = lambda iostr: yaml.load(iostr, Loader=_yaml_loader) +except ImportError: + yaml_load = None + +# Supported context formats. +CONTEXT_FORMATS = ['env', 'ENV', 'ini', 'json', 'yaml'] +if yaml_load is None: + CONTEXT_FORMATS.remove('yaml') + +# Format aliases dictionary. +# COMPAT: Chaining used instead of unpacking (*) for backwards compatibility. +CONTEXT_FORMATS_ALIASES = dict(itertools.chain( + zip(CONTEXT_FORMATS, CONTEXT_FORMATS), + filter(lambda t: t[1] in CONTEXT_FORMATS, [('yml', 'yaml')]) +)) + +# Variables for parsing dataspecs. +DATASPEC_SEP = ':' +DATASPEC_COMPONENTS_MAX = 3 + +# Supported formats for outputting template dependencies. +DEPENDENCIES_OUTPUT_FORMATS = ['make', 'json', 'yaml', 'delim'] +if yaml_load is None: + DEPENDENCIES_OUTPUT_FORMATS.remove('yaml') diff --git a/src/jj2cli/parsers.py b/src/jj2cli/parsers.py index 5f80cf8..a3221ea 100644 --- a/src/jj2cli/parsers.py +++ b/src/jj2cli/parsers.py @@ -1,34 +1,17 @@ -import itertools import json import logging import os import platform import re -import six import sys - -from os import linesep from pathlib import Path -from six.moves import collections_abc, configparser, filter, zip - -try: - import yaml - try: - yaml_loader = yaml.FullLoader - except AttributeError: - yaml_loader = yaml.SafeLoader -except ImportError: - yaml_loader = None - -# XXX: Chaining used instead of unpacking (*) for backwards compatibility. -FORMATS = ['env', 'ENV', 'ini', 'json', 'yaml'] -if yaml_loader is None: - FORMATS.remove('yaml') -FORMATS_ALIASES = dict(itertools.chain(zip(FORMATS, FORMATS), - filter(lambda t: t[1] in FORMATS, [('yml', 'yaml')]))) - -DATASPEC_SEP = ':' -DATASPEC_COMPONENTS_MAX = 3 + +import six +from six.moves import collections_abc, configparser + +from .defaults import (CONTEXT_FORMATS, CONTEXT_FORMATS_ALIASES, + DATASPEC_COMPONENTS_MAX, DATASPEC_SEP, yaml_load) + class InputDataType: """Factory for creating jj2cli input data types. @@ -76,12 +59,12 @@ def __call__(self, dspec): # post-process parsed components path, fmt, ctx_dst = dspec path = Path(path) if path not in ['', '-'] else None - if fmt in FORMATS_ALIASES: + if fmt in CONTEXT_FORMATS_ALIASES: # forced format is case-sensitive - fmt = FORMATS_ALIASES[fmt] - elif fmt in ['', '?', None] and path is not None and path.suffix[1:] in FORMATS_ALIASES: + fmt = CONTEXT_FORMATS_ALIASES[fmt] + elif fmt in ['', '?', None] and path is not None and path.suffix[1:] in CONTEXT_FORMATS_ALIASES: # file extensions are case-insensitive - fmt = FORMATS_ALIASES[path.suffix[1:].lower()] + fmt = CONTEXT_FORMATS_ALIASES[path.suffix[1:].lower()] else: fmt = None ctx_dst = None if ctx_dst in ['', None] else None @@ -131,7 +114,7 @@ def fmt(self): @fmt.setter def set_fmt(self, v): - if v in FORMATS: + if v in CONTEXT_FORMATS: self._fmt = v else: raise ValueError("Invalid format %s." % v) @@ -186,9 +169,9 @@ def _parse_json(self): return json.load(self._iostr) def _parse_yaml(self): - if yaml_loader is None: + if yaml_load is None: raise RuntimeError("YAML data parser invoked, but no YAML support is present.") - return yaml.load(self._iostr, Loader=yaml_loader) + return yaml_load(self._iostr) def dict_squash(d, u): diff --git a/src/jj2cli/render.py b/src/jj2cli/render.py index bcce5fe..c09abe8 100644 --- a/src/jj2cli/render.py +++ b/src/jj2cli/render.py @@ -8,6 +8,7 @@ import imp, inspect from . import filters +from .defaults import JINJA2_ENABLED_EXTENSIONS class FilePathLoader(jinja2.BaseLoader): """ Custom Jinja2 template loader which just loads a single template file """ @@ -35,12 +36,6 @@ def get_source(self, environment, template): class Jinja2TemplateRenderer(object): """ Template renderer """ - ENABLED_EXTENSIONS=( - 'jinja2.ext.i18n', - 'jinja2.ext.do', - 'jinja2.ext.loopcontrols', - ) - UNDEFINED = { 'strict': jinja2.StrictUndefined, # raises errors for undefined variables 'normal': jinja2.Undefined, # can be printed/iterated - error on other operations @@ -53,7 +48,7 @@ def __init__(self, cwd, undefined='strict', no_compact=False, j2_env_params={}): j2_env_params.setdefault('undefined', self.UNDEFINED[undefined]) j2_env_params.setdefault('trim_blocks', not no_compact) j2_env_params.setdefault('lstrip_blocks', not no_compact) - j2_env_params.setdefault('extensions', self.ENABLED_EXTENSIONS) + j2_env_params.setdefault('extensions', JINJA2_ENABLED_EXTENSIONS) j2_env_params.setdefault('loader', FilePathLoader(cwd)) # Environment From b17825d958468be96ae3ba8459be07329a2b8c83 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 5 Nov 2023 23:49:17 +0100 Subject: [PATCH 18/40] cli: Added jj2dep tool for analyzing template dependencies. --- setup.py | 1 + src/jj2cli/__init__.py | 5 ++--- src/jj2cli/cli.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 696d2f6..4eb7363 100755 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ 'console_scripts': [ 'j2 = jj2cli:render', # temporarily keep the old entry point 'jj2 = jj2cli:render', + 'jj2dep = jj2cli:dependencies', ] }, install_requires=[ diff --git a/src/jj2cli/__init__.py b/src/jj2cli/__init__.py index acfebcb..8bb0d40 100644 --- a/src/jj2cli/__init__.py +++ b/src/jj2cli/__init__.py @@ -1,5 +1,4 @@ #! /usr/bin/env python - """ j2cli main file """ import pkg_resources @@ -7,7 +6,7 @@ __email__ = "mstamat@gmail.com" __version__ = pkg_resources.get_distribution('jj2cli').version -from jj2cli.cli import main +from jj2cli.cli import render if __name__ == '__main__': - main() + render() diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index ebd99e7..ead19df 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -127,7 +127,7 @@ def render_command(argv): return result -def main(): +def render(): """ CLI entry point for rendering templates. """ try: output = render_command(sys.argv) @@ -135,3 +135,31 @@ def main(): return 1 outstream = getattr(sys.stdout, 'buffer', sys.stdout) outstream.write(output) + + +def dependencies(): + """ CLI entry point for analyzing template dependencies. """ + #version_info = (__version__, jinja2.__version__) + parser = argparse.ArgumentParser( + description='Analyze Jinja2 templates for dependencies.', + epilog='', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + p_input = parser.add_argument_group('input options') + p_output = parser.add_argument_group('output options') + p_output_mode = p_output.add_mutually_exclusive_group() + ### input options ############################################### + p_input.add_argument('templates', metavar='TEMPLATE', nargs='+', + type=argparse.FileType('r', encoding='utf-8')) + ### output options ############################################## + p_output.add_argument('-f', '--format', + default='make', choices=sorted(DEPENDENCIES_OUTPUT_FORMATS), + help='Specify output format for dependencies.') + p_output_mode.add_argument('-o', metavar='outfile', dest='output_file', + help="Output to a file instead of stdout.") + p_output_mode.add_argument('--per-file', action='store_true', dest='per_file', + help='Produce one output file per input file.') + + args = parser.parse_args() + print(args) + raise NotImplementedError("jj2dep has not yet been implemented.") From 33ac346a717ddf7c597bcb9a6b713b271c39973c Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Mon, 10 May 2021 22:23:32 +0200 Subject: [PATCH 19/40] setup/tests: Modernize tooling and fix tests. Discontinued: - setuptools (using poetry and pyproject.toml from now on) - Travis CI (won't bother contacting support to fix my account [1]) - nose (does not seem to be actively maintained [2]) - six ( - .python-version (used by pyenv) Updated: - tox configuration to use Python >=3.10 - .gitignore - make targets for development/testing Now using: - poetry - github actions - pytest [1] https://archive.ph/lOFGT [2] https://pypi.org/project/nose/#history --- .github/workflows/j2cli.yml | 21 ++ .gitignore | 41 +-- .python-version | 6 - .travis.yml | 31 -- Makefile | 52 +++- __etup.py | 43 +++ pyproject.toml | 119 ++++++++ requirements-dev.txt | 4 - setup.cfg | 8 - setup.py | 90 ------ src/jj2cli/__init__.py | 4 +- src/jj2cli/cli.py | 12 +- src/jj2cli/filters.py | 4 +- src/jj2cli/parsers.py | 13 +- src/jj2cli/render.py | 10 +- tests/__init__.py | 0 tests/render-test.py | 240 --------------- tests/requirements.txt | 1 + tests/resources/custom_tests.py | 2 +- tests/resources/data.yml | 4 - .../resources/data/badext_nginx_data_env.json | 1 + .../resources/data/badext_nginx_data_ini.json | 1 + .../resources/data/badext_nginx_data_json.ini | 1 + .../data/badext_nginx_data_yaml.json | 1 + .../{data.env => data/nginx_data.env} | 0 .../{data.ini => data/nginx_data.ini} | 0 .../{data.json => data/nginx_data.json} | 0 .../{data.yaml => data/nginx_data.yaml} | 0 tests/resources/data/nginx_data.yml | 1 + tests/resources/data/nginx_data_env | 1 + tests/resources/data/nginx_data_ini | 1 + tests/resources/data/nginx_data_json | 1 + tests/resources/data/nginx_data_yaml | 1 + tests/resources/name.j2 | 1 - tests/resources/out/nginx-env.conf | 1 + tests/resources/out/nginx.conf | 10 + .../{nginx-env.j2 => tpl/nginx-env.conf.j2} | 0 .../resources/{nginx.j2 => tpl/nginx.conf.j2} | 0 tests/tba | 81 +++++ tests/tba2 | 33 ++ tests/test_cli.py | 281 ++++++++++++++++++ tox.ini | 40 +-- 42 files changed, 707 insertions(+), 454 deletions(-) create mode 100644 .github/workflows/j2cli.yml delete mode 100644 .python-version delete mode 100644 .travis.yml create mode 100755 __etup.py create mode 100644 pyproject.toml delete mode 100644 requirements-dev.txt delete mode 100644 setup.cfg delete mode 100755 setup.py create mode 100644 tests/__init__.py delete mode 100644 tests/render-test.py create mode 100644 tests/requirements.txt delete mode 100644 tests/resources/data.yml create mode 120000 tests/resources/data/badext_nginx_data_env.json create mode 120000 tests/resources/data/badext_nginx_data_ini.json create mode 120000 tests/resources/data/badext_nginx_data_json.ini create mode 120000 tests/resources/data/badext_nginx_data_yaml.json rename tests/resources/{data.env => data/nginx_data.env} (100%) rename tests/resources/{data.ini => data/nginx_data.ini} (100%) rename tests/resources/{data.json => data/nginx_data.json} (100%) rename tests/resources/{data.yaml => data/nginx_data.yaml} (100%) create mode 120000 tests/resources/data/nginx_data.yml create mode 120000 tests/resources/data/nginx_data_env create mode 120000 tests/resources/data/nginx_data_ini create mode 120000 tests/resources/data/nginx_data_json create mode 120000 tests/resources/data/nginx_data_yaml delete mode 100644 tests/resources/name.j2 create mode 120000 tests/resources/out/nginx-env.conf create mode 100644 tests/resources/out/nginx.conf rename tests/resources/{nginx-env.j2 => tpl/nginx-env.conf.j2} (100%) rename tests/resources/{nginx.j2 => tpl/nginx.conf.j2} (100%) create mode 100644 tests/tba create mode 100644 tests/tba2 create mode 100644 tests/test_cli.py diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml new file mode 100644 index 0000000..e861f2a --- /dev/null +++ b/.github/workflows/j2cli.yml @@ -0,0 +1,21 @@ +name: m000/j2cli +on: + push: + branches: + - "**/*" + pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + runs-on: ubuntu-latest + env: + TOXENV: py3.10-pyyaml6 + steps: + - uses: actions/checkout@v4.1.0 + - uses: actions/setup-python@v4.7.0 + with: + python-version: "3.10" + - run: pip install tox + - run: tox diff --git a/.gitignore b/.gitignore index 896d753..aa1e303 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,21 @@ -# ===[ APP ]=== # - -# ===[ PYTHON PACKAGE ]=== # +# Python Packaging +*.egg/ +*.egg-info/ +pyenv/ /build/ /dist/ /MANIFEST -/*.egg/ -/*.egg-info/ -# ===[ OTHER ]=== # +# Python Caches +__pycache__/ +*.py[cod] +*.pot +*.mo + +# Python Utils +.coverage +/htmlcov/ +/.tox/ # IDE Projects .idea @@ -24,21 +32,14 @@ *.DS_Store Thumbs.db -# Utils -/.tox/ -.sass-cache/ -.coverage - -# Generated -__pycache__ -*.py[cod] -*.pot -*.mo - # Runtime -/*.log -/*.pid +*.log +*.pid + +# Locals +local/ +*.local -# ===[ EXCLUDES ]=== # +# Forced Exclusions !.gitkeep !.htaccess diff --git a/.python-version b/.python-version deleted file mode 100644 index ec22fb4..0000000 --- a/.python-version +++ /dev/null @@ -1,6 +0,0 @@ -j2cli -2.7.16 -3.4.9 -3.5.6 -3.6.8 -3.7.2 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b0725a2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -os: linux -sudo: false -language: python - -matrix: - include: - - python: 2.7 - env: TOXENV=py27 - - python: 3.4 - env: TOXENV=py34 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7-dev - env: TOXENV=py37 - - python: pypy - env: TOXENV=pypy - - python: pypy3 - env: TOXENV=pypy - - {python: 3.6, env: TOXENV=py36-pyyaml5.1} - - {python: 3.6, env: TOXENV=py36-pyyaml3.13} - - {python: 3.6, env: TOXENV=py36-pyyaml3.12} - - {python: 3.6, env: TOXENV=py36-pyyaml3.11} - - {python: 3.6, env: TOXENV=py36-pyyaml3.10} -install: - - pip install tox -cache: - - pip -script: - - tox diff --git a/Makefile b/Makefile index e4b3601..671770d 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,24 @@ -all: +PYPACKAGE=jj2cli +TOX_QUICKTEST=py3.10-pyyaml6 +TOX_LINTTEST=lint + +.PHONY: help all clean test test-lint test-tox test-tox-quick test-pytest test-pytest-cover + +help: ##- Show this help. + @sed -e '/#\{2\}-/!d; s/\\$$//; s/:[^#\t]*/:/; s/#\{2\}- *//' $(MAKEFILE_LIST) -SHELL := /bin/bash +all: + @echo no default action # Package -.PHONY: clean clean: @rm -rf build/ dist/ *.egg-info/ README.md README.rst @pip install -e . # have to reinstall because we are using self -README.md: $(shell find j2cli/) $(wildcard misc/_doc/**) +README.md: $(shell find src/) $(wildcard misc/_doc/**) @python misc/_doc/README.py | python j2cli/__init__.py -f json -o $@ misc/_doc/README.md.j2 -.PHONY: build publish-test publish +.PHONY: build check-pyenv install-pyenv publish-test publish build: README.md @./setup.py build sdist bdist_wheel publish-test: README.md @@ -19,9 +26,34 @@ publish-test: README.md publish: README.md @twine upload dist/* +check-pyenv: +ifeq ($(VIRTUAL_ENV),) + $(error Not in a virtualenv) +else + @printf "Installing in %s.\n" "$(VIRTUAL_ENV)" +endif + +test-lint: ##- Run configured linters. + prospector + +test-tox: ##- Run all Tox tests. + tox + +test-tox-quick: ##- Run test only for the TOX_QUICKTEST profile. + tox -e $(TOX_QUICKTEST) + +test-tox-lint: ##- Run configured linters via Tox. + tox -e $(TOX_LINTTEST) + +test-pytest: + pytest + +#test-nose-cover: ## Use pytest to produce a test coverage report. + #nosetests --with-coverage --cover-package $(PYPACKAGE) + #75 $(COVERAGE_XML): .coveragerc + #76 pytest --cov-report xml:$(@) --cov=. + +test: test-lint test-tox test-nose ##- Run all linters and tests. -.PHONY: test test-tox -test: - @nosetests -test-tox: - @tox +install-pyenv: check-pyenv ##- Install to the current virtualenv. + pip install -e . diff --git a/__etup.py b/__etup.py new file mode 100755 index 0000000..283cd06 --- /dev/null +++ b/__etup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" # jj2cli - Juiced Jinja2 command-line tool + +`jj2cli` (previously `j2cli`) is a command-line tool for templating in +shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) +library. + +Features: + +* Jinja2 templating with support +* Support for data sources in various formats (ini, yaml, json, env) +* Mixing and matching data sources +* Template dependency analysis + +Inspired by [kolypto/j2cli](https://github.com/kolypto/j2cli) and +[mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli). +""" + +from setuptools import setup, find_packages +from sys import version_info as PYVER + + +setup( + long_description=__doc__, + long_description_content_type='text/markdown', + + packages=find_packages('src'), + package_dir={'': 'src'}, + #py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + + scripts=[], + entry_points={ + 'console_scripts': [ + 'j2 = jj2cli:render', # temporarily keep the old entry point + 'jj2 = jj2cli:render', + 'jj2dep = jj2cli:dependencies', + ] + }, + extras_require=dict(packages_extra), + zip_safe=False, + platforms='any', +) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0c9f547 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,119 @@ +# package ########################################################### +[build-system] +requires = ["poetry-core>=1.2.0"] +build-backend = "poetry.core.masonry.api" + +# pytest ############################################################ +[tool.pytest.ini_options] +pythonpath = [ + "./src", +] + +# ruff ############################################################## +[tool.ruff] +target-version = "py310" +line-length = 100 +select = ["C4", "C90", "E", "F", "PL", "PT", "Q", "W"] +ignore = [ + "E731", # Lambdas are our friends. + "Q000", # We don't mind double quotes. For now... + "TRY003", +] +exclude = [ + ".vscode", # needed ??? + "#*.py", # Convention. Unversioned files we don't want to lint or delete. +] + +[tool.ruff.per-file-ignores] + #"__init__.py" = ["F401"] + #"*/settings/*.py" = ["E266", "F401", "F403" , "F405"] + #"**/tests/**/*.py" = ["PLR0913"] + +[tool.ruff.mccabe] +max-complexity = 15 + +# Legend: +# A -> flake8-builtins +# B* -> flake8-bugbear +# C4 -> flake8-comprehensions +# C90 -> mccabe +# DJ -> flake8-django +# E -> pycodestyle error +# F -> pyflakes +# I -> isort +# PT* -> flake8-pytest-style +# PTH* -> flake8-pathlib +# Q -> flake8-quotes +# SIM* -> flake8-simplify +# TRY* -> tryceratops +# W -> pycodestyle warning +# E241 -> Multiple spaces after ','. +# E731 -> Do not assign a lambda expression, use a def. +# F401 -> Module imported but unused. +# F403 -> 'from module import *' used; unable to detect undefined names. +# F405 -> Name may be undefined, or defined from star imports: module. +# PLR0913 -> Too many arguments to function call. +# TRY003 -> Avoid specifying long messages outside the exception class. +# Q000 -> Single quotes found but double quotes preferred. +# +# * -> Considered to be enabled in the future. + +# prospector - mirror in ruff +#strictness: medium +#test-warnings: true + +#ignore-paths: + #- misc + #- docs +#ignore-patterns: + #- pyenv* + +#pylint: + #disable: + #- cyclic-import # doesn't seem to work properly + +#vulture: + # turn on locally to spot unused code + #run: false + +# poetry ############################################################ +[tool.poetry] +name = "jj2cli" +version = "0.6.0" +description = "Juiced Jinja2 command-line tool" +license = "BSD-2-Clause" +authors = [ + "Manolis Stamatogiannakis ", +] +readme = "README.md" +homepage = "https://github.com/m000/j2cli" +repository = "https://github.com/m000/j2cli" +keywords = ['Jinja2', 'templating', 'command-line', 'CLI'] +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Operating System :: OS Independent", + "Topic :: Software Development", + "Natural Language :: English", + "Programming Language :: Python :: 3", +] + +[tool.poetry.scripts] +j2 = "jj2cli:render" # temporarily keep the old entry point +jj2 = "jj2cli:render" +jj2dep = "jj2cli:dependencies" + +[tool.poetry.dependencies] +python = "^3.10" +jinja2 = "^3.1" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.0" +tox = "^4.0" + +[tool.poetry.group.yaml] +optional = true + +[tool.poetry.group.yaml.dependencies] +pyyaml = "^6.0" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f837c52..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -wheel -nose -exdoc -six diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d1dc52c..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bdist_wheel] -universal = 1 - -[metadata] -license_file = LICENSE - -[easy_install] - diff --git a/setup.py b/setup.py deleted file mode 100755 index 4eb7363..0000000 --- a/setup.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -""" # jj2cli - Juiced Jinja2 command-line tool - -`jj2cli` (previously `j2cli`) is a command-line tool for templating in -shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) -library. - -Features: - -* Jinja2 templating with support -* Support for data sources in various formats (ini, yaml, json, env) -* Mixing and matching data sources -* Template dependency analysis - -Inspired by [kolypto/j2cli](https://github.com/kolypto/j2cli) and -[mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli). -""" - -from setuptools import setup, find_packages -from sys import version_info as PYVER - - -### Compatibility packages. -packages_compat = [] -# Jinja2 -if PYVER < (2, 7) or (3, 0) <= PYVER < (3, 5): - packages_compat.append('jinja2 ~= 2.10.0') -else: - packages_compat.append('jinja2 ~= 2.11.0') -# Misc. -if PYVER < (3, 0): - packages_compat.append('shutilwhich ~= 1.1') - packages_compat.append('pathlib ~= 1.0') - -### Packages for optional functionality. -packages_extra = [] -# yaml support -if PYVER < (2, 7) or (2, 7) < PYVER < (3, 4): - # XXX: Python2.6 - packages_extra.append(('yaml', 'pyyaml <= 3.11')) -else: - packages_extra.append(('yaml', 'pyyaml > 5.4')) - - -setup( - name='jj2cli', - version='0.4.0', - author='Manolis Stamatogiannakis', - author_email='mstamat@gmail.com', - - url='https://github.com/m000/j2cli', # XXX: fix before release - license='BSD', - description='Juiced Jinja2 command-line tool.', - long_description=__doc__, # can't do open('README.md').read() because we're describing self - long_description_content_type='text/markdown', - keywords=['Jinja2', 'templating', 'command-line', 'CLI'], - - packages=find_packages('src'), - package_dir={'': 'src'}, - #py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], - include_package_data=True, - - scripts=[], - entry_points={ - 'console_scripts': [ - 'j2 = jj2cli:render', # temporarily keep the old entry point - 'jj2 = jj2cli:render', - 'jj2dep = jj2cli:dependencies', - ] - }, - install_requires=[ - 'six >= 1.13', - packages_compat, - ], - extras_require=dict(packages_extra), - zip_safe=False, - test_suite='nose.collector', - - platforms='any', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Operating System :: OS Independent', - 'Topic :: Software Development', - 'Natural Language :: English', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - ], -) diff --git a/src/jj2cli/__init__.py b/src/jj2cli/__init__.py index 8bb0d40..0a20deb 100644 --- a/src/jj2cli/__init__.py +++ b/src/jj2cli/__init__.py @@ -1,10 +1,10 @@ #! /usr/bin/env python """ j2cli main file """ -import pkg_resources +import importlib.metadata __author__ = "Manolis Stamatogiannakis" __email__ = "mstamat@gmail.com" -__version__ = pkg_resources.get_distribution('jj2cli').version +__version__ = importlib.metadata.version('jj2cli') from jj2cli.cli import render diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index ead19df..40a35be 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -1,18 +1,16 @@ import argparse -import imp import io import logging import os import sys +from functools import reduce +from importlib.machinery import SourceFileLoader + import jinja2 import jinja2.loaders import jinja2.meta -from functools import reduce - -from . import __version__ -from . import filters -from . import parsers +from . import __version__, filters, parsers from .customize import CustomizationModule from .defaults import CONTEXT_FORMATS from .render import Jinja2TemplateRenderer @@ -87,7 +85,7 @@ def render_command(argv): # Customization if args.customize is not None: customize = CustomizationModule( - imp.load_source('customize-module', args.customize) + SourceFileLoader('customize-module', args.customize).load_module() ) else: customize = CustomizationModule(None) diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index ed133f0..b36eb82 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -2,7 +2,7 @@ import os import re import sys -from jinja2 import is_undefined, contextfilter +from jinja2 import is_undefined, pass_context if sys.version_info >= (3,0): from shutil import which @@ -119,7 +119,7 @@ def align_suffix(text, delim, column=None, spaces_after_delim=1): align_suffix.column_previous = None -@contextfilter +@pass_context def ctxlookup(context, key): """ Lookup the value of a key in the template context. """ diff --git a/src/jj2cli/parsers.py b/src/jj2cli/parsers.py index a3221ea..14be114 100644 --- a/src/jj2cli/parsers.py +++ b/src/jj2cli/parsers.py @@ -1,3 +1,4 @@ +import configparser import json import logging import os @@ -5,9 +6,7 @@ import re import sys from pathlib import Path - -import six -from six.moves import collections_abc, configparser +from typing import Mapping from .defaults import (CONTEXT_FORMATS, CONTEXT_FORMATS_ALIASES, DATASPEC_COMPONENTS_MAX, DATASPEC_SEP, yaml_load) @@ -162,7 +161,7 @@ def as_dict(self): d[k].pop('__name__', None) return d ini = MyConfigParser() - ini.readfp(self._iostr) + ini.read_file(self._iostr) return ini.as_dict() def _parse_json(self): @@ -183,11 +182,11 @@ def dict_squash(d, u): :return: Updated version of d. :rtype: dict """ - for k, v in six.iteritems(u): + for k, v in u.items(): dv = d.get(k, {}) - if not isinstance(dv, collections_abc.Mapping): + if not isinstance(dv, Mapping): d[k] = v - elif isinstance(v, collections_abc.Mapping): + elif isinstance(v, Mapping): d[k] = dict_squash(dv, v) else: d[k] = v diff --git a/src/jj2cli/render.py b/src/jj2cli/render.py index c09abe8..970dc1a 100644 --- a/src/jj2cli/render.py +++ b/src/jj2cli/render.py @@ -1,15 +1,17 @@ +import inspect import io -import os import logging +import os +from importlib.machinery import SourceFileLoader import jinja2 import jinja2.loaders - -import imp, inspect +import jinja2.meta from . import filters from .defaults import JINJA2_ENABLED_EXTENSIONS + class FilePathLoader(jinja2.BaseLoader): """ Custom Jinja2 template loader which just loads a single template file """ @@ -70,7 +72,7 @@ def import_tests(self, filename): self.register_tests(self._import_functions(filename)) def _import_functions(self, filename): - m = imp.load_source('imported-funcs', filename) + m = SourceFileLoader('imported-funcs', filename).load_module() return dict((name, func) for name, func in inspect.getmembers(m) if inspect.isfunction(func)) def render(self, template_path, context): diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/render-test.py b/tests/render-test.py deleted file mode 100644 index 791764a..0000000 --- a/tests/render-test.py +++ /dev/null @@ -1,240 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import unicode_literals - -import unittest -import os, sys, io, os.path, tempfile -from copy import copy -from contextlib import contextmanager -from jinja2.exceptions import UndefinedError - -from j2cli.cli import render_command - -@contextmanager -def mktemp(contents): - """ Create a temporary file with the given contents, and yield its path """ - _, path = tempfile.mkstemp() - fp = io.open(path, 'wt+', encoding='utf-8') - fp.write(contents) - fp.flush() - try: - yield path - finally: - fp.close() - os.unlink(path) - - -@contextmanager -def mock_environ(new_env): - old_env = copy(os.environ) - os.environ.update(new_env) - yield - os.environ.clear() - os.environ.update(old_env) - - -class RenderTest(unittest.TestCase): - def setUp(self): - os.chdir( - os.path.dirname(__file__) - ) - - def _testme(self, argv, expected_output, stdin=None, env=None): - """ Helper test shortcut """ - with mock_environ(env or {}): - result = render_command(os.getcwd(), env or {}, stdin, argv) - if isinstance(result, bytes): - result = result.decode('utf-8') - self.assertEqual(result, expected_output) - - #: The expected output - expected_output = """server { - listen 80; - server_name localhost; - - root /var/www/project; - index index.htm; - - access_log /var/log/nginx//http.access.log combined; - error_log /var/log/nginx//http.error.log; -} -""" - - def _testme_std(self, argv, stdin=None, env=None): - self._testme(argv, self.expected_output, stdin, env) - - def test_ini(self): - # Filename - self._testme_std(['resources/nginx.j2', 'resources/data.ini']) - # Format - self._testme_std(['--format=ini', 'resources/nginx.j2', 'resources/data.ini']) - # Stdin - self._testme_std(['--format=ini', 'resources/nginx.j2'], stdin=open('resources/data.ini')) - self._testme_std(['--format=ini', 'resources/nginx.j2', '-'], stdin=open('resources/data.ini')) - - def test_json(self): - # Filename - self._testme_std(['resources/nginx.j2', 'resources/data.json']) - # Format - self._testme_std(['--format=json', 'resources/nginx.j2', 'resources/data.json']) - # Stdin - self._testme_std(['--format=json', 'resources/nginx.j2'], stdin=open('resources/data.json')) - self._testme_std(['--format=json', 'resources/nginx.j2', '-'], stdin=open('resources/data.json')) - - def test_yaml(self): - try: - import yaml - except ImportError: - raise unittest.SkipTest('Yaml lib not installed') - - # Filename - self._testme_std(['resources/nginx.j2', 'resources/data.yml']) - self._testme_std(['resources/nginx.j2', 'resources/data.yaml']) - # Format - self._testme_std(['--format=yaml', 'resources/nginx.j2', 'resources/data.yml']) - # Stdin - self._testme_std(['--format=yaml', 'resources/nginx.j2'], stdin=open('resources/data.yml')) - self._testme_std(['--format=yaml', 'resources/nginx.j2', '-'], stdin=open('resources/data.yml')) - - def test_env(self): - # Filename - self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) - self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) - # Format - self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) - self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) - # Stdin - self._testme_std(['--format=env', 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) - self._testme_std([ 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) - - # Environment! - # In this case, it's not explicitly provided, but implicitly gotten from the environment - env = dict(NGINX_HOSTNAME='localhost', NGINX_WEBROOT='/var/www/project', NGINX_LOGS='/var/log/nginx/') - self._testme_std(['--format=env', 'resources/nginx-env.j2'], env=env) - self._testme_std([ 'resources/nginx-env.j2'], env=env) - - def test_import_env(self): - # Import environment into a variable - with mktemp('{{ a }}/{{ env.B }}') as template: - with mktemp('{"a":1}') as context: - self._testme(['--format=json', '--import-env=env', template, context], '1/2', env=dict(B='2')) - # Import environment into global scope - with mktemp('{{ a }}/{{ B }}') as template: - with mktemp('{"a":1,"B":1}') as context: - self._testme(['--format=json', '--import-env=', template, context], '1/2', env=dict(B='2')) - - def test_env_file__equals_sign_in_value(self): - # Test whether environment variables with "=" in the value are parsed correctly - with mktemp('{{ A|default('') }}/{{ B }}/{{ C }}') as template: - with mktemp('A\nB=1\nC=val=1\n') as context: - self._testme(['--format=env', template, context], '/1/val=1') - - def test_unicode(self): - # Test how unicode is handled - # I'm using Russian language for unicode :) - with mktemp('Проверка {{ a }} связи!') as template: - with mktemp('{"a": "широкополосной"}') as context: - self._testme(['--format=json', template, context], 'Проверка широкополосной связи!') - - # Test case from issue #17: unicode environment variables - if sys.version_info[0] == 2: - # Python 2: environment variables are bytes - self._testme(['resources/name.j2'], u'Hello Jürgen!\n', env=dict(name=b'J\xc3\xbcrgen')) - else: - # Python 3: environment variables are unicode strings - self._testme(['resources/name.j2'], u'Hello Jürgen!\n', env=dict(name=u'Jürgen')) - - def test_filters__env(self): - with mktemp('user_login: kolypto') as yml_file: - with mktemp('{{ user_login }}:{{ "USER_PASS"|env }}') as template: - # Test: template with an env variable - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) - - # environment cleaned up - assert 'USER_PASS' not in os.environ - - # Test: KeyError - with self.assertRaises(KeyError): - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict()) - - # Test: default - with mktemp('{{ user_login }}:{{ "USER_PASS"|env("-none-") }}') as template: - self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) - - # Test: using as a function - with mktemp('{{ user_login }}:{{ env("USER_PASS") }}') as template: - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) - - with self.assertRaises(KeyError): - # Variable not set - self._testme(['--format=yaml', template, yml_file], '', env=dict()) - - # Test: using as a function, with a default - with mktemp('{{ user_login }}:{{ env("USER_PASS", "-none-") }}') as template: - self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) - self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) - - - def test_custom_filters(self): - with mktemp('{{ a|parentheses }}') as template: - self._testme(['--format=env', '--filters=resources/custom_filters.py', template], '(1)', env=dict(a='1')) - - def test_custom_tests(self): - with mktemp('{% if a|int is custom_odd %}odd{% endif %}') as template: - self._testme(['--format=env', '--tests=resources/custom_tests.py', template], 'odd', env=dict(a='1')) - - def test_output_file(self): - with mktemp('{{ a }}') as template: - try: - self._testme(['-o', '/tmp/j2-out', template], '', env=dict(a='123')) - self.assertEqual('123', io.open('/tmp/j2-out', 'r').read()) - finally: - os.unlink('/tmp/j2-out') - - def test_undefined(self): - """ Test --undefined """ - # `name` undefined: error - self.assertRaises(UndefinedError, self._testme, ['resources/name.j2'], u'Hello !\n', env=dict()) - # `name` undefined: no error - self._testme(['--undefined', 'resources/name.j2'], u'Hello !\n', env=dict()) - - def test_jinja2_extensions(self): - """ Test that an extension is enabled """ - with mktemp('{% do [] %}') as template: - # `do` tag is an extension - self._testme([template], '') - - - def test_customize(self): - """ Test --customize """ - # Test: j2_environment_params() - # Custom tag start/end - with mktemp('<% if 1 %>1<% else %>2<% endif %>') as template: - self._testme(['--customize=resources/customize.py', template], '1') - - # Test: j2_environment() - # custom function: my_function - with mktemp('<< my_function("hey") >>') as template: - self._testme(['--customize=resources/customize.py', template], 'my function says "hey"') - - # Test: alter_context() - # Extra variable: ADD=127 - with mktemp('<< ADD >>') as template: - self._testme(['--customize=resources/customize.py', template], '127') - - # Test: extra_filters() - with mktemp('<< ADD|parentheses >>') as template: - self._testme(['--customize=resources/customize.py', template], '(127)') - - # Test: extra_tests() - with mktemp('<% if ADD|int is custom_odd %>odd<% endif %>') as template: - self._testme(['--customize=resources/customize.py', template], 'odd') - - # reset - # otherwise it will load the same module even though its name has changed - del sys.modules['customize-module'] - - # Test: no hooks in a file - # Got to restore to the original configuration and use {% %} again - with mktemp('{% if 1 %}1{% endif %}') as template: - self._testme(['--customize=render-test.py', template], '1') diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..e079f8a --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1 @@ +pytest diff --git a/tests/resources/custom_tests.py b/tests/resources/custom_tests.py index 7292712..c3e6ef8 100644 --- a/tests/resources/custom_tests.py +++ b/tests/resources/custom_tests.py @@ -1,3 +1,3 @@ def custom_odd(n): - return True if (n % 2) else False + return bool(n % 2) diff --git a/tests/resources/data.yml b/tests/resources/data.yml deleted file mode 100644 index efd4cee..0000000 --- a/tests/resources/data.yml +++ /dev/null @@ -1,4 +0,0 @@ -nginx: - hostname: localhost - webroot: /var/www/project - logs: /var/log/nginx/ diff --git a/tests/resources/data/badext_nginx_data_env.json b/tests/resources/data/badext_nginx_data_env.json new file mode 120000 index 0000000..46700b2 --- /dev/null +++ b/tests/resources/data/badext_nginx_data_env.json @@ -0,0 +1 @@ +nginx_data.env \ No newline at end of file diff --git a/tests/resources/data/badext_nginx_data_ini.json b/tests/resources/data/badext_nginx_data_ini.json new file mode 120000 index 0000000..433c79d --- /dev/null +++ b/tests/resources/data/badext_nginx_data_ini.json @@ -0,0 +1 @@ +nginx_data.ini \ No newline at end of file diff --git a/tests/resources/data/badext_nginx_data_json.ini b/tests/resources/data/badext_nginx_data_json.ini new file mode 120000 index 0000000..94052c7 --- /dev/null +++ b/tests/resources/data/badext_nginx_data_json.ini @@ -0,0 +1 @@ +nginx_data.json \ No newline at end of file diff --git a/tests/resources/data/badext_nginx_data_yaml.json b/tests/resources/data/badext_nginx_data_yaml.json new file mode 120000 index 0000000..6f587f5 --- /dev/null +++ b/tests/resources/data/badext_nginx_data_yaml.json @@ -0,0 +1 @@ +nginx_data.yaml \ No newline at end of file diff --git a/tests/resources/data.env b/tests/resources/data/nginx_data.env similarity index 100% rename from tests/resources/data.env rename to tests/resources/data/nginx_data.env diff --git a/tests/resources/data.ini b/tests/resources/data/nginx_data.ini similarity index 100% rename from tests/resources/data.ini rename to tests/resources/data/nginx_data.ini diff --git a/tests/resources/data.json b/tests/resources/data/nginx_data.json similarity index 100% rename from tests/resources/data.json rename to tests/resources/data/nginx_data.json diff --git a/tests/resources/data.yaml b/tests/resources/data/nginx_data.yaml similarity index 100% rename from tests/resources/data.yaml rename to tests/resources/data/nginx_data.yaml diff --git a/tests/resources/data/nginx_data.yml b/tests/resources/data/nginx_data.yml new file mode 120000 index 0000000..6f587f5 --- /dev/null +++ b/tests/resources/data/nginx_data.yml @@ -0,0 +1 @@ +nginx_data.yaml \ No newline at end of file diff --git a/tests/resources/data/nginx_data_env b/tests/resources/data/nginx_data_env new file mode 120000 index 0000000..46700b2 --- /dev/null +++ b/tests/resources/data/nginx_data_env @@ -0,0 +1 @@ +nginx_data.env \ No newline at end of file diff --git a/tests/resources/data/nginx_data_ini b/tests/resources/data/nginx_data_ini new file mode 120000 index 0000000..433c79d --- /dev/null +++ b/tests/resources/data/nginx_data_ini @@ -0,0 +1 @@ +nginx_data.ini \ No newline at end of file diff --git a/tests/resources/data/nginx_data_json b/tests/resources/data/nginx_data_json new file mode 120000 index 0000000..94052c7 --- /dev/null +++ b/tests/resources/data/nginx_data_json @@ -0,0 +1 @@ +nginx_data.json \ No newline at end of file diff --git a/tests/resources/data/nginx_data_yaml b/tests/resources/data/nginx_data_yaml new file mode 120000 index 0000000..6f587f5 --- /dev/null +++ b/tests/resources/data/nginx_data_yaml @@ -0,0 +1 @@ +nginx_data.yaml \ No newline at end of file diff --git a/tests/resources/name.j2 b/tests/resources/name.j2 deleted file mode 100644 index 6cb905f..0000000 --- a/tests/resources/name.j2 +++ /dev/null @@ -1 +0,0 @@ -Hello {{name}}! diff --git a/tests/resources/out/nginx-env.conf b/tests/resources/out/nginx-env.conf new file mode 120000 index 0000000..4ed253b --- /dev/null +++ b/tests/resources/out/nginx-env.conf @@ -0,0 +1 @@ +nginx.conf \ No newline at end of file diff --git a/tests/resources/out/nginx.conf b/tests/resources/out/nginx.conf new file mode 100644 index 0000000..00547e1 --- /dev/null +++ b/tests/resources/out/nginx.conf @@ -0,0 +1,10 @@ +server { + listen 80; + server_name localhost; + + root /var/www/project; + index index.htm; + + access_log /var/log/nginx//http.access.log combined; + error_log /var/log/nginx//http.error.log; +} diff --git a/tests/resources/nginx-env.j2 b/tests/resources/tpl/nginx-env.conf.j2 similarity index 100% rename from tests/resources/nginx-env.j2 rename to tests/resources/tpl/nginx-env.conf.j2 diff --git a/tests/resources/nginx.j2 b/tests/resources/tpl/nginx.conf.j2 similarity index 100% rename from tests/resources/nginx.j2 rename to tests/resources/tpl/nginx.conf.j2 diff --git a/tests/tba b/tests/tba new file mode 100644 index 0000000..a00a6fc --- /dev/null +++ b/tests/tba @@ -0,0 +1,81 @@ + def rest_filters__env(self): + pass + #with mktemp('user_login: kolypto') as yml_file: + #with mktemp('{{ user_login }}:{{ "USER_PASS"|env }}') as template: + # Test: template with an env variable + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) + + # environment cleaned up + #assert 'USER_PASS' not in os.environ + + # Test: KeyError + #with self.assertRaises(KeyError): + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict()) + + # Test: default + #with mktemp('{{ user_login }}:{{ "USER_PASS"|env("-none-") }}') as template: + #self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) + + # Test: using as a function + #with mktemp('{{ user_login }}:{{ env("USER_PASS") }}') as template: + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) + + #with self.assertRaises(KeyError): + # Variable not set + #self._testme(['--format=yaml', template, yml_file], '', env=dict()) + + # Test: using as a function, with a default + #with mktemp('{{ user_login }}:{{ env("USER_PASS", "-none-") }}') as template: + #self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) + #self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) + + + def rest_custom_filters(self): + pass + #with mktemp('{{ a|parentheses }}') as template: + #self._testme(['--format=env', '--filters=resources/custom_filters.py', template], '(1)', env=dict(a='1')) + + #def rest_custom_tests(self): + #with mktemp('{% if a|int is custom_odd %}odd{% endif %}') as template: + #self._testme(['--format=env', '--tests=resources/custom_tests.py', template], 'odd', env=dict(a='1')) + + def rest_jinja2_extensions(self): + """ Test that an extension is enabled """ + #with mktemp('{% do [] %}') as template: + ## `do` tag is an extension + #self._testme([template], '') + + + def rest_customize(self): + """ Test --customize """ + # Test: j2_environment_params() + # Custom tag start/end + #with mktemp('<% if 1 %>1<% else %>2<% endif %>') as template: + #self._testme(['--customize=resources/customize.py', template], '1') + + # Test: j2_environment() + # custom function: my_function + #with mktemp('<< my_function("hey") >>') as template: + #self._testme(['--customize=resources/customize.py', template], 'my function says "hey"') + + # Test: alter_context() + # Extra variable: ADD=127 + #with mktemp('<< ADD >>') as template: + #self._testme(['--customize=resources/customize.py', template], '127') + + # Test: extra_filters() + #with mktemp('<< ADD|parentheses >>') as template: + #self._testme(['--customize=resources/customize.py', template], '(127)') + + # Test: extra_tests() + #with mktemp('<% if ADD|int is custom_odd %>odd<% endif %>') as template: + #self._testme(['--customize=resources/customize.py', template], 'odd') + + # reset + # otherwise it will load the same module even though its name has changed + #del sys.modules['customize-module'] + + # Test: no hooks in a file + # Got to restore to the original configuration and use {% %} again + #with mktemp('{% if 1 %}1{% endif %}') as template: + #self._testme(['--customize=render-test.py', template], '1') diff --git a/tests/tba2 b/tests/tba2 new file mode 100644 index 0000000..7aa69f5 --- /dev/null +++ b/tests/tba2 @@ -0,0 +1,33 @@ + def rest_import_env(self): + pass + # Import environment into a variable + #with mktemp('{{ a }}/{{ env.B }}') as template: + #with mktemp('{"a":1}') as context: + #self._testme(['--format=json', '--import-env=env', template, context], '1/2', env=dict(B='2')) + # Import environment into global scope + #with mktemp('{{ a }}/{{ B }}') as template: + #with mktemp('{"a":1,"B":1}') as context: + #self._testme(['--format=json', '--import-env=', template, context], '1/2', env=dict(B='2')) + + def rest_env_file__equals_sign_in_value(self): + pass + # Test whether environment variables with "=" in the value are parsed correctly + #with mktemp('{{ A|default('') }}/{{ B }}/{{ C }}') as template: + #with mktemp('A\nB=1\nC=val=1\n') as context: + #self._testme(['--format=env', template, context], '/1/val=1') + + def rest_unicode(self): + pass + # Test how unicode is handled + # I'm using Russian language for unicode :) + #with mktemp('Проверка {{ a }} связи!') as template: + #with mktemp('{"a": "широкополосной"}') as context: + #self._testme(['--format=json', template, context], 'Проверка широкополосной связи!') + + # Test case from issue #17: unicode environment variables + #if sys.version_info[0] == 2: + # Python 2: environment variables are bytes + #self._testme(['resources/tpl/name.j2'], u'Hello Jürgen!\n', env=dict(name=b'J\xc3\xbcrgen')) + #else: + # Python 3: environment variables are unicode strings + #self._testme(['resources/tpl/name.j2'], u'Hello Jürgen!\n', env=dict(name=u'Jürgen')) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7222a89 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- +import importlib +import logging +import os +import shlex +import sys +import tempfile +import unittest +from contextlib import contextmanager +from pathlib import Path +from typing import Iterable + +from jj2cli.cli import render_command +from jj2cli.parsers import DATASPEC_SEP + + +@contextmanager +def stdin_from(fin, *args, **kwargs): + """Use the contents of the specified file as stdin, and yield the original stdin.""" + if not isinstance(fin, Path): + fin = Path(fin) + stdin_bak = sys.stdin + sys.stdin = fin.open(*args, **kwargs) + logging.debug("STDIN from: %s", fin) + try: + yield + finally: + sys.stdin.close() + sys.stdin = stdin_bak + +@contextmanager +def temp_file(contents, suffix=None, text=True): + """Create a temporary file with the specified contents, and yield its path.""" + fd, f = tempfile.mkstemp(suffix=suffix, text=text) + fp = os.fdopen(fd, 'w') + fp.write(contents) + fp.close() + f = Path(f) + logging.debug("TEMP created: %s", f) + try: + yield f + finally: + f.unlink() + +@contextmanager +def temp_files(specs): + """Create multiple temporary files and yield their paths.""" + tempfiles = [] + for spec in specs: + if len(spec) == 3: + contents, suffix, text = spec + elif len(spec) == 2: + contents, suffix, text = spec + (True,) + else: + raise ValueError("Bad spec for temp file: %s" % repr(spec)) + fd, f = tempfile.mkstemp(suffix=suffix, text=text) + fp = os.fdopen(fd, 'w') + fp.write(contents) + fp.close() + f = Path(f) + tempfiles.append(f) + logging.debug("TEMP created: %s", f) + try: + yield tuple(tempfiles) + finally: + map(Path.unlink, tempfiles) + +@contextmanager +def environment(env): + """Temporarily set values from env in the environment.""" + env_bak = os.environ + os.environ = env_bak.copy() + os.environ.update(env) + try: + yield + finally: + os.environ = env_bak + +class RenderTest(unittest.TestCase): + WORKDIR = Path(__file__).parent + TPLDIR = Path('resources') / 'tpl' + DATADIR = Path('resources') / 'data' + OUTDIR = Path('resources') / 'out' + + def setUp(self): + os.chdir(self.WORKDIR) + + def _render_prep(self, tpl, data, expected_output, extra_args): + """ Helper for processing common options for test runners. + + data paths are expected to be relative + """ + tpl = self.TPLDIR / tpl + _data = data if isinstance(data, Iterable) and not isinstance(data, str) else [data] + data = [] + for dspec in _data: + dspec = str(dspec) if isinstance(dspec, Path) else dspec + p, sep, modifiers = dspec.partition(DATASPEC_SEP) + p = str(self.DATADIR / p) if (self.DATADIR / p).is_file() else p + data.append('%s%s%s' % (p, sep, modifiers)) + expected_output = (Path(expected_output) + if expected_output is not None + else self.OUTDIR / tpl.stem).read_text() + extra_args = [] if not extra_args else shlex.split(extra_args) + + return (str(tpl), data, expected_output, extra_args) + + def _class_for_name(self, fullcname): + """ Helper for getting a class object from its string representation. + """ + mname, _, cname = fullcname.rpartition('.') + try: + m = importlib.import_module(mname if mname else 'builtins') + c = getattr(m, cname) + return c + except (ImportError, AttributeError): + return None + + # pylint: disable=too-many-arguments + def _render_test(self, tpl, data=None, expected_output=None, + extra_args=None, exception=None, exception_msg=None): + """ Helper for rendering `tpl` using `data` and checking the results + against `expected_results`. Rendering is expected to succeed + without errors. + """ + tpl, data, expected_output, extra_args = self._render_prep( + tpl, data, expected_output, extra_args) + argv = ['dummy_command_name', *extra_args, tpl, *data] + logging.debug("PASSED_ARGS render_command: %s", argv) + + if exception is None: + result = render_command(argv) + if isinstance(result, bytes): + # XXX: maybe render_command() should just return utf-8? + result = result.decode('utf-8') + self.assertEqual(result, expected_output) + elif exception_msg is None: + c = self._class_for_name(exception) + self.assertRaises(c, render_command, argv) + else: + c = self._class_for_name(exception) + self.assertRaisesRegex(c, exception_msg, render_command, argv) + # pylint: enable=too-many-arguments + + def test_ENV(self): + """ Tests rendering with environment variables. + """ + with environment({"MYVAR": "test"}), temp_files(( + ("XXX{{ MYVAR }}XXX", ".j2"), + ("MYVAR=bad", ".env"), + ("XXXtestXXX", ".out"), + )) as (tpl, in_ignored, out_normal): + self._render_test(tpl, ":ENV", out_normal) + self._render_test(tpl, "-:ENV", out_normal, extra_args='--') + self._render_test(tpl, "%s:ENV" % (in_ignored), out_normal) + + def test_env(self): + """ Tests rendering with a single data file in env format. + """ + # simple render + self._render_test("nginx-env.conf.j2", "nginx_data.env") + # file + fallback format + self._render_test("nginx-env.conf.j2", "nginx_data_env", extra_args='--fallback-format=env') + # file + format override + self._render_test("nginx-env.conf.j2", "badext_nginx_data_env.json:env") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_env"): + self._render_test("nginx-env.conf.j2", "-", extra_args='--fallback-format=env') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_env"): + self._render_test("nginx-env.conf.j2", ":env") + with stdin_from(self.DATADIR / "nginx_data_env"): + self._render_test("nginx-env.conf.j2", "-:env", extra_args='--') + # file + default fallback format - failure + self._render_test("nginx.conf.j2", "nginx_data_env", + exception='configparser.MissingSectionHeaderError', + exception_msg='no section headers') + + def test_ini(self): + """ Tests rendering with a single data file in ini format. + """ + # simple render + self._render_test("nginx.conf.j2", "nginx_data.ini") + # file + fallback format + self._render_test("nginx.conf.j2", "nginx_data_ini", extra_args='--fallback-format=ini') + # file + format override + self._render_test("nginx.conf.j2", "badext_nginx_data_ini.json:ini") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_ini"): + self._render_test("nginx.conf.j2", "-", extra_args='--fallback-format=ini') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_ini"): + self._render_test("nginx.conf.j2", ":ini") + with stdin_from(self.DATADIR / "nginx_data_ini"): + self._render_test("nginx.conf.j2", "-:ini", extra_args='--') + # file + default fallback format - success + self._render_test("nginx.conf.j2", "nginx_data_ini") + + def test_json(self): + """ Tests rendering with a single data file in json format. + """ + # simple render + self._render_test("nginx.conf.j2", "nginx_data.json") + # file + fallback format + self._render_test("nginx.conf.j2", "nginx_data_json", extra_args='--fallback-format=json') + # file + format override + self._render_test("nginx.conf.j2", "badext_nginx_data_json.ini:json") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_json"): + self._render_test("nginx.conf.j2", "-", extra_args='--fallback-format=json') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_json"): + self._render_test("nginx.conf.j2", ":json") + with stdin_from(self.DATADIR / "nginx_data_json"): + self._render_test("nginx.conf.j2", "-:json", extra_args='--') + # file + default fallback format - failure + self._render_test("nginx.conf.j2", "nginx_data_json", + exception='configparser.MissingSectionHeaderError', + exception_msg='no section headers') + + def test_yaml(self): + """ Tests rendering with a single data file in yaml format. + """ + try: + importlib.import_module('yaml') + except ImportError: + raise unittest.SkipTest('yaml module not available') + # simple render + self._render_test("nginx.conf.j2", "nginx_data.yaml") + self._render_test("nginx.conf.j2", "nginx_data.yml") + # file + fallback format + self._render_test("nginx.conf.j2", "nginx_data_yaml", extra_args='--fallback-format=yaml') + # file + format override + self._render_test("nginx.conf.j2", "badext_nginx_data_yaml.json:yaml") + # stdin + fallback format + with stdin_from(self.DATADIR / "nginx_data_yaml"): + self._render_test("nginx.conf.j2", "-", extra_args='--fallback-format=yaml') + # stdin + format override + with stdin_from(self.DATADIR / "nginx_data_yaml"): + self._render_test("nginx.conf.j2", ":yaml") + with stdin_from(self.DATADIR / "nginx_data_yaml"): + self._render_test("nginx.conf.j2", "-:yaml", extra_args='--') + # file + default fallback format - failure + self._render_test("nginx.conf.j2", "nginx_data_yaml", + exception='configparser.MissingSectionHeaderError', + exception_msg='no section headers') + + def test_ignore_missing(self): + """ Tests the -I/--ignore missing flag. + """ + self._render_test("nginx.conf.j2", ["nginx_data_json", "nginx_data_missing"], + exception='FileNotFoundError', + exception_msg='nginx_data_missing', + extra_args='-f json') + self._render_test("nginx.conf.j2", ["nginx_data_json", "nginx_data_missing"], + extra_args='-I -f json') + self._render_test("nginx.conf.j2", ["nginx_data_json", "nginx_data_missing"], + extra_args='--ignore-missing -f json') + + def test_undefined(self): + """ Tests the -U/--undefined flag. + """ + with temp_files(( + ("XXX{{ undefined_var }}XXX", ".j2"), + ("{}", ".json"), + ("XXXXXX", ".out"), + ("XXX{{ undefined_var }}XXX", ".out"), + )) as (tpl, data, out_normal, out_debug): + # default (strict) + self._render_test(tpl, data, out_normal, + exception='jinja2.exceptions.UndefinedError', + exception_msg='undefined_var') + # strict + self._render_test(tpl, data, out_normal, + exception='jinja2.exceptions.UndefinedError', + exception_msg='undefined_var', + extra_args='--undefined strict') + # normal + self._render_test(tpl, data, out_normal, extra_args='--undefined normal') + # debug + self._render_test(tpl, data, out_debug, extra_args='--undefined debug') diff --git a/tox.ini b/tox.ini index b7db8b5..352dd54 100644 --- a/tox.ini +++ b/tox.ini @@ -1,26 +1,32 @@ [tox] -envlist=py{27,34,35,36,37},pypy, - py36-pyyaml5.1 - py36-pyyaml3.13 - py36-pyyaml3.12 - py36-pyyaml3.11 - py36-pyyaml3.10 +envlist= + py{3.10}-pyyaml{0,6} skip_missing_interpreters=True [testenv] +basepython= + py3.10: python3.10 deps= - -rrequirements-dev.txt - py{27,34,35,36},pypy: -e.[yaml] - py37: pyyaml - py36-pyyaml5.1: pyyaml==5.1 - py36-pyyaml3.13: pyyaml==3.13 - py36-pyyaml3.12: pyyaml==3.12 - py36-pyyaml3.11: pyyaml==3.11 - py36-pyyaml3.10: pyyaml==3.10 + -rtests/requirements.txt + pyyaml0: null + pyyaml6: pyyaml~=6.0 +allowlist_externals = pytest commands= - nosetests {posargs:tests/} -whitelist_externals=make + pytest {posargs:tests/} + +#[testenv:lint] +#basepython= +# py{3.10}-pyyaml{6} +#deps= +# -rtests/requirements.txt +# pyyaml6: pyyaml>=6 +#commands= +# prospector [testenv:dev] -deps=-rrequirements-dev.txt +deps= + -rtest/requirements.txt usedevelop=True + +# To see how these expand to tests, see: +# https://tox.readthedocs.io/en/latest/example/basic.html#compressing-dependency-matrix From 35bf635f6da723887e835f0b8ca731534117351a Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Fri, 21 May 2021 16:00:31 +0200 Subject: [PATCH 20/40] Removed docs produced by exdoc tool. --- misc/_doc/README.md.j2 | 214 ----------------------------------------- misc/_doc/README.py | 26 ----- 2 files changed, 240 deletions(-) delete mode 100644 misc/_doc/README.md.j2 delete mode 100755 misc/_doc/README.py diff --git a/misc/_doc/README.md.j2 b/misc/_doc/README.md.j2 deleted file mode 100644 index d068d7f..0000000 --- a/misc/_doc/README.md.j2 +++ /dev/null @@ -1,214 +0,0 @@ -[![Build Status](https://travis-ci.org/kolypto/j2cli.svg)](https://travis-ci.org/kolypto/j2cli) -[![Pythons](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.4%E2%80%933.7%20%7C%20pypy-blue.svg)](.travis.yml) - -j2cli - Jinja2 Command-Line Tool -================================ - -`j2cli` is a command-line tool for templating in shell-scripts, -leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. - -Features: - -* Jinja2 templating -* INI, YAML, JSON data sources supported -* Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) - -Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) - -## Installation - -``` -pip install j2cli -``` - -To enable the YAML support with [pyyaml](http://pyyaml.org/): - -``` -pip install j2cli[yaml] -``` - -## Tutorial - -Suppose, you want to have an nginx configuration file template, `nginx.j2`: - -{% raw %}```jinja2 -server { - listen 80; - server_name {{ nginx.hostname }}; - - root {{ nginx.webroot }}; - index index.htm; -} -```{% endraw %} - -And you have a JSON file with the data, `nginx.json`: - -```json -{ - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project" - } -} -``` - -This is how you render it into a working configuration file: - -```bash -$ j2 -f json nginx.j2 nginx.json > nginx.conf -``` - -The output is saved to `nginx.conf`: - -``` -server { - listen 80; - server_name localhost; - - root /var/www/project; - index index.htm; -} -``` - -Alternatively, you can use the `-o nginx.conf` option. - -## Tutorial with environment variables - -Suppose, you have a very simple template, `person.xml`: - -{% raw %}```jinja2 -{{ name }}{{ age }} -```{% endraw %} - -What is the easiest way to use j2 here? -Use environment variables in your bash script: - -```bash -$ export name=Andrew -$ export age=31 -$ j2 /tmp/person.xml -Andrew31 -``` - -## Using environment variables - -Even when you use yaml or json as the data source, you can always access environment variables -using the `env()` function: - -{% raw %}```jinja2 -Username: {{ login }} -Password: {{ env("APP_PASSWORD") }} -```{% endraw %} - - -## Usage - -Compile a template using INI-file data source: - - $ j2 config.j2 data.ini - -Compile using JSON data source: - - $ j2 config.j2 data.json - -Compile using YAML data source (requires PyYAML): - - $ j2 config.j2 data.yaml - -Compile using JSON data on stdin: - - $ curl http://example.com/service.json | j2 --format=json config.j2 - -Compile using environment variables (hello Docker!): - - $ j2 config.j2 - -Or even read environment variables from a file: - - $ j2 --format=env config.j2 data.env - -Or pipe it: (note that you'll have to use the "-" in this particular case): - - $ j2 --format=env config.j2 - < data.env - - -# Reference -`j2` accepts the following arguments: - -* `template`: Jinja2 template file to render -* `data`: (optional) path to the data used for rendering. - The default is `-`: use stdin. Specify it explicitly when using env! - -Options: - -* `--format, -f`: format for the data file. The default is `?`: guess from file extension. -* `--import-env VAR, -e EVAR`: import all environment variables into the template as `VAR`. - To import environment variables into the global scope, give it an empty string: `--import-env=`. - (This will overwrite any existing variables!) -* `-o outfile`: Write rendered template to a file -* `--undefined`: Allow undefined variables to be used in templates (no error will be raised) - -* `--filters filters.py`: Load custom Jinja2 filters and tests from a Python file. - Will load all top-level functions and register them as filters. - This option can be used multiple times to import several files. -* `--tests tests.py`: Load custom Jinja2 filters and tests from a Python file. -* `--customize custom.py`: A Python file that implements hooks to fine-tune the j2cli behavior. - This is fairly advanced stuff, use it only if you really need to customize the way Jinja2 is initialized. - See [Customization](#customization) for more info. - -There is some special behavior with environment variables: - -* When `data` is not provided (data is `-`), `--format` defaults to `env` and thus reads environment variables -* When `--format=env`, it can read a special "environment variables" file made like this: `env > /tmp/file.env` - -## Formats - -{% for name, format in formats|dictsort() %} -### {{ name }} -{{ format.doc }} -{% endfor %} - - - -Extras -====== - -## Filters - -{% for name, filter in extras.filters|dictsort() %} -### `{{ filter.qsignature }}` -{{ filter.doc }} -{% endfor %} - - - -Customization -============= - -j2cli now allows you to customize the way the application is initialized: - -* Pass additional keywords to Jinja2 environment -* Modify the context before it's used for rendering -* Register custom filters and tests - -This is done through *hooks* that you implement in a customization file in Python language. -Just plain functions at the module level. - -The following hooks are available: - -* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for - [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). -* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. -* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be - used for template rendering. You can do all sorts of pre-processing here. -* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 -* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 - -All of them are optional. - -The example customization.py file for your reference: - -```python -{% include "tests/resources/customize.py" %} -``` - diff --git a/misc/_doc/README.py b/misc/_doc/README.py deleted file mode 100755 index c81d7d2..0000000 --- a/misc/_doc/README.py +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env python - -import json -import inspect -from exdoc import doc, getmembers - -import j2cli -import j2cli.context -import j2cli.extras.filters - - -README = { - 'formats': { - name: doc(f) - for name, f in j2cli.context.FORMATS.items() - }, - 'extras': { - 'filters': {k: doc(v) - for k, v in getmembers(j2cli.extras.filters) - if inspect.isfunction(v) and inspect.getmodule(v) is j2cli.extras.filters} - } -} - -assert 'yaml' in README['formats'], 'Looks like the YAML library is not installed!' - -print(json.dumps(README)) From 5926e135b38ff653a60a0ed243dd62721bf8c499 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Thu, 21 Nov 2019 12:07:25 +0100 Subject: [PATCH 21/40] documentation: Rehaul. Tool rename. Keep only the important stuff in README.md. Extended documentation moved under docs directory. --- README.md | 534 ++++++++++++++--------------------------------- docs/advanced.md | 103 +++++++++ docs/examples.md | 104 +++++++++ docs/filters.md | 52 +++++ docs/formats.md | 90 ++++++++ 5 files changed, 501 insertions(+), 382 deletions(-) create mode 100644 docs/advanced.md create mode 100644 docs/examples.md create mode 100644 docs/filters.md create mode 100644 docs/formats.md diff --git a/README.md b/README.md index b34a436..4d6a5ac 100644 --- a/README.md +++ b/README.md @@ -1,138 +1,116 @@ -[![Build Status](https://travis-ci.org/kolypto/j2cli.svg)](https://travis-ci.org/kolypto/j2cli) -[![Pythons](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.4%E2%80%933.7%20%7C%20pypy-blue.svg)](.travis.yml) - -j2cli - Jinja2 Command-Line Tool -================================ - -`j2cli` is a command-line tool for templating in shell-scripts, -leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. - -Features: - -* Jinja2 templating -* INI, YAML, JSON data sources supported -* Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) - -Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) - -## Installation - -``` -pip install j2cli -``` - -To enable the YAML support with [pyyaml](http://pyyaml.org/): - -``` -pip install j2cli[yaml] -``` - -## Tutorial - -Suppose, you want to have an nginx configuration file template, `nginx.j2`: - -```jinja2 -server { - listen 80; - server_name {{ nginx.hostname }}; - - root {{ nginx.webroot }}; - index index.htm; -} -``` - -And you have a JSON file with the data, `nginx.json`: - -```json -{ - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project" - } -} -``` - -This is how you render it into a working configuration file: - -```bash -$ j2 -f json nginx.j2 nginx.json > nginx.conf -``` - -The output is saved to `nginx.conf`: - -``` -server { - listen 80; - server_name localhost; - - root /var/www/project; - index index.htm; -} -``` - -Alternatively, you can use the `-o nginx.conf` option. - -## Tutorial with environment variables - -Suppose, you have a very simple template, `person.xml`: - -```jinja2 -{{ name }}{{ age }} -``` - -What is the easiest way to use j2 here? -Use environment variables in your bash script: - -```bash -$ export name=Andrew -$ export age=31 -$ j2 /tmp/person.xml -Andrew31 -``` - -## Using environment variables - -Even when you use yaml or json as the data source, you can always access environment variables -using the `env()` function: - -```jinja2 -Username: {{ login }} -Password: {{ env("APP_PASSWORD") }} -``` - - -## Usage - -Compile a template using INI-file data source: - - $ j2 config.j2 data.ini - -Compile using JSON data source: - - $ j2 config.j2 data.json - -Compile using YAML data source (requires PyYAML): - - $ j2 config.j2 data.yaml - -Compile using JSON data on stdin: - - $ curl http://example.com/service.json | j2 --format=json config.j2 - -Compile using environment variables (hello Docker!): - - $ j2 config.j2 - -Or even read environment variables from a file: - - $ j2 --format=env config.j2 data.env - -Or pipe it: (note that you'll have to use the "-" in this particular case): - - $ j2 --format=env config.j2 - < data.env - - -# Reference +[![Build Status](https://travis-ci.com/m000/j2cli.svg?branch=heresy-refactor)](https://travis-ci.com/m000/j2cli/tree/heresy-refactor) +[![Pythons](https://img.shields.io/badge/python%7B3.8%2C%203.9%7D-blue.svg)](.travis.yml) +# jj2cli - Juiced Jinja2 command-line tool + +jj2cli (previously `j2cli`) is a command-line tool for templating in +shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) +library. +It supports [several formats](#supported-formats) for loading the context +used for rendering the Jinja2 templates. Loading environment variables as +rendering context is also supported. + +> **Warning** +> This branch is WIP, towards completely spinning-off the tool from +> the [upstream][j2cli]. The aim is to keep the HEAD of the branch +> usable. However, until the spin-off is complete you should expect: +> - frequent history rewrites +> - cli option changes +> - breakage if you use an older Python (<3.10) +> +> Having said that, you are welcome to use this branch and start an +> issue if you encounter any problems or have feedback. + +## jj2cli features and roadmap + +The following planned/implemented features differentiate jj2cli from +its upstreams. + +- [ ] Focus on modern Python, initially ≥3.10. This is to allow modernizing + the codebase. Support for Python ≥3.8 may be considered later, if there + are appealing reasons for that. +- [ ] Switch to more modern tooling. + * [x] [pytest][pytest] (to replace [nose][nose]) + * [ ] [ruff][ruff] (to replace [prospector][prospector]) +- [ ] Rendering of multiple templates using the same context in one go. + Rendering a couple of dozens template one-by-one is fairly slow. + This should make the tool snappier to use, but also means that + the command line interface will need to change. +- [ ] Template dependency analysis to allow better integration with tools + like [make][make]. Such tools are otherwise oblivious to Jinja2 template + inheritance/inclusion. +- [ ] Extended library of Jinja2 filters. This should allow using jj2cli + out of the box in a wider range of use cases. +- [x] Support of *context squashing* (see [below](#context-squashing)), + to eliminate the need to preprocess context data with external tools. + +## Getting started + +### Installation +```sh +# simple install +$ pip install jj2cli +# install with yaml support +$ pip install jj2cli yaml] +``` + +### Basic usage +```sh +# render config from template config.j2 +$ j2 config.j2 config.json -o config + +# render using yaml data from stdin +$ wget -O - http://example.com/config.yml | j2 --format=yml config.j2 +``` + +For an extensive list of examples, see [docs/examples.md](docs/examples.md). + +## Context data + +### Supported formats +jj2cli supports importing context from several different sources: + + * [JSON][json]: A language-independent data-serialization format, originally derived + from JavaScript. + * [YAML][yaml]: A data-serialization language, designed to be human-readable. + * [INI][ini]: Windows-style configuration files. + * env: Simple [unix-style][ini] environment variable assignments. + +For examples with each supported format, see [docs/formats.md](docs/formats.md). + +### Context squashing +One of the main strengths of jj2cli is that it is not limited to using a single +data file as context. Several data files—perhaps in different formats—can be +used to construct the rendering context. +As the contents of the data files may "overlap", jj2cli *recursively squashes* +their contents to produce the context that will be used for rendering. +The order of squashing is *from left to right*. I.e. the contents of a data file +may be overriden by any data files specified *after it* on the command line. + +Here is a simple example illustrating how context squashing works: + + * `a.json` contents: + ```json + {"a": 1, "c": {"x": 2, "y": 3}} + ``` + * `b.json` contents: + ```json + {"b": 2, "c": {"y": 4}} + ``` + * effective context when rendering with `a.json` and `b.json` (in that order): + ```json + {"a": 1, "b": 2, "c": {"x": 2, "y": 4}} + ``` + +### Loading data as a context subtree +By default, loaded data are squashed with the top-level context. However, this +may not always be desired, especially when . E.g., when you load all the environment variables +from the shell, the variables may overwri + +For this, jj2cli supports attaching the data from a +source under a variable of the top-level context. + + +## Reference `j2` accepts the following arguments: * `template`: Jinja2 template file to render @@ -146,13 +124,13 @@ Options: To import environment variables into the global scope, give it an empty string: `--import-env=`. (This will overwrite any existing variables!) * `-o outfile`: Write rendered template to a file -* `--undefined`: Allow undefined variables to be used in templates (no error will be raised) - +* `--undefined={strict, normal, debug}`: Specify the behaviour of jj2 for undefined + variables. Refer to [Jinja2 docs][jinja2-undefined] for details. * `--filters filters.py`: Load custom Jinja2 filters and tests from a Python file. Will load all top-level functions and register them as filters. This option can be used multiple times to import several files. * `--tests tests.py`: Load custom Jinja2 filters and tests from a Python file. -* `--customize custom.py`: A Python file that implements hooks to fine-tune the j2cli behavior. +* `--customize custom.py`: A Python file that implements hooks to fine-tune the jj2cli behavior. This is fairly advanced stuff, use it only if you really need to customize the way Jinja2 is initialized. See [Customization](#customization) for more info. @@ -161,247 +139,39 @@ There is some special behavior with environment variables: * When `data` is not provided (data is `-`), `--format` defaults to `env` and thus reads environment variables * When `--format=env`, it can read a special "environment variables" file made like this: `env > /tmp/file.env` -## Formats - - -### env -Data input from environment variables. - -Render directly from the current environment variable values: - - $ j2 config.j2 - -Or alternatively, read the values from a dotenv file: - -``` -NGINX_HOSTNAME=localhost -NGINX_WEBROOT=/var/www/project -NGINX_LOGS=/var/log/nginx/ -``` - -And render with: - - $ j2 config.j2 data.env - $ env | j2 --format=env config.j2 - -If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: - - $ j2 config.j2 - < data.env - -### ini -INI data input format. - -data.ini: - -``` -[nginx] -hostname=localhost -webroot=/var/www/project -logs=/var/log/nginx/ -``` - -Usage: - - $ j2 config.j2 data.ini - $ cat data.ini | j2 --format=ini config.j2 - -### json -JSON data input format - -data.json: - -``` -{ - "nginx":{ - "hostname": "localhost", - "webroot": "/var/www/project", - "logs": "/var/log/nginx/" - } -} -``` - -Usage: - - $ j2 config.j2 data.json - $ cat data.json | j2 --format=ini config.j2 - -### yaml -YAML data input format. - -data.yaml: - -``` -nginx: - hostname: localhost - webroot: /var/www/project - logs: /var/log/nginx -``` - -Usage: - - $ j2 config.j2 data.yml - $ cat data.yml | j2 --format=yaml config.j2 - - - - -Extras -====== - -## Filters - - -### `docker_link(value, format='{addr}:{port}')` -Given a Docker Link environment variable value, format it into something else. - -This first parses a Docker Link value like this: - - DB_PORT=tcp://172.17.0.5:5432 - -Into a dict: - -```python -{ - 'proto': 'tcp', - 'addr': '172.17.0.5', - 'port': '5432' -} -``` - -And then uses `format` to format it, where the default format is '{addr}:{port}'. - -More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) - -### `env(varname, default=None)` -Use an environment variable's value inside your template. - -This filter is available even when your data source is something other that the environment. - -Example: - -```jinja2 -User: {{ user_login }} -Pass: {{ "USER_PASSWORD"|env }} -``` - -You can provide the default value: - -```jinja2 -Pass: {{ "USER_PASSWORD"|env("-none-") }} -``` - -For your convenience, it's also available as a function: - -```jinja2 -User: {{ user_login }} -Pass: {{ env("USER_PASSWORD") }} -``` - -Notice that there must be quotes around the environment variable name - - - - -Customization -============= - -j2cli now allows you to customize the way the application is initialized: - -* Pass additional keywords to Jinja2 environment -* Modify the context before it's used for rendering -* Register custom filters and tests - -This is done through *hooks* that you implement in a customization file in Python language. -Just plain functions at the module level. - -The following hooks are available: - -* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for - [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). -* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. -* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be - used for template rendering. You can do all sorts of pre-processing here. -* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 -* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 - -All of them are optional. - -The example customization.py file for your reference: - -```python -# -# Example customize.py file for j2cli -# Contains potional hooks that modify the way j2cli is initialized - - -def j2_environment_params(): - """ Extra parameters for the Jinja2 Environment """ - # Jinja2 Environment configuration - # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment - return dict( - # Just some examples - - # Change block start/end strings - block_start_string='<%', - block_end_string='%>', - # Change variable strings - variable_start_string='<<', - variable_end_string='>>', - # Remove whitespace around blocks - trim_blocks=True, - lstrip_blocks=True, - # Enable line statements: - # http://jinja.pocoo.org/docs/2.10/templates/#line-statements - line_statement_prefix='#', - # Keep \n at the end of a file - keep_trailing_newline=True, - # Enable custom extensions - # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions - extensions=('jinja2.ext.i18n',), - ) - - -def j2_environment(env): - """ Modify Jinja2 environment - - :param env: jinja2.environment.Environment - :rtype: jinja2.environment.Environment - """ - env.globals.update( - my_function=lambda v: 'my function says "{}"'.format(v) - ) - return env - - -def alter_context(context): - """ Modify the context and return it """ - # An extra variable - context['ADD'] = '127' - return context - - -def extra_filters(): - """ Declare some custom filters. - - Returns: dict(name = function) - """ - return dict( - # Example: {{ var | parentheses }} - parentheses=lambda t: '(' + t + ')', - ) - - -def extra_tests(): - """ Declare some custom tests - - Returns: dict(name = function) - """ - return dict( - # Example: {% if a|int is custom_odd %}odd{% endif %} - custom_odd=lambda n: True if (n % 2) else False - ) - -# - -``` - +## Extras + +### Filters +For convenience, jj2cli offers several additional Jinja2 filters that can be used +in your templates. These filters should help you avoid having to implement an +[advanced customization module](#advanced-customization) for many use cases. + +See [docs/filters.md](docs/filters.md) for details on the available filters. + +### Advanced customization +jj2cli offers several *hooks* that allow for more advanced customization of its +operation. This includes: + + * passing additional keywords to Jinja2 environment + * modifying the context before it's used for rendering + * registering custom filters and tests + +See [docs/advanced.md](docs/advanced.md) for details on advanced customization. + +## Credits +jj2cli is inspired by and builds on [kolypto/j2cli][j2cli] and +[mattrobenolt/jinja2-cli][jinja2-cli] tools. + +[docker]: http://www.docker.com/ +[env]: https://en.wikipedia.org/wiki/Environment_variable#Unix +[ini]: https://en.wikipedia.org/wiki/INI_file +[j2cli]: https://github.com/kolypto/j2cli +[jinja2-cli]: https://github.com/mattrobenolt/jinja2-cli +[jinja2-undefined]: https://jinja.palletsprojects.com/en/2.10.x/api/#undefined-types +[json]: https://en.wikipedia.org/wiki/JSON +[make]: https://www.gnu.org/software/make/ +[nose]: https://nose.readthedocs.io/ +[prospector]: https://prospector.landscape.io/en/master/ +[pytest]: https://docs.pytest.org/ +[ruff]: https://docs.astral.sh/ruff/ +[yaml]: https://en.wikipedia.org/wiki/YAML diff --git a/docs/advanced.md b/docs/advanced.md new file mode 100644 index 0000000..04a0e4c --- /dev/null +++ b/docs/advanced.md @@ -0,0 +1,103 @@ +Customization +============= + +j2cli now allows you to customize the way the application is initialized: + +* Pass additional keywords to Jinja2 environment +* Modify the context before it's used for rendering +* Register custom filters and tests + +This is done through *hooks* that you implement in a customization file in Python language. +Just plain functions at the module level. + +The following hooks are available: + +* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for + [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). +* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. +* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be + used for template rendering. You can do all sorts of pre-processing here. +* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 +* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 + +All of them are optional. + +The example customization.py file for your reference: + +```python +# +# Example customize.py file for j2cli +# Contains potional hooks that modify the way j2cli is initialized + + +def j2_environment_params(): + """ Extra parameters for the Jinja2 Environment """ + # Jinja2 Environment configuration + # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment + return dict( + # Just some examples + + # Change block start/end strings + block_start_string='<%', + block_end_string='%>', + # Change variable strings + variable_start_string='<<', + variable_end_string='>>', + # Remove whitespace around blocks + trim_blocks=True, + lstrip_blocks=True, + # Enable line statements: + # http://jinja.pocoo.org/docs/2.10/templates/#line-statements + line_statement_prefix='#', + # Keep \n at the end of a file + keep_trailing_newline=True, + # Enable custom extensions + # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions + extensions=('jinja2.ext.i18n',), + ) + + +def j2_environment(env): + """ Modify Jinja2 environment + + :param env: jinja2.environment.Environment + :rtype: jinja2.environment.Environment + """ + env.globals.update( + my_function=lambda v: 'my function says "{}"'.format(v) + ) + return env + + +def alter_context(context): + """ Modify the context and return it """ + # An extra variable + context['ADD'] = '127' + return context + + +def extra_filters(): + """ Declare some custom filters. + + Returns: dict(name = function) + """ + return dict( + # Example: {{ var | parentheses }} + parentheses=lambda t: '(' + t + ')', + ) + + +def extra_tests(): + """ Declare some custom tests + + Returns: dict(name = function) + """ + return dict( + # Example: {% if a|int is custom_odd %}odd{% endif %} + custom_odd=lambda n: True if (n % 2) else False + ) + +# + +``` + diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..a5ca363 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,104 @@ +## Tutorial + +Suppose, you want to have an nginx configuration file template, `nginx.j2`: + +```jinja2 +server { + listen 80; + server_name {{ nginx.hostname }}; + + root {{ nginx.webroot }}; + index index.htm; +} +``` + +And you have a JSON file with the data, `nginx.json`: + +```json +{ + "nginx":{ + "hostname": "localhost", + "webroot": "/var/www/project" + } +} +``` + +This is how you render it into a working configuration file: + +```bash +$ j2 -f json nginx.j2 nginx.json > nginx.conf +``` + +The output is saved to `nginx.conf`: + +``` +server { + listen 80; + server_name localhost; + + root /var/www/project; + index index.htm; +} +``` + +Alternatively, you can use the `-o nginx.conf` option. + +## Tutorial with environment variables + +Suppose, you have a very simple template, `person.xml`: + +```jinja2 +{{ name }}{{ age }} +``` + +What is the easiest way to use j2 here? +Use environment variables in your bash script: + +```bash +$ export name=Andrew +$ export age=31 +$ j2 /tmp/person.xml +Andrew31 +``` + +## Using environment variables + +Even when you use yaml or json as the data source, you can always access environment variables +using the `env()` function: + +```jinja2 +Username: {{ login }} +Password: {{ env("APP_PASSWORD") }} +``` + + +## Usage + +Compile a template using INI-file data source: + + $ j2 config.j2 data.ini + +Compile using JSON data source: + + $ j2 config.j2 data.json + +Compile using YAML data source (requires PyYAML): + + $ j2 config.j2 data.yaml + +Compile using JSON data on stdin: + + $ curl http://example.com/service.json | j2 --format=json config.j2 + +Compile using environment variables (hello Docker!): + + $ j2 config.j2 + +Or even read environment variables from a file: + + $ j2 --format=env config.j2 data.env + +Or pipe it: (note that you'll have to use the "-" in this particular case): + + $ j2 --format=env config.j2 - < data.env + diff --git a/docs/filters.md b/docs/filters.md new file mode 100644 index 0000000..a5748e9 --- /dev/null +++ b/docs/filters.md @@ -0,0 +1,52 @@ +## Filters + + +### `docker_link(value, format='{addr}:{port}')` +Given a Docker Link environment variable value, format it into something else. + +This first parses a Docker Link value like this: + + DB_PORT=tcp://172.17.0.5:5432 + +Into a dict: + +```python +{ + 'proto': 'tcp', + 'addr': '172.17.0.5', + 'port': '5432' +} +``` + +And then uses `format` to format it, where the default format is '{addr}:{port}'. + +More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) + +### `env(varname, default=None)` +Use an environment variable's value inside your template. + +This filter is available even when your data source is something other that the environment. + +Example: + +```jinja2 +User: {{ user_login }} +Pass: {{ "USER_PASSWORD"|env }} +``` + +You can provide the default value: + +```jinja2 +Pass: {{ "USER_PASSWORD"|env("-none-") }} +``` + +For your convenience, it's also available as a function: + +```jinja2 +User: {{ user_login }} +Pass: {{ env("USER_PASSWORD") }} +``` + +Notice that there must be quotes around the environment variable name + + diff --git a/docs/formats.md b/docs/formats.md new file mode 100644 index 0000000..cbfe164 --- /dev/null +++ b/docs/formats.md @@ -0,0 +1,90 @@ +# Supported Formats + +Following, we show how to use the different data file formats supported +bu j2cli to render an nginx configuration file template, `nginx.j2`: + +```jinja2 +server { + listen 80; + server_name {{ nginx.hostname }}; + + root {{ nginx.webroot }}; + index index.htm; +} +``` + +## JSON + +Data file contents: +```json +{ + "nginx":{ + "hostname": "localhost", + "webroot": "/var/www/project", + "logs": "/var/log/nginx/" + } +} +``` + +Usage: + + $ j2 config.j2 data.json -o config + $ j2 -f json config.j2 - < data.json > config + + +## YAML + +Data file contents: +```yaml +nginx: + hostname: localhost + webroot: /var/www/project + logs: /var/log/nginx +``` + +Usage: + + $ j2 config.j2 data.yaml -o config + $ j2 -f yaml config.j2 - < data.yaml > config + + +## INI + +Data file contents: +```ini +[nginx] +hostname=localhost +webroot=/var/www/project +logs=/var/log/nginx/ +``` + +Usage: + + $ j2 config.j2 data.ini -o config + $ j2 -f ini config.j2 - < data.ini > config + + +## env + +### From file +Data file contents: +```sh +NGINX_HOSTNAME=localhost +NGINX_WEBROOT=/var/www/project +NGINX_LOGS=/var/log/nginx/ +``` + +Usage: + + $ j2 config.j2 data.env -o config + $ j2 -f env config.j2 - < data.env > config + + +### From shell environment variables +Render directly from the current environment variable values: + +Usage: + + $ export NGINX_HOSTNAME=localhost NGINX_WEBROOT=/var/www/project NGINX_LOGS=/var/log/nginx/ + $ env | j2 -f env config.j2 - > config + From 10b3c72864ed79587801abb15ffe392b9c2e21bf Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Fri, 3 Nov 2023 22:07:36 +0100 Subject: [PATCH 22/40] documentation: Update copyrights. --- LICENSE | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 7a916f8..01a0786 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,7 @@ -Copyright (c) 2014, Mark Vartanyan +Copyright (c) 2019-2023, Manolis Stamatogiannakis +Copyright (c) 2014-2019, Mark Vartanyan +Copyright (c) 2012-2013, Matt Robenolt + All rights reserved. Redistribution and use in source and binary forms, with or without modification, From 74f7e551af7d32b808f52a97dba418ce10ce96a2 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Mon, 17 May 2021 15:36:30 +0200 Subject: [PATCH 23/40] changelog: Updates for v0.5.0. --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab80cc1..f662507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## 0.5.0 (2021-05-31) +* Project spun-off, renamed to jj2cli. Version bumped-up to 0.5.0 to + show parity plus a few extra features. Future changes to version will + not have any significance regarding to the parent j2cli project. +* New: Support for multiple input files, squashed in a single context prior + to rendering. +* New: Support for using *data-specs* to describe input files. + - Allows mixing/matching different input formats. + - Allows attaching input format in a specific location of the context. + - Support for list-formatted inputs via aforementioned attaching. +* New: `--ignore-missing`/`-I` flag, for ignoring non-existing input files. +* Change: `--undefined`/`-U` now allows you to set the behaviour of Jinja2 + for undefined variables. +* Change: `--fallback-format`/`-f` now sets the *fallback* format, rather than + forcing the use of a specific format. Forcing a specific format can be + achieved via *data-specs*. +* Change: Currently only Python 3.8 and 3.9 are supported. The goal is to + eventually support Python >=3.6 and Python 2.7. + ## 0.3.12 (2019-08-18) * Fix: use `env` format from stdin From e99652eac30b49d86a5762e520f722b981530c10 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Mon, 10 May 2021 22:25:47 +0200 Subject: [PATCH 24/40] linting: Linter fixes across the codebase. --- src/jj2cli/cli.py | 3 ++- src/jj2cli/customize.py | 6 +----- src/jj2cli/filters.py | 41 ++++++++++++++++++++---------------- src/jj2cli/parsers.py | 3 +-- src/jj2cli/render.py | 6 ++++-- tests/resources/customize.py | 4 ++-- 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/jj2cli/cli.py b/src/jj2cli/cli.py index 40a35be..c66b60e 100644 --- a/src/jj2cli/cli.py +++ b/src/jj2cli/cli.py @@ -101,7 +101,7 @@ def render_command(argv): # Renderer renderer = Jinja2TemplateRenderer(os.getcwd(), args.undefined, args.no_compact, j2_env_params=customize.j2_environment_params()) - customize.j2_environment(renderer._env) + customize.j2_environment(renderer._env) # pylint: disable=protected-access # Filters, Tests renderer.register_filters(filters.EXTRA_FILTERS) @@ -133,6 +133,7 @@ def render(): return 1 outstream = getattr(sys.stdout, 'buffer', sys.stdout) outstream.write(output) + return 0 def dependencies(): diff --git a/src/jj2cli/customize.py b/src/jj2cli/customize.py index 366321f..d759f21 100644 --- a/src/jj2cli/customize.py +++ b/src/jj2cli/customize.py @@ -1,13 +1,9 @@ -class CustomizationModule(object): +class CustomizationModule: """ The interface for customization functions, defined as module-level functions """ def __init__(self, module=None): if module is not None: - def howcall(*args): - print(args) - exit(1) - # Import every module function as a method on ourselves for name in self._IMPORTED_METHOD_NAMES: try: diff --git a/src/jj2cli/filters.py b/src/jj2cli/filters.py index b36eb82..edd1be9 100644 --- a/src/jj2cli/filters.py +++ b/src/jj2cli/filters.py @@ -5,9 +5,9 @@ from jinja2 import is_undefined, pass_context if sys.version_info >= (3,0): - from shutil import which + from shutil import which # pylint: disable=import-error elif sys.version_info >= (2,5): - from shutilwhich import which + from shutilwhich import which # pylint: disable=import-error else: assert False, "Unsupported Python version: %s" % sys.version_info @@ -18,7 +18,7 @@ else: assert False, "Unsupported Python version: %s" % sys.version_info -def docker_link(value, format='{addr}:{port}'): +def docker_link(value, fmt='{addr}:{port}'): """ Given a Docker Link environment variable value, format it into something else. XXX: The name of the filter is not very informative. This is actually a partial URI parser. @@ -36,12 +36,12 @@ def docker_link(value, format='{addr}:{port}'): } ``` - And then uses `format` to format it, where the default format is '{addr}:{port}'. + And then uses `fmt` to format it, where the default format is '{addr}:{port}'. More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) :param value: Docker link (from an environment variable) - :param format: The format to apply. Supported placeholders: `{proto}`, `{addr}`, `{port}` + :param fmt: The format to apply. Supported placeholders: `{proto}`, `{addr}`, `{port}` :return: Formatted string """ # pass undefined values on down the pipeline @@ -55,7 +55,7 @@ def docker_link(value, format='{addr}:{port}'): d = m.groupdict() # Format - return format.format(**d) + return fmt.format(**d) def env(varname, default=None): @@ -85,37 +85,41 @@ def env(varname, default=None): Notice that there must be quotes around the environment variable name """ - if default is not None: - # With the default, there's never an error - return os.getenv(varname, default) - else: + if default is None: # Raise KeyError when not provided return os.environ[varname] + # With the default, there's never an error + return os.getenv(varname, default) + + def align_suffix(text, delim, column=None, spaces_after_delim=1): """ Align the suffixes of lines in text, starting from the specified delim. + + Example: XXX """ s='' if column is None or column == 'auto': - column = max(map(lambda l: l.find(delim), text.splitlines())) + column = max(map(lambda ln: ln.find(delim), text.splitlines())) elif column == 'previous': column = align_suffix.column_previous - for l in map(lambda s: s.split(delim, 1), text.splitlines()): - if len(l) < 2: + for ln in map(lambda s: s.split(delim, 1), text.splitlines()): + if len(ln) < 2: # no delimiter occurs - s += l[0].rstrip() + os.linesep - elif l[0].strip() == '': + s += ln[0].rstrip() + os.linesep + elif ln[0].strip() == '': # no content before delimiter - leave as-is - s += l[0] + delim + l[1] + os.linesep + s += ln[0] + delim + ln[1] + os.linesep else: # align - s += l[0].rstrip().ljust(column) + delim + spaces_after_delim*' ' + l[1].strip() + os.linesep + s += ln[0].rstrip().ljust(column) + delim + spaces_after_delim*' ' + ln[1].strip() + os.linesep align_suffix.column_previous = column return s + align_suffix.column_previous = None @@ -140,11 +144,13 @@ def sh_opt(text, name, delim=" ", quote=False): text = sh_quote(text) return '%s%s%s' % (name, delim, text) + def sh_optq(text, name, delim=" "): """ Quote text and format as a command line option. """ return sh_opt(text, name, delim, quote=True) + # Filters to be loaded EXTRA_FILTERS = { 'sh_quote': sh_quote, @@ -163,4 +169,3 @@ def sh_optq(text, name, delim=" "): 'align_suffix': align_suffix, 'ctxlookup': ctxlookup, } - diff --git a/src/jj2cli/parsers.py b/src/jj2cli/parsers.py index 14be114..d64071d 100644 --- a/src/jj2cli/parsers.py +++ b/src/jj2cli/parsers.py @@ -127,8 +127,7 @@ def parse(self, ignore_missing=False, fallback_format='ini'): if isinstance(self._iostr, FileNotFoundError): if ignore_missing is True: return {} - else: - raise self._iostr + raise self._iostr return getattr(self, '_parse_%s' % fmt)() def _parse_ENV(self): diff --git a/src/jj2cli/render.py b/src/jj2cli/render.py index 970dc1a..7bf3fbe 100644 --- a/src/jj2cli/render.py +++ b/src/jj2cli/render.py @@ -22,6 +22,7 @@ def __init__(self, cwd, encoding='utf-8'): def get_source(self, environment, template): # Path filename = os.path.join(self.cwd, template) + logging.debug("TEMPLATE_PATH %s", filename) # Read try: @@ -35,7 +36,7 @@ def get_source(self, environment, template): return contents, filename, uptodate -class Jinja2TemplateRenderer(object): +class Jinja2TemplateRenderer: """ Template renderer """ UNDEFINED = { @@ -44,8 +45,9 @@ class Jinja2TemplateRenderer(object): 'debug': jinja2.DebugUndefined, # return the debug info when printed } - def __init__(self, cwd, undefined='strict', no_compact=False, j2_env_params={}): + def __init__(self, cwd, undefined='strict', no_compact=False, j2_env_params=None): # Custom env params + j2_env_params = j2_env_params if j2_env_params is not None else {} j2_env_params.setdefault('keep_trailing_newline', True) j2_env_params.setdefault('undefined', self.UNDEFINED[undefined]) j2_env_params.setdefault('trim_blocks', not no_compact) diff --git a/tests/resources/customize.py b/tests/resources/customize.py index 21beab0..993a169 100644 --- a/tests/resources/customize.py +++ b/tests/resources/customize.py @@ -37,7 +37,7 @@ def j2_environment(env): :rtype: jinja2.environment.Environment """ env.globals.update( - my_function=lambda v: 'my function says "{}"'.format(v) + my_function='my function says "{}"'.format ) return env @@ -67,7 +67,7 @@ def extra_tests(): """ return dict( # Example: {% if a|int is custom_odd %}odd{% endif %} - custom_odd=lambda n: True if (n % 2) else False + custom_odd=lambda n: bool(n % 2) ) # {% endraw %} From 982f620e9dfde31e7f8b900822593d6a26e8f439 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 10:25:33 +0100 Subject: [PATCH 25/40] f --- .github/workflows/j2cli.yml | 57 +++++- poetry.lock | 376 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/requirements.txt | 1 - tox.ini | 26 ++- 5 files changed, 442 insertions(+), 20 deletions(-) create mode 100644 poetry.lock delete mode 100644 tests/requirements.txt diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index e861f2a..799a615 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -1,3 +1,6 @@ +# Related documentation: +# https://jacobian.org/til/github-actions-poetry/ +# https://github.com/snok/install-poetry name: m000/j2cli on: push: @@ -9,13 +12,51 @@ concurrency: cancel-in-progress: true jobs: test: - runs-on: ubuntu-latest - env: - TOXENV: py3.10-pyyaml6 + strategy: + fail-fast: false + matrix: + os: [ "ubuntu-latest" ] + python-version: [ "3.10", "3.11" ] + runs-on: ${{ matrix.os }} +# env: +# TOXENV: py${{ matrix.python-version }}-pyyaml6 steps: - - uses: actions/checkout@v4.1.0 - - uses: actions/setup-python@v4.7.0 + - name: Repository checkout + uses: actions/checkout@v4 + - name: Python ${{ matrix.python-version }} setup + uses: actions/setup-python@v4 with: - python-version: "3.10" - - run: pip install tox - - run: tox + python-version: ${{ matrix.python-version }} + - name: Load cached Poetry installation + id: cached-poetry + uses: actions/cache@v3 + with: + path: ~/.local # the path depends on the OS + key: poetry-0 # increment to reset cache + - name: Install Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 + with: + version: 1.7.0 + virtualenvs-create: true + virtualenvs-in-project: true # create .venv in test directory + - name: Install Poetry development dependencies # main dependencies are handled by tox + if: steps.cached-poetry.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root --only=dev + - name: Load cached tox environment + id: cached-toxenv + uses: actions/cache@v3 + with: + path: .tox/py${{ matrix.python-version }}-** + key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('tox.ini') }} + - name: Tox tests (py${{ matrix.python-version }}) + run: poetry run tox + - name: Send coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + #files: ./coverage1.xml,./coverage2.xml # optional + #flags: unittests # optional + #name: codecov-umbrella # optional + fail_ci_if_error: false + verbose: true diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..d1731d8 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,376 @@ +# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "distlib" +version = "0.3.7" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, + {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.3" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pyproject-api" +version = "1.6.1" +description = "API to interact with the python pyproject.toml based projects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyproject_api-1.6.1-py3-none-any.whl", hash = "sha256:4c0116d60476b0786c88692cf4e325a9814965e2469c5998b830bba16b183675"}, + {file = "pyproject_api-1.6.1.tar.gz", hash = "sha256:1817dc018adc0d1ff9ca1ed8c60e1623d5aaca40814b953af14a9cf9a5cae538"}, +] + +[package.dependencies] +packaging = ">=23.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.8.19)", "sphinx (<7.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "setuptools (>=68.1.2)", "wheel (>=0.41.2)"] + +[[package]] +name = "pytest" +version = "7.4.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tox" +version = "4.11.3" +description = "tox is a generic virtualenv management and test command line tool" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tox-4.11.3-py3-none-any.whl", hash = "sha256:599af5e5bb0cad0148ac1558a0b66f8fff219ef88363483b8d92a81e4246f28f"}, + {file = "tox-4.11.3.tar.gz", hash = "sha256:5039f68276461fae6a9452a3b2c7295798f00a0e92edcd9a3b78ba1a73577951"}, +] + +[package.dependencies] +cachetools = ">=5.3.1" +chardet = ">=5.2" +colorama = ">=0.4.6" +filelock = ">=3.12.3" +packaging = ">=23.1" +platformdirs = ">=3.10" +pluggy = ">=1.3" +pyproject-api = ">=1.6.1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} +virtualenv = ">=20.24.3" + +[package.extras] +docs = ["furo (>=2023.8.19)", "sphinx (>=7.2.4)", "sphinx-argparse-cli (>=1.11.1)", "sphinx-autodoc-typehints (>=1.24)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2023.4.21)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +testing = ["build[virtualenv] (>=0.10)", "covdefaults (>=2.3)", "detect-test-pollution (>=1.1.1)", "devpi-process (>=1)", "diff-cover (>=7.7)", "distlib (>=0.3.7)", "flaky (>=3.7)", "hatch-vcs (>=0.3)", "hatchling (>=1.18)", "psutil (>=5.9.5)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-xdist (>=3.3.1)", "re-assert (>=1.1)", "time-machine (>=2.12)", "wheel (>=0.41.2)"] + +[[package]] +name = "virtualenv" +version = "20.24.6" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.24.6-py3-none-any.whl", hash = "sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381"}, + {file = "virtualenv-20.24.6.tar.gz", hash = "sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<4" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "d92c295edbd95dee79e1497dd9a79b266dadfae7ffcae67d106ae84b56503bbc" diff --git a/pyproject.toml b/pyproject.toml index 0c9f547..5f24295 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ # package ########################################################### [build-system] -requires = ["poetry-core>=1.2.0"] +requires = ["poetry-core>=1.2.0", "pytest>=7.4", "tox>=4.11"] build-backend = "poetry.core.masonry.api" # pytest ############################################################ diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index e079f8a..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pytest diff --git a/tox.ini b/tox.ini index 352dd54..e2dd9b7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,28 @@ [tox] envlist= - py{3.10}-pyyaml{0,6} + py{3.10, 3.11}-pyyaml{0,6} skip_missing_interpreters=True [testenv] +# See: https://tox.wiki/en/latest/config.html#tox-ini basepython= py3.10: python3.10 + py3.11: python3.11 deps= - -rtests/requirements.txt pyyaml0: null pyyaml6: pyyaml~=6.0 -allowlist_externals = pytest +allowlist_externals = + poetry + pytest commands= + poetry install pytest {posargs:tests/} +usedevelop=True + +#poetry install --no-interaction +#poetry run pytest {posargs:tests/} + +# NB: matrix should extrapolate to poetry with/without settings rather than specific versions #[testenv:lint] #basepython= @@ -23,10 +33,6 @@ commands= #commands= # prospector -[testenv:dev] -deps= - -rtest/requirements.txt -usedevelop=True - -# To see how these expand to tests, see: -# https://tox.readthedocs.io/en/latest/example/basic.html#compressing-dependency-matrix +#[testenv:dev] +#deps= +#-rtest/requirements.txt From b05f822b267d517b8d7be1aeb1cd51d6bd5e9cf9 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 10:36:57 +0100 Subject: [PATCH 26/40] x --- .github/workflows/j2cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 799a615..c7c7737 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -48,7 +48,7 @@ jobs: uses: actions/cache@v3 with: path: .tox/py${{ matrix.python-version }}-** - key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('tox.ini') }} + key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini') }} - name: Tox tests (py${{ matrix.python-version }}) run: poetry run tox - name: Send coverage reports to Codecov From 6baa61b5e76b228767951a56c56e1f100e6f2e8b Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 10:39:49 +0100 Subject: [PATCH 27/40] x --- .github/workflows/j2cli.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index c7c7737..840e2e6 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -49,6 +49,10 @@ jobs: with: path: .tox/py${{ matrix.python-version }}-** key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini') }} + - name: Tox id1 (py${{ matrix.python-version }}) + run: which tox || true + - name: Tox id2 (py${{ matrix.python-version }}) + run: poetry run which tox || true - name: Tox tests (py${{ matrix.python-version }}) run: poetry run tox - name: Send coverage reports to Codecov From d48ed5ea5588c2a3e0db408dc7b18c3e716354eb Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 10:50:03 +0100 Subject: [PATCH 28/40] x --- .github/workflows/j2cli.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 840e2e6..739e7d6 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -27,6 +27,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Tox env1 (py${{ matrix.python-version }}) + run: env - name: Load cached Poetry installation id: cached-poetry uses: actions/cache@v3 @@ -40,9 +42,13 @@ jobs: version: 1.7.0 virtualenvs-create: true virtualenvs-in-project: true # create .venv in test directory + - name: Tox env2 (py${{ matrix.python-version }}) + run: env - name: Install Poetry development dependencies # main dependencies are handled by tox if: steps.cached-poetry.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root --only=dev + - name: Tox env3 (py${{ matrix.python-version }}) + run: env - name: Load cached tox environment id: cached-toxenv uses: actions/cache@v3 From 13025a4d8d6da65fd14c241cfa31ea4e42f67f28 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 10:58:13 +0100 Subject: [PATCH 29/40] x --- .github/workflows/j2cli.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 739e7d6..6e59241 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -47,6 +47,9 @@ jobs: - name: Install Poetry development dependencies # main dependencies are handled by tox if: steps.cached-poetry.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root --only=dev + - name: Activate Poetry development dependencies # main dependencies are handled by tox + if: steps.cached-poetry.outputs.cache-hit == 'true' + run: source $VENV - name: Tox env3 (py${{ matrix.python-version }}) run: env - name: Load cached tox environment From bcc5d66e1bcb9ae9c040970d530a5323c0d09a60 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:01:04 +0100 Subject: [PATCH 30/40] x --- .github/workflows/j2cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 6e59241..c06e99b 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -49,7 +49,7 @@ jobs: run: poetry install --no-interaction --no-root --only=dev - name: Activate Poetry development dependencies # main dependencies are handled by tox if: steps.cached-poetry.outputs.cache-hit == 'true' - run: source $VENV + run: source .venv/bin/activate - name: Tox env3 (py${{ matrix.python-version }}) run: env - name: Load cached tox environment From 9b59279c89a21a509a7dd2e585408e17e1c7806f Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:11:37 +0100 Subject: [PATCH 31/40] x --- .github/workflows/j2cli.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index c06e99b..1806842 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -29,12 +29,16 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tox env1 (py${{ matrix.python-version }}) run: env + - name: Find1 (py${{ matrix.python-version }}) + run: find . - name: Load cached Poetry installation id: cached-poetry uses: actions/cache@v3 with: path: ~/.local # the path depends on the OS key: poetry-0 # increment to reset cache + - name: Find2 (py${{ matrix.python-version }}) + run: find . - name: Install Poetry if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 From d94e8353cab4ba9358e46d8d28d96308b9adff69 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:14:43 +0100 Subject: [PATCH 32/40] x --- .github/workflows/j2cli.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 1806842..9b5be3e 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -54,6 +54,8 @@ jobs: - name: Activate Poetry development dependencies # main dependencies are handled by tox if: steps.cached-poetry.outputs.cache-hit == 'true' run: source .venv/bin/activate + - name: Find3 (py${{ matrix.python-version }}) + run: find . - name: Tox env3 (py${{ matrix.python-version }}) run: env - name: Load cached tox environment From fe4360587506e3efb9a233a124eae91f10f871fb Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:15:04 +0100 Subject: [PATCH 33/40] l --- .github/workflows/j2cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 9b5be3e..5d7812b 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -36,7 +36,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.local # the path depends on the OS - key: poetry-0 # increment to reset cache + key: poetry-1 # increment to reset cache - name: Find2 (py${{ matrix.python-version }}) run: find . - name: Install Poetry From 1c843d7acbe43961063bd41f9c42472abc3bd169 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:26:14 +0100 Subject: [PATCH 34/40] ll --- .github/workflows/j2cli.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 5d7812b..8f5946a 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -36,7 +36,7 @@ jobs: uses: actions/cache@v3 with: path: ~/.local # the path depends on the OS - key: poetry-1 # increment to reset cache + key: poetry-2 # increment to reset cache - name: Find2 (py${{ matrix.python-version }}) run: find . - name: Install Poetry @@ -79,3 +79,7 @@ jobs: #name: codecov-umbrella # optional fail_ci_if_error: false verbose: true + - name: WTF1 (py${{ matrix.python-version }}) + run: ls -al manifest.txt || true + - name: WTF2 (py${{ matrix.python-version }}) + run: cat manifest.txt || true From 0bf092600808363b6cdedaca3949762163c5c327 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:34:27 +0100 Subject: [PATCH 35/40] xxx --- .github/workflows/j2cli.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 8f5946a..4a7c711 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -35,8 +35,8 @@ jobs: id: cached-poetry uses: actions/cache@v3 with: - path: ~/.local # the path depends on the OS - key: poetry-2 # increment to reset cache + path: .venv # the path depends on the OS + key: poetry-3 # increment to reset cache - name: Find2 (py${{ matrix.python-version }}) run: find . - name: Install Poetry From 191bc588cfb17de8e6777f1caa0a3e5e909d2fc1 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:41:58 +0100 Subject: [PATCH 36/40] l --- .github/workflows/j2cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 4a7c711..1dbf388 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -63,7 +63,7 @@ jobs: uses: actions/cache@v3 with: path: .tox/py${{ matrix.python-version }}-** - key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini') }} + key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini', ${{ GITHUB_WORKFLOW_FILE }}) }} - name: Tox id1 (py${{ matrix.python-version }}) run: which tox || true - name: Tox id2 (py${{ matrix.python-version }}) From 9b8f87b963d8f96dbba00e93b26ad90fbe5b4e15 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:43:15 +0100 Subject: [PATCH 37/40] lala --- .github/workflows/j2cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 1dbf388..4a7c711 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -63,7 +63,7 @@ jobs: uses: actions/cache@v3 with: path: .tox/py${{ matrix.python-version }}-** - key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini', ${{ GITHUB_WORKFLOW_FILE }}) }} + key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini') }} - name: Tox id1 (py${{ matrix.python-version }}) run: which tox || true - name: Tox id2 (py${{ matrix.python-version }}) From 8fecb2e59fb2da520c3488ac4dc36af75fb2ba13 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:52:40 +0100 Subject: [PATCH 38/40] f --- .github/workflows/j2cli.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 4a7c711..6d53352 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -27,8 +27,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Tox env1 (py${{ matrix.python-version }}) - run: env + - name: Env-Inspect1 (py${{ matrix.python-version }}) + run: (env | sort; echo "----"; which tox; echo "----"; which poetry) || true - name: Find1 (py${{ matrix.python-version }}) run: find . - name: Load cached Poetry installation From 1a605c70c322376522df2352667c0adc10275598 Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Sun, 12 Nov 2023 11:57:35 +0100 Subject: [PATCH 39/40] fff --- .github/workflows/j2cli.yml | 39 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 6d53352..1b4ec11 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -27,18 +27,18 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Env-Inspect1 (py${{ matrix.python-version }}) - run: (env | sort; echo "----"; which tox; echo "----"; which poetry) || true - - name: Find1 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect1 (py${{ matrix.python-version }}) run: find . + - name: Load cached Poetry installation id: cached-poetry uses: actions/cache@v3 with: path: .venv # the path depends on the OS key: poetry-3 # increment to reset cache - - name: Find2 (py${{ matrix.python-version }}) - run: find . - name: Install Poetry if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 @@ -46,30 +46,34 @@ jobs: version: 1.7.0 virtualenvs-create: true virtualenvs-in-project: true # create .venv in test directory - - name: Tox env2 (py${{ matrix.python-version }}) - run: env - name: Install Poetry development dependencies # main dependencies are handled by tox if: steps.cached-poetry.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root --only=dev + + - name: Env-Inspect2 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect2 (py${{ matrix.python-version }}) + run: find . + - name: Activate Poetry development dependencies # main dependencies are handled by tox if: steps.cached-poetry.outputs.cache-hit == 'true' run: source .venv/bin/activate - - name: Find3 (py${{ matrix.python-version }}) + + - name: Env-Inspect3 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect3 (py${{ matrix.python-version }}) run: find . - - name: Tox env3 (py${{ matrix.python-version }}) - run: env + - name: Load cached tox environment id: cached-toxenv uses: actions/cache@v3 with: path: .tox/py${{ matrix.python-version }}-** key: toxenv-py${{ matrix.python-version }}-${{ hashFiles('poetry.lock', 'tox.ini') }} - - name: Tox id1 (py${{ matrix.python-version }}) - run: which tox || true - - name: Tox id2 (py${{ matrix.python-version }}) - run: poetry run which tox || true + - name: Tox tests (py${{ matrix.python-version }}) run: poetry run tox + - name: Send coverage reports to Codecov uses: codecov/codecov-action@v3 with: @@ -79,7 +83,8 @@ jobs: #name: codecov-umbrella # optional fail_ci_if_error: false verbose: true - - name: WTF1 (py${{ matrix.python-version }}) - run: ls -al manifest.txt || true - - name: WTF2 (py${{ matrix.python-version }}) - run: cat manifest.txt || true + + - name: Env-Inspect4 (py${{ matrix.python-version }}) + run: (echo "\n--- env"; env | sort; echo "\n--- tox"; which tox; echo "\n--- poetry"; which poetry) || true + - name: File-Inspect4 (py${{ matrix.python-version }}) + run: find . From 43927944249b68a25d7e8efa78bd89e1e9e6082b Mon Sep 17 00:00:00 2001 From: Manolis Stamatogiannakis Date: Mon, 11 Dec 2023 20:10:23 +0100 Subject: [PATCH 40/40] x --- .github/workflows/j2cli.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/j2cli.yml b/.github/workflows/j2cli.yml index 1b4ec11..3d742c8 100644 --- a/.github/workflows/j2cli.yml +++ b/.github/workflows/j2cli.yml @@ -33,12 +33,6 @@ jobs: - name: File-Inspect1 (py${{ matrix.python-version }}) run: find . - - name: Load cached Poetry installation - id: cached-poetry - uses: actions/cache@v3 - with: - path: .venv # the path depends on the OS - key: poetry-3 # increment to reset cache - name: Install Poetry if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1