diff --git a/.vscode/settings.json b/.vscode/settings.json index 3fb9b2dc..94f1a175 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,6 +25,7 @@ "fstring", "getcwd", "graminit", + "htmlcov", "ispy", "kwnames", "lnum", @@ -45,6 +46,7 @@ "tokenisation", "tvchld", "unindent", + "xenial", "ziel" ], "python.pythonPath": ".venv/bin/python", diff --git a/Dockerfile b/Dockerfile index 1ea744e2..dadd631f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # basic info FROM library/ubuntu -LABEL version 0.6.0 +LABEL version 0.6.1.dev1 LABEL description "Ubuntu Environment for F2FORMAT" # prepare environment @@ -23,7 +23,7 @@ RUN apt-get update \ COPY . /tmp/f2format RUN cd /tmp/f2format \ && python3 /f2format/setup.py install \ - && rm -rf /tmp/f2fomat + && rm -rf /tmp/f2format # cleanup RUN rm -rf /var/lib/apt/lists/*\ @@ -34,6 +34,9 @@ RUN rm -rf /var/lib/apt/lists/*\ && apt-get autoclean \ && apt-get clean +# final setup +RUN ln -sf /usr/bin/python3.6 /usr/bin/python3 + # setup entrypoint ENTRYPOINT [ "python3", "-m", "f2format" ] CMD [ "--help" ] diff --git a/Makefile b/Makefile index cfc95055..b291cbd3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean docker release pipenv pypi setup dist test +.PHONY: clean docker release pipenv pypi setup dist test coverage include .env @@ -32,6 +32,11 @@ test-unittest: test-interactive: pipenv run python test/test_driver.py +coverage: + pipenv run coverage run test.py + pipenv run coverage html + open htmlcov/index.html + # setup pipenv setup-pipenv: clean-pipenv pipenv install --dev @@ -187,7 +192,7 @@ release: --description "$(message)" # run distribution process -dist: test +dist: test-unittest $(MAKE) message="$(message)" \ setup clean pypi \ git-upload release setup-formula diff --git a/README.md b/README.md index 9173ed66..cdee8e2f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Also, it always tries to maintain the original layout of source code, and accura ## Installation -> Note that `f2format` only supports Python versions __since 3.3__ +> Note that `f2format` only supports Python versions __since 3.3__ 🐍   For macOS users, `f2format` is now available through [Homebrew](https://brew.sh): @@ -60,7 +60,7 @@ pip install -e . git pull ``` -## Usage +## Basic Usage ### CLI @@ -110,6 +110,28 @@ var = f'foo{(1+2)*3:>5}bar{"a", "b"!r}boo' var = 'foo{:>5}bar{!r}boo'.format((1+2)*3, ("a", "b")) ``` +### Docker + + > Well... it's not published to the Docker Hub yet ;) + +  Considering `f2format` may be used in scenerios where Python is not reachable. +We provide also a Docker image for those poor little guys. + +  See +[`Dockerfile`](https://github.com/JarryShaw/f2format/blob/master/Dockerfile) for more +information. + +### Bundled Executable + + > coming soooooooooooon... + +  For the worst case, we also provide bundled executables of `f2format`. In such case, +you may simply download it then voila, it's ready for you. + +  Special thanks to [PyInstaller](https://www.pyinstaller.org) ❤️ + +## Developer Reference + ### Automator   [`make-demo.sh`](https://github.com/JarryShaw/f2format/blob/master/make-demo.sh) provides a @@ -170,6 +192,8 @@ Returns: ### Codec + > NB: this project is now deprecated, because I just cannot figure out how to play w/ codecs :) +   [`f2format-codec`](https://github.com/JarryShaw/f2format-codec) registers a codec in Python interpreter, which grants you the compatibility to write directly in Python 3.6 *f-string* syntax even through running with a previous version of Python. diff --git a/docs/f2format.rst b/docs/f2format.rst index 2b27746c..569ad476 100644 --- a/docs/f2format.rst +++ b/docs/f2format.rst @@ -6,8 +6,8 @@ f2format back-port compiler for Python 3.6 f-string literals --------------------------------------------------- -:Version: v0.6.0 -:Date: May 05, 2019 +:Version: v0.6.1.dev1 +:Date: June 01, 2019 :Manual section: 1 :Author: Jarry Shaw, a newbie programmer, is the author, owner and maintainer diff --git a/man/f2format.1 b/man/f2format.1 index 5574eb43..b265f9e6 100644 --- a/man/f2format.1 +++ b/man/f2format.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH F2FORMAT 1 "May 05, 2019" "v0.6.0" "" +.TH F2FORMAT 1 "June 01, 2019" "v0.6.1.dev1" "" .SH NAME f2format \- back-port compiler for Python 3.6 f-string literals . diff --git a/setup.py b/setup.py index e02e5a76..cd15014c 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ long_desc = file.read() # version string -__version__ = '0.6.0' +__version__ = '0.6.1.dev1' # set-up script for pip distribution setup( diff --git a/src/__init__.py b/src/__init__.py index 2b5734a7..46c64d4d 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -6,7 +6,7 @@ from f2format.core import * -__all__ = ['f2format', 'convert', 'ConvertError'] +__all__ = ['f2format', 'convert'] ROOT = os.path.dirname(os.path.realpath(__file__)) diff --git a/src/__main__.py b/src/__main__.py index d563d463..c72d5ccb 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -6,7 +6,7 @@ import sys import uuid -from f2format.core import LOCALE_ENCODING, PARSO_VERSION, f2format +from f2format.core import LOCALE_ENCODING, F2FORMAT_VERSION, f2format # multiprocessing may not be supported try: # try first @@ -25,12 +25,12 @@ del multiprocessing # version string -__version__ = '0.6.0' +__version__ = '0.6.1.dev1' # macros __cwd__ = os.getcwd() __archive__ = os.path.join(__cwd__, 'archive') -__f2format_version__ = os.getenv('F2FORMAT_VERSION', PARSO_VERSION[-1]) +__f2format_version__ = os.getenv('F2FORMAT_VERSION', F2FORMAT_VERSION[-1]) __f2format_encoding__ = os.getenv('F2FORMAT_ENCODING', LOCALE_ENCODING) @@ -54,7 +54,7 @@ def get_parser(): convert_group.add_argument('-c', '--encoding', action='store', default=__f2format_encoding__, metavar='CODING', help='encoding to open source files (%s)' % __f2format_encoding__) convert_group.add_argument('-v', '--python', action='store', metavar='VERSION', - default=__f2format_version__, choices=PARSO_VERSION, + default=__f2format_version__, choices=F2FORMAT_VERSION, help='convert against Python version (%s)' % __f2format_version__) parser.add_argument('file', nargs='+', metavar='SOURCE', default=__cwd__, diff --git a/src/core.py b/src/core.py index f2479041..227aef4c 100755 --- a/src/core.py +++ b/src/core.py @@ -11,6 +11,7 @@ ############################################################################### import collections.abc +import glob import io import locale import re @@ -24,7 +25,6 @@ sys.modules.pop('tokenize', None) sys.modules.pop('token', None) -del sys ############################################################################### __all__ = ['f2format', 'convert', 'ConvertError'] @@ -36,7 +36,12 @@ 'on': True, 'off': False} # macros -PARSO_VERSION = ('3.6', '3.7', '3.8') +sys_version = '%s.%s' % sys.version_info[:2] +grammar_regex = re.compile(r"grammar(\d)(\d)\.txt") +parso_version = filter(lambda version: version >= '3.6', # when Python starts to have f-string + map(lambda path: '%s.%s' % grammar_regex.match(os.path.split(path)[1]).groups(), + glob.glob(os.path.join(parso.__path__[0], 'python', 'grammar??.txt')))) +F2FORMAT_VERSION = sorted(filter(lambda version: version <= sys_version, parso_version)) LOCALE_ENCODING = locale.getpreferredencoding() F2FORMAT_QUIET = BOOLEAN_STATES.get(os.getenv('F2FORMAT_QUIET', '0').casefold(), False) @@ -92,12 +97,22 @@ def convert(string, lineno=None): def parse(string, error_recovery=False): try: return parso.parse(string, error_recovery=error_recovery, - version=os.getenv('F2FORMAT_VERSION', PARSO_VERSION[-1])) + version=os.getenv('F2FORMAT_VERSION', F2FORMAT_VERSION[-1])) except parso.ParserSyntaxError as error: message = '%s: <%s: %r> from %r' % (error.message, error.error_leaf.token_type, error.error_leaf.value, string) raise ConvertError(message).with_traceback(error.__traceback__) + def is_tuple(expr): + stripped_expr = expr.strip() + startswith = stripped_expr.startswith('(') + endswith = stripped_expr.endswith(')') + if startswith and endswith: # pragma: no cover + return False + if not (startswith or endswith): + return True + raise ConvertError('malformed node or string:: %r' % expr) # pragma: no cover + source = strarray(string) # strarray source (mutable) f_string = [list()] # [[token, ...], [...], ...] -> concatenable strings @@ -146,32 +161,41 @@ def parse(string, error_recovery=False): # parso.python.tree.PythonNode.children[0] -> parso.python.tree.FStringStart, regex: /^((f|rf|fr)('''|'|"""|"))/ # parso.python.tree.PythonNode.children[-1] -> parso.python.tree.FStringEnd, regex: /('''|'|"""|")$/ for obj in tmpval.children[1:-1]: # traverse parso.python.tree.PythonNode.children -> list # noqa - if obj.type == 'fstring_expr': # expression part (in braces), parso.python.tree.PythonNode # noqa - obj_children = obj.children # parso.python.tree.PythonNode.children -> list - # _[0] -> parso.python.tree.Operator, '{' # noqa - # _[1] -> %undetermined%, expression literal (f_expression) # noqa - # _[2] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa - # _[3] -> parso.python.tree.Operator, '}' # noqa - start_expr_pos = obj_children[1].start_pos # _[1].start_pos -> tuple, (line, offset) # noqa - end_expr_pos = obj_children[1].end_pos # _[1].end_pos -> tuple, (line, offset) # noqa - - start_expr = token_lineno[start_expr_pos[0]] + start_expr_pos[1] - end_expr = token_lineno[end_expr_pos[0]] + end_expr_pos[1] - tmpent.append(slice(start_expr, end_expr)) # entry of expression literal (f_expression) - - if obj_children[2].type == 'fstring_format_spec': - for node in obj_children[2].children: # traverse format specifications (format_spec) - if node.type == 'fstring_expr': # expression part (in braces), parso.python.tree.PythonNode # noqa - node_chld = node.children # parso.python.tree.PythonNode.children -> list # noqa - # _[0] -> parso.python.tree.Operator, '{' # noqa - # _[1] -> %undetermined%, expression literal (f_expression) # noqa - # _[2] -> parso.python.tree.Operator, '}' # noqa - start_spec_pos = node_chld[1].start_pos # _[1].start_pos -> tuple, (line, offset) # noqa - end_spec_pos = node_chld[1].end_pos # _[1].end_pos -> tuple, (line, offset) # noqa - - start_spec = token_lineno[start_spec_pos[0]] + start_spec_pos[1] - end_spec = token_lineno[end_spec_pos[0]] + end_spec_pos[1] - tmpent.append(slice(start_spec, end_spec)) # entry of format specification (format_spec) # noqa + if obj.type != 'fstring_expr': # expression part (in braces), parso.python.tree.PythonNode # noqa + continue + + obj_children = obj.children # parso.python.tree.PythonNode.children -> list + # _[0] -> parso.python.tree.Operator, '{' # noqa + # _[1] -> %undetermined%, expression literal (f_expression) # noqa + # _[2] -> %optional%, parso.python.tree.PythonNode, conversion (fstring_conversion) # noqa + # _[3] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa + # _[4] -> parso.python.tree.Operator, '}' # noqa + start_expr_pos = obj_children[0].end_pos # _[0].end_pos -> tuple, (line, offset) # noqa + end_expr_pos = obj_children[2].start_pos # _[2].start_pos -> tuple, (line, offset) # noqa + + start_expr = token_lineno[start_expr_pos[0]] + start_expr_pos[1] + end_expr = token_lineno[end_expr_pos[0]] + end_expr_pos[1] + tmpent.append(slice(start_expr, end_expr)) # entry of expression literal (f_expression) + + for obj_child in obj_children: + if obj_child.type != 'fstring_format_spec': + continue + for node in obj_child.children: # traverse format specifications (format_spec) + if node.type != 'fstring_expr': # expression part (in braces), parso.python.tree.PythonNode # noqa + continue + + node_chld = node.children # parso.python.tree.PythonNode.children -> list # noqa + # _[0] -> parso.python.tree.Operator, '{' # noqa + # _[1] -> %undetermined%, expression literal (f_expression) # noqa + # _[2] -> %optional%, parso.python.tree.PythonNode, conversion (fstring_conversion) # noqa + # _[3] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa + # _[4] -> parso.python.tree.Operator, '}' # noqa + start_spec_pos = node_chld[0].end_pos # _[0].end_pos -> tuple, (line, offset) # noqa + end_spec_pos = node_chld[2].start_pos # _[2].start_pos -> tuple, (line, offset) # noqa + + start_spec = token_lineno[start_spec_pos[0]] + start_spec_pos[1] + end_spec = token_lineno[end_spec_pos[0]] + end_spec_pos[1] + tmpent.append(slice(start_spec, end_spec)) # entry of format specification (format_spec) # noqa # print('length:', length, '###', token_string[:length], '###', token_string[length:]) ### entryl.append((token, tmpent)) # each token with a concatenation entry list @@ -184,9 +208,8 @@ def parse(string, error_recovery=False): # print(token.string, entries) ### for entry in entries: # walk entries temp_expr = token.string[entry] # original expression - val = parse(temp_expr, error_recovery=True).children[0] # parse AST - if val.type == 'testlist_star_expr' and \ - re.fullmatch(r'\(.*\)', temp_expr, re.DOTALL) is None: # if expression is implicit tuple + val = parse(temp_expr.strip(), error_recovery=True).children[0] # parse AST + if val.type == 'testlist_star_expr' and is_tuple(temp_expr): # if expression is implicit tuple real_expr = '(%s)' % temp_expr # add parentheses else: real_expr = temp_expr # or keep original @@ -197,11 +220,12 @@ def parse(string, error_recovery=False): # pprint.pprint(expr) ### # convert end of f-string to str.format literal - end = lineno[tokens[-1].end[0]] + tokens[-1].end[1] - if len(source) == end: - source[end:end+1] = '.format(%s)' % (', '.join(expr)) - else: - source[end:end+1] = '.format(%s)%s' % (', '.join(expr), source[end]) + if expr: + end = lineno[tokens[-1].end[0]] + tokens[-1].end[1] + if len(source) == end: + source[end:end+1] = '.format(%s)' % (', '.join(expr)) + else: + source[end:end+1] = '.format(%s)%s' % (', '.join(expr), source[end]) # for each token, convert expression literals and brace '{}' escape sequences for token, entries in reversed(entryl): # using reversed to keep offset in leading context @@ -237,7 +261,7 @@ def f2format(filename): """ if not F2FORMAT_QUIET: - print('Now converting %r...' % filename) # pragma: no cover + print('Now converting %r...' % filename) # fetch encoding encoding = os.getenv('F2FORMAT_ENCODING', LOCALE_ENCODING) diff --git a/test.py b/test.py index 63fa5ee7..28b6c070 100644 --- a/test.py +++ b/test.py @@ -10,15 +10,12 @@ import tempfile import unittest -from f2format.__main__ import get_parser -from f2format.__main__ import main as main_func -from f2format.core import ConvertError, convert -from f2format.core import f2format as core_func - class TestF2format(unittest.TestCase): def test_get_parser(self): + from f2format.__main__ import get_parser + parser = get_parser() args = parser.parse_args(['-n', '-q', '-p/tmp/', '-cgb2312', '-v3.6', @@ -37,7 +34,13 @@ def test_get_parser(self): self.assertEqual(args.file, ['test1.py', 'test2.py'], 'python source files and folders to be converted') + # reset environ + del sys.modules['f2format'] + del sys.modules['f2format.__main__'] + def test_main_func(self): + from f2format.__main__ import main as main_func + src_files = glob.glob(os.path.join(os.path.dirname(__file__), 'test', 'test_?.py')) dst_files = list() @@ -53,9 +56,17 @@ def test_main_func(self): with open(os.devnull, 'w') as devnull: with contextlib.redirect_stderr(devnull): with self.assertRaises(SystemExit): - main_func(['comp', 'docker', 'docs']) + main_func(['-p', os.path.join(tempdir, 'archive'), + 'comp', 'docker', 'docs']) + with contextlib.redirect_stderr(devnull): + with self.assertRaises(SystemExit): + main_func(['--no-archive', 'comp', 'docker', 'docs']) with contextlib.redirect_stdout(devnull): main_func(dst_files) + with contextlib.redirect_stdout(devnull): + temp_args = ['--no-archive', '--quiet'] + temp_args.extend(dst_files) + main_func(temp_args) for dst in dst_files: src = os.path.join(os.path.dirname(__file__), 'test', @@ -65,30 +76,50 @@ def test_main_func(self): new = subprocess.run([sys.executable, dst], stdout=subprocess.PIPE) self.assertEqual(old, new.stdout.decode()) + # reset environ + del sys.modules['f2format'] + del sys.modules['f2format.__main__'] + @unittest.skipIf(sys.version_info[:2] < (3, 6), "not supported in this Python version") def test_core_func(self): - src_files = glob.glob(os.path.join(os.path.dirname(__file__), - 'test', 'test_?.py')) + def test_core_func_main(): + from f2format.core import f2format as core_func - os.environ['F2FORMAT_QUIET'] = '1' - with tempfile.TemporaryDirectory() as tempdir: - for src in src_files: - name = os.path.split(src)[1] - dst = os.path.join(tempdir, name) - shutil.copy(src, dst) + src_files = glob.glob(os.path.join(os.path.dirname(__file__), + 'test', 'test_?.py')) - # run f2format - core_func(dst) + with open(os.devnull, 'w') as devnull: + with contextlib.redirect_stdout(devnull): + with tempfile.TemporaryDirectory() as tempdir: + for src in src_files: + name = os.path.split(src)[1] + dst = os.path.join(tempdir, name) + shutil.copy(src, dst) - old = subprocess.run([sys.executable, src], stdout=subprocess.PIPE) - new = subprocess.run([sys.executable, dst], stdout=subprocess.PIPE) - self.assertEqual(old.stdout.decode(), new.stdout.decode()) + # run f2format + core_func(dst) + + old = subprocess.run([sys.executable, src], stdout=subprocess.PIPE) + new = subprocess.run([sys.executable, dst], stdout=subprocess.PIPE) + self.assertEqual(old.stdout.decode(), new.stdout.decode()) + + # reset environ + del sys.modules['f2format'] + del sys.modules['f2format.core'] + + os.environ['F2FORMAT_QUIET'] = '1' + test_core_func_main() + + os.environ['F2FORMAT_QUIET'] = '0' + test_core_func_main() # reset environ del os.environ['F2FORMAT_QUIET'] def test_convert(self): + from f2format.core import ConvertError, convert + # normal convertion src = """var = f'foo{(1+2)*3:>5}bar{"a", "b"!r}boo'""" dst = convert(src) @@ -100,7 +131,10 @@ def test_convert(self): convert("""var = f'foo{{(1+2)*3:>5}bar{"a", "b"!r}boo'""") # reset environ + del sys.modules['f2format'] + del sys.modules['f2format.core'] del os.environ['F2FORMAT_VERSION'] + if __name__ == '__main__': unittest.main() diff --git a/test/test_5.py b/test/test_5.py index 7330d7bf..5d9d6af3 100644 --- a/test/test_5.py +++ b/test/test_5.py @@ -1 +1,2 @@ print(f'{1, 2} { 3, 4 } {(5, 6)}') +print(f'Ha ha! Gotcha!') diff --git a/test/test_5.pyw b/test/test_5.pyw index 7330d7bf..5d9d6af3 100644 --- a/test/test_5.pyw +++ b/test/test_5.pyw @@ -1 +1,2 @@ print(f'{1, 2} { 3, 4 } {(5, 6)}') +print(f'Ha ha! Gotcha!') diff --git a/test/test_5.txt b/test/test_5.txt index c40f3cec..0a944229 100644 --- a/test/test_5.txt +++ b/test/test_5.txt @@ -1 +1,2 @@ (1, 2) (3, 4) (5, 6) +Ha ha! Gotcha!