New distribution [0.6.1.dev1]

enhanced convertion results
fixed convertion bugs when expression has whitespaces around
resvied test cases
updated README

.vscode/settings.json

@@ -25,6 +25,7 @@
         "fstring",
         "getcwd",
         "graminit",
+        "htmlcov",
         "ispy",
         "kwnames",
         "lnum",
@@ -45,6 +46,7 @@
         "tokenisation",
         "tvchld",
         "unindent",
+        "xenial",
         "ziel"
     ],
     "python.pythonPath": ".venv/bin/python",

Dockerfile

@@ -1,6 +1,6 @@
 # basic info
 FROM library/ubuntu
-LABEL version 0.6.0
+LABEL version 0.6.1.dev1
 LABEL description "Ubuntu Environment for F2FORMAT"
 
 # prepare environment
@@ -23,7 +23,7 @@ RUN apt-get update \
 COPY . /tmp/f2format
 RUN cd /tmp/f2format \
  && python3 /f2format/setup.py install \
- && rm -rf /tmp/f2fomat
+ && rm -rf /tmp/f2format
 
 # cleanup
 RUN rm -rf /var/lib/apt/lists/*\
@@ -34,6 +34,9 @@ RUN rm -rf /var/lib/apt/lists/*\
  && apt-get autoclean \
  && apt-get clean
 
+# final setup
+RUN ln -sf /usr/bin/python3.6 /usr/bin/python3
+
 # setup entrypoint
 ENTRYPOINT [ "python3", "-m", "f2format" ]
 CMD [ "--help" ]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean docker release pipenv pypi setup dist test
+.PHONY: clean docker release pipenv pypi setup dist test coverage
 
 include .env
 
@@ -32,6 +32,11 @@ test-unittest:
 test-interactive:
 	pipenv run python test/test_driver.py
 
+coverage:
+	pipenv run coverage run test.py
+	pipenv run coverage html
+	open htmlcov/index.html
+
 # setup pipenv
 setup-pipenv: clean-pipenv
 	pipenv install --dev
@@ -187,7 +192,7 @@ release:
 		--description "$(message)"
 
 # run distribution process
-dist: test
+dist: test-unittest
 	$(MAKE) message="$(message)" \
 		setup clean pypi \
 		git-upload release setup-formula

README.md

@@ -33,7 +33,7 @@ Also, it always tries to maintain the original layout of source code, and accura
 
 ## Installation
 
-> Note that `f2format` only supports Python versions __since 3.3__
+> Note that `f2format` only supports Python versions __since 3.3__ 🐍
 
   For macOS users, `f2format` is now available through [Homebrew](https://brew.sh):
 
@@ -60,7 +60,7 @@ pip install -e .
 git pull
 ```
 
-## Usage
+## Basic Usage
 
 ### CLI
 
@@ -110,6 +110,28 @@ var = f'foo{(1+2)*3:>5}bar{"a", "b"!r}boo'
 var = 'foo{:>5}bar{!r}boo'.format((1+2)*3, ("a", "b"))
 ```
 
+### Docker
+
+ > Well... it's not published to the Docker Hub yet ;)
+
+  Considering `f2format` may be used in scenerios where Python is not reachable.
+We provide also a Docker image for those poor little guys.
+
+  See
+[`Dockerfile`](https://github.com/JarryShaw/f2format/blob/master/Dockerfile) for more
+information.
+
+### Bundled Executable
+
+ > coming soooooooooooon...
+
+  For the worst case, we also provide bundled executables of `f2format`. In such case,
+you may simply download it then voila, it's ready for you.
+
+  Special thanks to [PyInstaller](https://www.pyinstaller.org) ❤️
+
+## Developer Reference
+
 ### Automator
 
   [`make-demo.sh`](https://github.com/JarryShaw/f2format/blob/master/make-demo.sh) provides a
@@ -170,6 +192,8 @@ Returns:
 
 ### Codec
 
+ > NB: this project is now deprecated, because I just cannot figure out how to play w/ codecs :)
+
   [`f2format-codec`](https://github.com/JarryShaw/f2format-codec) registers a codec in Python
 interpreter, which grants you the compatibility to write directly in Python 3.6 *f-string* syntax
 even through running with a previous version of Python.

docs/f2format.rst

@@ -6,8 +6,8 @@ f2format
 back-port compiler for Python 3.6 f-string literals
 ---------------------------------------------------
 
-:Version: v0.6.0
-:Date: May 05, 2019
+:Version: v0.6.1.dev1
+:Date: June 01, 2019
 :Manual section: 1
 :Author:
     Jarry Shaw, a newbie programmer, is the author, owner and maintainer

man/f2format.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH F2FORMAT 1 "May 05, 2019" "v0.6.0" ""
+.TH F2FORMAT 1 "June 01, 2019" "v0.6.1.dev1" ""
 .SH NAME
 f2format \- back-port compiler for Python 3.6 f-string literals
 .

setup.py

@@ -13,7 +13,7 @@
     long_desc = file.read()
 
 # version string
-__version__ = '0.6.0'
+__version__ = '0.6.1.dev1'
 
 # set-up script for pip distribution
 setup(

src/__init__.py

@@ -6,7 +6,7 @@
 
 from f2format.core import *
 
-__all__ = ['f2format', 'convert', 'ConvertError']
+__all__ = ['f2format', 'convert']
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 

src/__main__.py

@@ -6,7 +6,7 @@
 import sys
 import uuid
 
-from f2format.core import LOCALE_ENCODING, PARSO_VERSION, f2format
+from f2format.core import LOCALE_ENCODING, F2FORMAT_VERSION, f2format
 
 # multiprocessing may not be supported
 try:        # try first
@@ -25,12 +25,12 @@
     del multiprocessing
 
 # version string
-__version__ = '0.6.0'
+__version__ = '0.6.1.dev1'
 
 # macros
 __cwd__ = os.getcwd()
 __archive__ = os.path.join(__cwd__, 'archive')
-__f2format_version__ = os.getenv('F2FORMAT_VERSION', PARSO_VERSION[-1])
+__f2format_version__ = os.getenv('F2FORMAT_VERSION', F2FORMAT_VERSION[-1])
 __f2format_encoding__ = os.getenv('F2FORMAT_ENCODING', LOCALE_ENCODING)
 
 
@@ -54,7 +54,7 @@ def get_parser():
     convert_group.add_argument('-c', '--encoding', action='store', default=__f2format_encoding__, metavar='CODING',
                                help='encoding to open source files (%s)' % __f2format_encoding__)
     convert_group.add_argument('-v', '--python', action='store', metavar='VERSION',
-                               default=__f2format_version__, choices=PARSO_VERSION,
+                               default=__f2format_version__, choices=F2FORMAT_VERSION,
                                help='convert against Python version (%s)' % __f2format_version__)
 
     parser.add_argument('file', nargs='+', metavar='SOURCE', default=__cwd__,

src/core.py

@@ -11,6 +11,7 @@
 ###############################################################################
 
 import collections.abc
+import glob
 import io
 import locale
 import re
@@ -24,7 +25,6 @@
 
 sys.modules.pop('tokenize', None)
 sys.modules.pop('token', None)
-del sys
 ###############################################################################
 
 __all__ = ['f2format', 'convert', 'ConvertError']
@@ -36,7 +36,12 @@
                   'on': True, 'off': False}
 
 # macros
-PARSO_VERSION = ('3.6', '3.7', '3.8')
+sys_version = '%s.%s' % sys.version_info[:2]
+grammar_regex = re.compile(r"grammar(\d)(\d)\.txt")
+parso_version = filter(lambda version: version >= '3.6',  # when Python starts to have f-string
+                       map(lambda path: '%s.%s' % grammar_regex.match(os.path.split(path)[1]).groups(),
+                           glob.glob(os.path.join(parso.__path__[0], 'python', 'grammar??.txt'))))
+F2FORMAT_VERSION = sorted(filter(lambda version: version <= sys_version, parso_version))
 LOCALE_ENCODING = locale.getpreferredencoding()
 F2FORMAT_QUIET = BOOLEAN_STATES.get(os.getenv('F2FORMAT_QUIET', '0').casefold(), False)
 
@@ -92,12 +97,22 @@ def convert(string, lineno=None):
     def parse(string, error_recovery=False):
         try:
             return parso.parse(string, error_recovery=error_recovery,
-                               version=os.getenv('F2FORMAT_VERSION', PARSO_VERSION[-1]))
+                               version=os.getenv('F2FORMAT_VERSION', F2FORMAT_VERSION[-1]))
         except parso.ParserSyntaxError as error:
             message = '%s: <%s: %r> from %r' % (error.message, error.error_leaf.token_type,
                                                 error.error_leaf.value, string)
             raise ConvertError(message).with_traceback(error.__traceback__)
 
+    def is_tuple(expr):
+        stripped_expr = expr.strip()
+        startswith = stripped_expr.startswith('(')
+        endswith = stripped_expr.endswith(')')
+        if startswith and endswith:  # pragma: no cover
+            return False
+        if not (startswith or endswith):
+            return True
+        raise ConvertError('malformed node or string:: %r' % expr)  # pragma: no cover
+
     source = strarray(string)       # strarray source (mutable)
     f_string = [list()]             # [[token, ...], [...], ...] -> concatenable strings
 
@@ -146,32 +161,41 @@ def parse(string, error_recovery=False):
                 # parso.python.tree.PythonNode.children[0] -> parso.python.tree.FStringStart, regex: /^((f|rf|fr)('''|'|"""|"))/
                 # parso.python.tree.PythonNode.children[-1] -> parso.python.tree.FStringEnd, regex: /('''|'|"""|")$/
                 for obj in tmpval.children[1:-1]:               # traverse parso.python.tree.PythonNode.children -> list # noqa
-                    if obj.type == 'fstring_expr':                      # expression part (in braces), parso.python.tree.PythonNode # noqa
-                        obj_children = obj.children                             # parso.python.tree.PythonNode.children -> list
-                                                                                # _[0] -> parso.python.tree.Operator, '{' # noqa
-                                                                                # _[1] -> %undetermined%, expression literal (f_expression) # noqa
-                                                                                # _[2] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa
-                                                                                # _[3] -> parso.python.tree.Operator, '}' # noqa
-                        start_expr_pos = obj_children[1].start_pos              # _[1].start_pos -> tuple, (line, offset) # noqa
-                        end_expr_pos = obj_children[1].end_pos                  # _[1].end_pos -> tuple, (line, offset) # noqa
-
-                        start_expr = token_lineno[start_expr_pos[0]] + start_expr_pos[1]
-                        end_expr = token_lineno[end_expr_pos[0]] + end_expr_pos[1]
-                        tmpent.append(slice(start_expr, end_expr))              # entry of expression literal (f_expression)
-
-                        if obj_children[2].type == 'fstring_format_spec':
-                            for node in obj_children[2].children:               # traverse format specifications (format_spec)
-                                if node.type == 'fstring_expr':                         # expression part (in braces), parso.python.tree.PythonNode # noqa
-                                    node_chld = node.children                                   # parso.python.tree.PythonNode.children -> list # noqa
-                                                                                                # _[0] -> parso.python.tree.Operator, '{' # noqa
-                                                                                                # _[1] -> %undetermined%, expression literal (f_expression) # noqa
-                                                                                                # _[2] -> parso.python.tree.Operator, '}' # noqa
-                                    start_spec_pos = node_chld[1].start_pos                     # _[1].start_pos -> tuple, (line, offset) # noqa
-                                    end_spec_pos = node_chld[1].end_pos                         # _[1].end_pos -> tuple, (line, offset) # noqa
-
-                                    start_spec = token_lineno[start_spec_pos[0]] + start_spec_pos[1]
-                                    end_spec = token_lineno[end_spec_pos[0]] + end_spec_pos[1]
-                                    tmpent.append(slice(start_spec, end_spec))                  # entry of format specification (format_spec) # noqa
+                    if obj.type != 'fstring_expr':                      # expression part (in braces), parso.python.tree.PythonNode # noqa
+                        continue
+
+                    obj_children = obj.children                             # parso.python.tree.PythonNode.children -> list
+                                                                            # _[0] -> parso.python.tree.Operator, '{' # noqa
+                                                                            # _[1] -> %undetermined%, expression literal (f_expression) # noqa
+                                                                            # _[2] -> %optional%, parso.python.tree.PythonNode, conversion (fstring_conversion) # noqa
+                                                                            # _[3] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa
+                                                                            # _[4] -> parso.python.tree.Operator, '}' # noqa
+                    start_expr_pos = obj_children[0].end_pos                # _[0].end_pos -> tuple, (line, offset) # noqa
+                    end_expr_pos = obj_children[2].start_pos                # _[2].start_pos -> tuple, (line, offset) # noqa
+
+                    start_expr = token_lineno[start_expr_pos[0]] + start_expr_pos[1]
+                    end_expr = token_lineno[end_expr_pos[0]] + end_expr_pos[1]
+                    tmpent.append(slice(start_expr, end_expr))          # entry of expression literal (f_expression)
+
+                    for obj_child in obj_children:
+                        if obj_child.type != 'fstring_format_spec':
+                            continue
+                        for node in obj_child.children:                 # traverse format specifications (format_spec)
+                            if node.type != 'fstring_expr':                 # expression part (in braces), parso.python.tree.PythonNode # noqa
+                                continue
+
+                            node_chld = node.children                       # parso.python.tree.PythonNode.children -> list # noqa
+                                                                            # _[0] -> parso.python.tree.Operator, '{' # noqa
+                                                                            # _[1] -> %undetermined%, expression literal (f_expression) # noqa
+                                                                            # _[2] -> %optional%, parso.python.tree.PythonNode, conversion (fstring_conversion) # noqa
+                                                                            # _[3] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa
+                                                                            # _[4] -> parso.python.tree.Operator, '}' # noqa
+                            start_spec_pos = node_chld[0].end_pos           # _[0].end_pos -> tuple, (line, offset) # noqa
+                            end_spec_pos = node_chld[2].start_pos           # _[2].start_pos -> tuple, (line, offset) # noqa
+
+                            start_spec = token_lineno[start_spec_pos[0]] + start_spec_pos[1]
+                            end_spec = token_lineno[end_spec_pos[0]] + end_spec_pos[1]
+                            tmpent.append(slice(start_spec, end_spec))      # entry of format specification (format_spec) # noqa
                     # print('length:', length, '###', token_string[:length], '###', token_string[length:]) ###
             entryl.append((token, tmpent))          # each token with a concatenation entry list
 
@@ -184,9 +208,8 @@ def parse(string, error_recovery=False):
             # print(token.string, entries) ###
             for entry in entries:       # walk entries
                 temp_expr = token.string[entry]                                 # original expression
-                val = parse(temp_expr, error_recovery=True).children[0]         # parse AST
-                if val.type == 'testlist_star_expr' and \
-                        re.fullmatch(r'\(.*\)', temp_expr, re.DOTALL) is None:  # if expression is implicit tuple
+                val = parse(temp_expr.strip(), error_recovery=True).children[0] # parse AST
+                if val.type == 'testlist_star_expr' and is_tuple(temp_expr):    # if expression is implicit tuple
                     real_expr = '(%s)' % temp_expr                              # add parentheses
                 else:
                     real_expr = temp_expr                                       # or keep original
@@ -197,11 +220,12 @@ def parse(string, error_recovery=False):
         # pprint.pprint(expr) ###
 
         # convert end of f-string to str.format literal
-        end = lineno[tokens[-1].end[0]] + tokens[-1].end[1]
-        if len(source) == end:
-            source[end:end+1] = '.format(%s)' % (', '.join(expr))
-        else:
-            source[end:end+1] = '.format(%s)%s' % (', '.join(expr), source[end])
+        if expr:
+            end = lineno[tokens[-1].end[0]] + tokens[-1].end[1]
+            if len(source) == end:
+                source[end:end+1] = '.format(%s)' % (', '.join(expr))
+            else:
+                source[end:end+1] = '.format(%s)%s' % (', '.join(expr), source[end])
 
         # for each token, convert expression literals and brace '{}' escape sequences
         for token, entries in reversed(entryl):     # using reversed to keep offset in leading context
@@ -237,7 +261,7 @@ def f2format(filename):
 
     """
     if not F2FORMAT_QUIET:
-        print('Now converting %r...' % filename)  # pragma: no cover
+        print('Now converting %r...' % filename)
 
     # fetch encoding
     encoding = os.getenv('F2FORMAT_ENCODING', LOCALE_ENCODING)