Merge pull request #35 from jmanuel1/python-3.12

Upgrade to Python 3.12

.github/workflows/python-app.yml

@@ -21,28 +21,32 @@ jobs:
     - uses: actions/checkout@v3
       with:
         ref: ${{ github.event.pull_request.head.sha }}
-    - name: Set up Python 3.7
+    - name: Set up Python 3.12
       uses: actions/setup-python@v3
       with:
-        python-version: "3.7"
+        python-version: "3.12"
     - name: Install test dependencies
       run: |
         pip install -e .[test]
     - name: Test
       run: |
         coverage run -m nose2 --pretty-assert concat.tests
     - name: Collect coverage into one file
+      # https://stackoverflow.com/a/58859404
+      if: always()
       run: |
         coverage combine
         coverage xml
         coverage lcov
     - name: Report test coverage to DeepSource
+      if: always()
       uses: deepsourcelabs/test-coverage-action@master
       with:
         key: python
         coverage-file: coverage.xml
         dsn: ${{ secrets.DEEPSOURCE_DSN }}
     - name: Coveralls
+      if: always()
       uses: coverallsapp/github-action@master
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -10,7 +10,7 @@ Examples are in the examples directory. To see the (out of date and incomplete)
 spec, go to
 [http://jmanuel1.github.io/concat-spec/](http://jmanuel1.github.io/concat-spec/).
 
-Python 3.7 required.
+Python 3.12 required.
 
 Development
 -----------
@@ -43,8 +43,8 @@ though `pywinpty` claims to require `x86_64-pc-windows-msvc` to build from
 source. Then set the environment variable `CARGO_BUILD_TARGET` to
 `i686-pc-windows-msvc` and try to install `pywinpty` again.
 
-Run the tests and get coverage info using `tox run`. (Make sure you've installed
-the development dependencies first.)
+Run the tests and get coverage info using `coverage run -m nose2 --pretty-assert
+concat.tests`. (Make sure you've installed the development dependencies first.)
 
 **Nota Bene**: If you have `concat` installed globally, make sure to create and
 enter a `virtualenv` before testing, so you don't end up running the installed

concat/astutils.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from typing import (
     Union,
     List,
@@ -11,6 +12,7 @@
 import ast
 import concat.visitors
 import concat.parse
+import textwrap
 
 
 # Typedefs
@@ -38,42 +40,59 @@ def are_on_same_line_and_offset_by(
 # Python AST Manipulation utilities
 
 
+# TODO: exposing names from modules
+def python_safe_name(id: str, ctx: ast.expr_context) -> ast.Name:
+    """Python 3.8+ disallows None, True, and False as names in compiled code."""
+    return ast.Name(python_safe_name_mangle(id), ctx)
+
+
+def python_safe_name_mangle(id: str) -> str:
+    if id in ['True', 'False', 'None']:
+        return f'@@concat_python_safe_rename_{id}'
+    return id
+
+
 def pop_stack(index: int = -1) -> ast.Call:
-    load = ast.Load()
-    stack = ast.Name(id='stack', ctx=load)
-    pop = ast.Attribute(value=stack, attr='pop', ctx=load)
+    stack = ast.Name(id='stack', ctx=ast.Load())
+    pop = ast.Attribute(value=stack, attr='pop', ctx=ast.Load())
     pop_call = ast.Call(func=pop, args=[ast.Num(index)], keywords=[])
     return pop_call
 
 
 def to_transpiled_quotation(
     words: Words, default_location: Tuple[int, int], visitors: _TranspilerDict
 ) -> ast.expr:
-    quote = concat.parse.QuoteWordNode(
-        list(words), list(words)[0].location if words else default_location
-    )
+    location = list(words)[0].location if words else default_location
+    end_location = list(words)[-1].end_location if words else default_location
+    quote = concat.parse.QuoteWordNode(list(words), location, end_location)
     py_quote = visitors['quote-word'].visit(quote)
     return cast(ast.expr, py_quote)
 
 
 def pack_expressions(expressions: Iterable[ast.expr]) -> ast.Subscript:
-    load = ast.Load()
-    subtuple = ast.Tuple(elts=[*expressions], ctx=load)
-    index = ast.Index(value=ast.Num(n=-1))
-    last = ast.Subscript(value=subtuple, slice=index, ctx=load)
+    subtuple = ast.Tuple(elts=[*expressions], ctx=ast.Load())
+    index = ast.Constant(-1)
+    last = ast.Subscript(value=subtuple, slice=index, ctx=ast.Load())
     return last
 
 
+def copy_location(py_node: ast.AST, node: concat.parse.Node) -> None:
+    py_node.lineno, py_node.col_offset = node.location  # type: ignore
+    py_node.end_lineno, py_node.end_col_offset = node.end_location  # type: ignore
+
+
 def to_python_decorator(
     word: 'concat.parse.WordNode', visitors: _TranspilerDict
 ) -> ast.Lambda:
     push_func = cast(
         ast.Expression, ast.parse('stack.append(func)', mode='eval')
     ).body
+    clear_locations(push_func)
     py_word = cast(ast.expr, visitors['word'].visit(word))
     body = pack_expressions([push_func, py_word, pop_stack()])
     func_arg = ast.arg('func', None)
     arguments = ast.arguments(
+        posonlyargs=[],
         args=[func_arg],
         vararg=None,
         kwonlyargs=[],
@@ -82,6 +101,7 @@ def to_python_decorator(
         kw_defaults=[],
     )
     decorator = ast.Lambda(args=arguments, body=body)
+    copy_location(decorator, word)
     return decorator
 
 
@@ -111,10 +131,12 @@ def statementfy(node: Union[ast.expr, ast.stmt]) -> ast.stmt:
 
 
 def parse_py_qualified_name(name: str) -> Union[ast.Name, ast.Attribute]:
-    return cast(
+    py_node = cast(
         Union[ast.Name, ast.Attribute],
         cast(ast.Expression, ast.parse(name, mode='eval')).body,
     )
+    clear_locations(py_node)
+    return py_node
 
 
 def assert_all_nodes_have_locations(tree: ast.AST) -> None:
@@ -135,23 +157,24 @@ def flatten(list: List[Union['concat.parse.WordNode', Words]]) -> Words:
 
 
 def call_concat_function(func: ast.expr) -> ast.Call:
-    load = ast.Load()
-    stack = ast.Name(id='stack', ctx=load)
-    stash = ast.Name(id='stash', ctx=load)
+    stack = ast.Name(id='stack', ctx=ast.Load())
+    stash = ast.Name(id='stash', ctx=ast.Load())
     call_node = ast.Call(func=func, args=[stack, stash], keywords=[])
     return call_node
 
 
 def abstract(func: ast.expr) -> ast.Lambda:
     args = ast.arguments(
-        [ast.arg('stack', None), ast.arg('stash', None)],
-        None,
-        [],
-        [],
-        None,
-        [],
+        posonlyargs=[],
+        args=[ast.arg('stack', None), ast.arg('stash', None)],
+        vararg=None,
+        kwonlyargs=[],
+        kw_defaults=[],
+        kwarg=None,
+        defaults=[],
     )
     py_node = ast.Lambda(args, func)
+    ast.copy_location(py_node, func)
     return py_node
 
 
@@ -169,7 +192,9 @@ def assign_self_pushing_module_type_to_all_components(
         assignment = '{}.__class__ = concat.stdlib.importlib.Module'.format(
             target
         )
-        yield ast.parse(assignment, mode='exec').body[0]  # type: ignore
+        py_node: ast.Assign = ast.parse(assignment, mode='exec').body[0]  # type: ignore
+        clear_locations(py_node)
+        yield py_node
 
 
 def append_to_stack(expr: ast.expr) -> ast.expr:
@@ -189,3 +214,54 @@ def get_explicit_positional_function_parameters(
 def wrap_in_statement(statments: Iterable[ast.stmt]) -> ast.stmt:
     true = ast.NameConstant(True)
     return ast.If(test=true, body=list(statments), orelse=[])
+
+
+def clear_locations(node: ast.AST) -> None:
+    if hasattr(node, 'lineno'):
+        del node.lineno
+    if hasattr(node, 'col_offset'):
+        del node.col_offset
+    if hasattr(node, 'end_lineno'):
+        del node.end_lineno
+    if hasattr(node, 'end_col_offset'):
+        del node.end_col_offset
+    for field in node._fields:
+        possible_child = getattr(node, field)
+        if isinstance(possible_child, ast.AST):
+            clear_locations(possible_child)
+        elif isinstance(possible_child, list):
+            for pc in possible_child:
+                if isinstance(pc, ast.AST):
+                    clear_locations(pc)
+
+
+def dump_locations(node: ast.AST) -> str:
+    string = type(node).__qualname__
+    if hasattr(node, 'lineno'):
+        string += ' lineno=' + str(node.lineno)
+    if hasattr(node, 'col_offset'):
+        string += ' col_offset=' + str(node.col_offset)
+    if hasattr(node, 'end_lineno'):
+        string += ' end_lineno=' + str(node.end_lineno)
+    if hasattr(node, 'end_col_offset'):
+        string += ' end_col_offset=' + str(node.end_col_offset)
+    for field in node._fields:
+        if not hasattr(node, field):
+            continue
+        possible_child = getattr(node, field)
+        if isinstance(possible_child, ast.AST):
+            string += '\n ' + field + ':'
+            string += '\n' + textwrap.indent(
+                dump_locations(possible_child), '  '
+            )
+        elif isinstance(possible_child, list):
+            string += '\n ' + field + ':'
+            for i, pc in enumerate(possible_child):
+                if isinstance(pc, ast.AST):
+                    string += (
+                        '\n  '
+                        + str(i)
+                        + ':\n'
+                        + textwrap.indent(dump_locations(pc), '   ')
+                    )
+    return string

concat/execute.py

@@ -1,5 +1,6 @@
 """This module takes the transpiler/compiler output and executes it."""
 import ast
+import concat.astutils
 import concat.stdlib.compositional
 import concat.stdlib.execution
 import concat.stdlib.importlib
@@ -169,9 +170,18 @@ def push_func(stack: List[object], _: List[object]):
     globals.setdefault('case', concat.stdlib.execution.case)
     globals.setdefault('loop', concat.stdlib.execution.loop)
 
-    globals.setdefault('True', lambda s, _: s.append(True))
-    globals.setdefault('False', lambda s, _: s.append(False))
-    globals.setdefault('None', lambda s, _: s.append(None))
+    globals.setdefault(
+        concat.astutils.python_safe_name_mangle('True'),
+        lambda s, _: s.append(True),
+    )
+    globals.setdefault(
+        concat.astutils.python_safe_name_mangle('False'),
+        lambda s, _: s.append(False),
+    )
+    globals.setdefault(
+        concat.astutils.python_safe_name_mangle('None'),
+        lambda s, _: s.append(None),
+    )
     globals.setdefault('NotImplemented', lambda s, _: s.append(NotImplemented))
     globals.setdefault('Ellipsis', lambda s, _: s.append(...))
     globals.setdefault('...', lambda s, _: s.append(...))

concat/lex.py

@@ -125,13 +125,13 @@ def _tokens(self) -> Iterator['Token']:
                     if tok.value == ' ':
                         self._update_position(tok)
                         continue
-                    elif tok.value == '$':
-                        tok.type = 'DOLLARSIGN'
                     elif tok.value == '!':
                         tok.type = 'EXCLAMATIONMARK'
                 elif tok.value in {'def', 'import', 'from'}:
                     tok.type = tok.value.upper()
                     tok.is_keyword = True
+                elif tok.value == '$':
+                    tok.type = 'DOLLARSIGN'
                 elif tok.type != 'NAME' and tok.value in {
                     '...',
                     '-',
@@ -171,14 +171,16 @@ def _tokens(self) -> Iterator['Token']:
                 if tok.type == 'NAME':
                     type_map = {'as': 'AS', 'class': 'CLASS', 'cast': 'CAST'}
                     if tok.value in type_map:
-                        tok.type = type_map.get(tok.value)
+                        tok.type = type_map[tok.value]
                         tok.is_keyword = True
                 elif tok.type == 'STRING' and self.__is_bytes_literal(
                     tok.value
                 ):
                     tok.type = 'BYTES'
-                elif tok.type == 'ERRORTOKEN' and tok.value == '`':
+                elif tok.value == '`':
                     tok.type = 'BACKTICK'
+                elif tok.type == 'EXCLAMATION':
+                    tok.type = 'EXCLAMATIONMARK'
 
                 yield tok
 

concat/orderedset.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from concat.linked_list import LinkedList
 from typing import (
     AbstractSet,
@@ -10,6 +11,7 @@
 )
 
 _T = TypeVar('_T', covariant=True)
+_A = TypeVar('_A')
 
 
 class OrderedSet(AbstractSet[_T]):
@@ -64,11 +66,13 @@ def __init__(
             # elements might be an iterator that gets exhausted
             self._data = OrderedSet(self._order)
 
-    def __sub__(self, other: object) -> 'InsertionOrderedSet[_T]':
+    def __sub__(
+        self: InsertionOrderedSet[_A], other: object
+    ) -> 'InsertionOrderedSet[_A]':
         if not isinstance(other, AbstractSet):
             return NotImplemented
         data = self._data - other
-        new_set = InsertionOrderedSet[_T](
+        new_set = InsertionOrderedSet[_A](
             data, self._order.filter(lambda x: x not in other)
         )
         return new_set
@@ -95,10 +99,10 @@ def __len__(self) -> int:
         return len(self._data)
 
 
-def filter_duplicates(xs: LinkedList[_T]) -> LinkedList[_T]:
+def filter_duplicates(xs: LinkedList[_A]) -> LinkedList[_A]:
     found = set()
 
-    def predicate(x: _T) -> bool:
+    def predicate(x: _A) -> bool:
         nonlocal found
         if x in found:
             return False