fix: python 3.12 parsing error, according to PEP-709

The previous code raised an error when parsing list/set/dict
comprehensions; "IndexError: pop from empty list".

    current_table = self._table_stack.pop()  # <--- HERE

As per PEP-0709:

The symtable module will no longer produce child symbol tables for
comprehensions (but excluding generator expressions); instead, the
comprehension’s locals will be included in the parent function’s
symbol table and scope.

rplugin/python3/semshi/visitor.py

@@ -8,10 +8,21 @@
 from .node import ATTRIBUTE, IMPORTED, PARAMETER_UNUSED, SELF, Node
 from .util import debug_time
 
-# Node types which introduce a new scope
-BLOCKS = (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef,
-          ast.ListComp, ast.DictComp, ast.SetComp,
-          ast.GeneratorExp, ast.Lambda)
+# Node types which introduce a new scope and child symboltable
+BLOCKS = (
+    ast.Module, ast.Lambda,
+    ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef,
+    ast.GeneratorExp,
+)
+if sys.version_info < (3, 12):
+    # PEP-709: comprehensions no longer have dedicated stack frames; the
+    # comprehension's local will be included in the parent function's symtable
+    # (Note: generator expressions are excluded in Python 3.12)
+    BLOCKS = tuple(
+        list(BLOCKS) +
+        [ast.ListComp, ast.DictComp, ast.SetComp]
+    )
+
 FUNCTION_BLOCKS = (ast.FunctionDef, ast.Lambda, ast.AsyncFunctionDef)
 
 # Node types which don't require any action
@@ -83,6 +94,7 @@ def visit(self, node):
             return
         if type_ in SKIP:
             return
+
         if type_ is ast.Try:
             self._visit_try(node)
         elif type_ is ast.ExceptHandler:

test/test_parser.py

@@ -4,23 +4,24 @@
 
 import sys
 from pathlib import Path
-import sys
 from textwrap import dedent
 
 import pytest
-
 from semshi.node import (ATTRIBUTE, BUILTIN, FREE, GLOBAL, IMPORTED, LOCAL,
                          PARAMETER, PARAMETER_UNUSED, SELF, UNRESOLVED, Node,
                          group)
 from semshi.parser import Parser, UnparsableError
 
 from .conftest import make_parser, make_tree, parse
 
-
 # top-level functions are parsed as LOCAL in python<3.7,
 # but as GLOBAL in Python 3.8.
 MODULE_FUNC = GLOBAL if sys.version_info >= (3, 8) else LOCAL
 
+# Python 3.12: comprehensions no longer have their own variable scopes
+# https://peps.python.org/pep-0709/
+PEP_709 = sys.version_info >= (3, 12)
+
 
 def test_group():
     assert group('foo') == 'semshiFoo'
@@ -154,18 +155,47 @@ def test_name_len():
 def test_comprehension_scopes():
     names = parse(r'''
         #!/usr/bin/env python3
-        [a for b in c]
-        (d for e in f)
+        (a for b in c)
+        [d for e in f]
         {g for h in i}
         {j:k for l in m}
     ''')
     root = make_tree(names)
-    assert root['names'] == ['c', 'f', 'i', 'm']
-    assert root['listcomp']['names'] == ['a', 'b']
-    assert root['genexpr']['names'] == ['d', 'e']
-    assert root['setcomp']['names'] == ['g', 'h']
-    assert root['dictcomp']['names'] == ['j', 'k', 'l']
-
+    groups = {n.name: n.hl_group for n in names}
+    print(f"root = {root}")
+    print(f"groups = {groups}")
+
+    if not PEP_709:
+        assert root['names'] == ['c', 'f', 'i', 'm']
+        assert root['genexpr']['names'] == ['a', 'b']
+        assert root['listcomp']['names'] == ['d', 'e']
+        assert root['setcomp']['names'] == ['g', 'h']
+        assert root['dictcomp']['names'] == ['j', 'k', 'l']
+
+        # generator variables b, e, h, l are local within the scope
+        assert [name for name, group in groups.items() if group == LOCAL
+                ] == ['b', 'e', 'h', 'l']
+        assert [name for name, group in groups.items() if group == UNRESOLVED
+                ] == ['c', 'a', 'f', 'd', 'i', 'g', 'm', 'j', 'k']
+
+    else:
+        # PEP-709, Python 3.12+: comprehensions do not have scope of their own.
+        # so all the symbol is contained in the root node (ast.Module)
+        assert root['names'] == [
+            # in the order nodes are visited and evaluated
+            'c',  # generators have nested scope !!!
+            'f', 'd', 'e',
+            'i', 'g', 'h',
+            'm', 'j', 'k', 'l'
+        ]
+        # no comprehension children nodes
+        assert list(root.keys()) == ['names', 'genexpr']
+
+        # generator variables e, h, l have the scope of the top-level module
+        assert [name for name, group in groups.items() if group == GLOBAL
+                ] == ['e', 'h', 'l']   # b is defined within the generator scope
+        assert [name for name, group in groups.items() if group == UNRESOLVED
+                ] == ['c', 'a', 'f', 'd', 'i', 'g', 'm', 'j', 'k']
 
 def test_function_scopes():
     names = parse(r'''
@@ -177,10 +207,15 @@ def func2(j=k):
         func(x, y=p, **z)
     ''')
     root = make_tree(names)
+    print(f"root = {root}")
+
     assert root['names'] == [
-        'e', 'h', 'func', 'k', 'func2', 'func', 'x', 'p', 'z'
+        'e', 'h',
+        *(['g', 'g'] if PEP_709 else []),
+        'func', 'k', 'func2', 'func', 'x', 'p', 'z'
     ]
-    assert root['listcomp']['names'] == ['g', 'g']
+    if not PEP_709:
+        assert root['listcomp']['names'] == ['g', 'g']
     assert root['func']['names'] == ['a', 'b', 'c', 'd', 'f', 'i']
     assert root['func2']['names'] == ['j']
 
@@ -385,11 +420,21 @@ def test_nested_comprehension():
         [o for p, q, r in s]
     ''')
     root = make_tree(names)
-    assert root['names'] == ['c', 'n', 's']
-    assert root['listcomp']['names'] == [
-        'a', 'b', 'd', 'e', 'f', 'g', 'l', 'm', 'z', 'k', 'h', 'i', 'o', 'p',
-        'q', 'r'
-    ]
+    if not PEP_709:
+        assert root['names'] == ['c', 'n', 's']
+        assert root['listcomp']['names'] == [
+            'a', 'b', 'd', 'e', 'f', 'g', 'l', 'm', 'z', 'k', 'h', 'i', 'o', 'p',
+            'q', 'r'
+        ]
+    else:
+        # Python 3.12: all the 18 symbols are included in the root scope
+        assert root['names'] == [
+            *['c', 'a', 'b'], *['d', 'e', 'f', 'g'],
+            *['n', 'l', 'm'], *['z', 'x', 'y'], 'k', 'h', 'i',
+            *['s', 'o', 'p', 'q', 'r']
+        ]
+        assert 'listcomp' not in root
+
 
 def test_try_except_order():
     names = parse(r'''
@@ -847,7 +892,7 @@ def foo(x):
     ''')
     assert [n.hl_group for n in names if n.name == 'x'] == [
         PARAMETER,
-        FREE
+        PARAMETER if PEP_709 else FREE
     ]