gh-124697: Represent inlined comprehensions as subscopes in the symbol table (#156819)

Author: iritkatriel

Doc/library/symtable.rst

@@ -57,6 +57,16 @@ Examining Symbol Tables
 
       Used for the symbol table of a class.
 
+   .. attribute:: INLINED_COMPREHENSION
+      :value: "inlined comprehension"
+
+      Used for the symbol table of a list, set or dict comprehension that
+      is inlined into the enclosing code unit (see :pep:`709`). A symbol
+      table of this type represents a sub-scope of the enclosing code unit's
+      scope, and it does not correspond to a separate compilation unit.
+
+      .. versionadded:: next
+
    The following members refer to different flavors of
    :ref:`annotation scopes <annotation-scopes>`.
 

Doc/whatsnew/3.16.rst

@@ -573,6 +573,16 @@ symtable
   like the builtin :func:`compile`.
   (Contributed by Serhiy Storchaka in :gh:`153844`.)
 
+* Inlined list, set and dict comprehensions (:pep:`709`) are now represented
+  as their own symbol table entries, of type
+  :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. Each such entry is
+  a lexical child of the enclosing scope and records the comprehension's own
+  locals, cells, and free names. It does not correspond to a separate
+  compilation unit. For names loaded only inside an inlined list, set, or
+  dict comprehension, :meth:`symtable.Symbol.is_referenced` can now return
+  ``True`` on the enclosing function.
+  (Contributed by Irit Katriel in :gh:`124697`.)
+
 
 tkinter
 -------

Include/internal/pycore_compile.h

@@ -69,9 +69,8 @@ typedef struct {
     PyObject *u_varnames;  /* local variables */
     PyObject *u_cellvars;  /* cell variables */
     PyObject *u_freevars;  /* free variables */
-    PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only
-                               temporarily within an inlined comprehension. When
-                               value is True, treat as fast-local. */
+    PyObject *u_fasthidden; /* set of names that are fast-locals only
+                               temporarily within an inlined comprehension. */
 
     Py_ssize_t u_argcount;        /* number of arguments for block */
     Py_ssize_t u_posonlyargcount;        /* number of positional only arguments for block */
@@ -152,11 +151,10 @@ PyObject *_PyCompile_MaybeMangle(struct _PyCompiler *c, PyObject *name);
 int _PyCompile_MaybeAddStaticAttributeToClass(struct _PyCompiler *c, expr_ty e);
 int _PyCompile_GetRefType(struct _PyCompiler *c, PyObject *name);
 int _PyCompile_LookupCellvar(struct _PyCompiler *c, PyObject *name);
-int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope,
+int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled,
                              _PyCompile_optype *optype, Py_ssize_t *arg);
 
 int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c);
-int _PyCompile_IsInInlinedComp(struct _PyCompiler *c);
 int _PyCompile_ScopeType(struct _PyCompiler *c);
 int _PyCompile_OptimizationLevel(struct _PyCompiler *c);
 int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name);
@@ -180,16 +178,15 @@ enum {
 
 typedef struct {
     PyObject *pushed_locals;
-    PyObject *temp_symbols;
-    PyObject *fast_hidden;
     _PyJumpTargetLabel cleanup;
+    PySTEntryObject *saved_ste;
 } _PyCompile_InlinedComprehensionState;
 
-int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc,
-                                               PySTEntryObject *entry,
-                                               _PyCompile_InlinedComprehensionState *state);
-int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc,
-                                                _PyCompile_InlinedComprehensionState *state);
+int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c,
+                                              PySTEntryObject *entry,
+                                              _PyCompile_InlinedComprehensionState *state);
+int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c,
+                                             _PyCompile_InlinedComprehensionState *state);
 int _PyCompile_AddDeferredAnnotation(struct _PyCompiler *c, stmt_ty s,
                                      PyObject **conditional_annotation_index);
 void _PyCompile_EnterConditionalBlock(struct _PyCompiler *c);

Include/internal/pycore_symtable.h

@@ -33,6 +33,10 @@ typedef enum _block_type {
     // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two
     // do not support a bound or a constraint tuple).
     TypeVariableBlock,
+    // List/set/dict comprehension inlined into the enclosing compilation unit
+    // (PEP 709). Lexical child of that unit, not a separate code object.
+    // See InternalDocs/inlined_comprehensions.md.
+    InlinedComprehensionBlock,
 } _Py_block_ty;
 
 typedef enum _comprehension_type {
@@ -119,7 +123,6 @@ typedef struct _symtable_entry {
                                              should be created */
     unsigned ste_needs_classdict : 1; /* for class scopes, true if a closure
                                          over the class dict should be created */
-    unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */
     unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
     unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an
                                              enclosing class scope */
@@ -132,6 +135,7 @@ typedef struct _symtable_entry {
     int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
     _Py_SourceLocation ste_loc; /* source location of block */
     struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */
+    struct _symtable_entry *ste_parent; /* st entry for the enclosing block if this entry is a sub-scope, NULL otherwise */
     struct symtable *ste_table;
 } PySTEntryObject;
 
@@ -142,6 +146,7 @@ extern PyTypeObject PySTEntry_Type;
 extern long _PyST_GetSymbol(PySTEntryObject *, PyObject *);
 extern int _PyST_GetScope(PySTEntryObject *, PyObject *);
 extern int _PyST_IsFunctionLike(PySTEntryObject *);
+extern int _PyST_IsClassClosureName(PyObject *);
 
 extern struct symtable* _PySymtable_Build(
     struct _mod *mod,
@@ -172,7 +177,6 @@ _Py_IsPrivateName(PyObject *);
 #define DEF_ANNOT (2<<7)         /* this name is annotated */
 #define DEF_COMP_ITER (2<<8)     /* this name is a comprehension iteration variable */
 #define DEF_TYPE_PARAM (2<<9)    /* this name is a type parameter */
-#define DEF_COMP_CELL (2<<10)    /* this name is a cell in an inlined comprehension */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 

InternalDocs/README.md

@@ -23,6 +23,8 @@ Compiling Python Source Code
 
 - [Compiler Design](compiler.md)
 
+- [Inlined comprehensions](inlined_comprehensions.md)
+
 - [Changing Python's Grammar](changing_grammar.md)
 
 Runtime Objects

InternalDocs/compiler.md

@@ -354,6 +354,10 @@ AST node type).  Next, the AST tree is walked with the various code blocks that
 delineate the reach of a local variable as blocks are entered and exited using
 `symtable_enter_block()` and `symtable_exit_block()`, respectively.
 
+See [Inlined comprehensions](inlined_comprehensions.md) for how list, set,
+and dict comprehensions are represented as lexical subscopes of the
+enclosing unit.
+
 Once the symbol table is created, the `AST` is transformed by `compiler_codegen()`
 in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instructions.
 These are similar to bytecode, but in some cases they are more abstract, and are

InternalDocs/inlined_comprehensions.md

@@ -0,0 +1,132 @@
+Inlined comprehensions
+======================
+
+Since [PEP 709](https://peps.python.org/pep-0709/), list, set, and dict
+comprehensions are compiled into the enclosing compilation unit instead of
+creating a nested function and calling it. Generator expressions are not
+inlined; they still become their own code object.
+
+The resulting bytecode lives in the enclosing unit, but the comprehension
+still has its own locals: iteration variables must not leak into, or
+overwrite, names in the enclosing scope. The symbol table models that as a
+nested lexical scope; codegen then emits the comprehension inlined into its
+containing compilation unit.
+
+Which comprehensions are inlined
+--------------------------------
+
+`symtable_handle_comprehension()` in
+[`Python/symtable.c`](../Python/symtable.c) inlines a comprehension when it
+is not a generator expression and the current block cannot see class scope
+(`!ste_can_see_class_scope`). Annotation scopes that can see a class keep
+the historical nested-function compilation so class-local names are not
+treated as comprehension locals.
+
+The outermost iterator expression is always evaluated in the enclosing
+scope. The rest of the comprehension (targets, `if` clauses, inner
+generators, and the element/value expressions) is visited in the
+comprehension's own block.
+
+Symbol table
+------------
+
+An inlined comprehension gets an `InlinedComprehensionBlock` entry
+([`pycore_symtable.h`](../Include/internal/pycore_symtable.h)). That entry
+is a child of the enclosing block, with `ste_parent` pointing at the
+enclosing `PySTEntryObject`. It is a lexical subscope, not a compilation
+unit: there is no separate code object, `co_consts` entry, or compiler
+scope for it.
+
+Uses and bindings inside the comprehension are recorded on that child
+table. Because those loads are in the enclosing compilation unit, a
+`FREE` use is also marked `USE` on the parent table. For
+`def inner(): return [x for y in ()]`, both `inner.lookup("x")` and the
+inlined child's lookup are `FREE|USE`.
+
+### Analysis
+
+`analyze_block()` records this block's declarations, analyzes children,
+then classifies uses. `finalize_inlined_comprehension()` copies
+`USE` from an inlined child onto the parent before that second pass, and
+drops inlined-only frees so `analyze_cells()` does not promote those
+names to cells.
+
+A name that is `FREE` in the comprehension and bound in the parent is
+dropped from the parent's free set unless:
+
+* a real nested unit (function, lambda, or genexp) still needs it as a
+  cell, or
+* a sibling nested scope already marked it free.
+
+That keeps iteration variables as fast locals when they are only used by
+nested inlined comprehensions. A nested lambda that captures the name
+still forces a cell on the binding comprehension.
+
+Class-closure names (`__class__` and friends) loaded from an inlined
+comprehension do not require a class cell unless a nested function,
+lambda, or genexp captures them. Compile treats the inlined loads as
+implicit globals. `is_free_in_any_child()` walks through inlined children
+and only counts `FREE` on non-inlined descendants.
+
+Compiler
+--------
+
+Codegen stays in the enclosing compiler unit. Around the inlined region,
+`_PyCompile_EnterInlinedComprehensionScope()` /
+`_PyCompile_ExitInlinedComprehensionScope()` in
+[`Python/compile.c`](../Python/compile.c) swap `c->u->u_ste` so name
+lookup uses the comprehension's symbol table. The saved `u_ste` is
+restored on both success and error.
+
+`_PyCompile_ResolveNameop()` calls `compiler_resolve_inlined_free()`,
+which walks `ste_parent` while the current table is inlined and the name
+is `FREE` or missing (scope `0`). Missing names include loads synthesized
+by codegen, such as the implicit receiver for zero-argument `super()`.
+The walk stops at a class: nested scopes do not see class locals.
+Class-closure names that would otherwise be free through a class become
+`GLOBAL_IMPLICIT`.
+
+### Isolating iteration variables
+
+`codegen_push_inlined_comprehension_locals()` in
+[`Python/codegen.c`](../Python/codegen.c) isolates names bound in the
+comprehension:
+
+* `LOAD_FAST_AND_CLEAR` saves the enclosing value (possibly `NULL`) and
+  clears the slot.
+* `MAKE_CELL` runs if the name is a cell for this comprehension.
+* In module and class units the name is added to `u_fasthidden` so
+  assemble can set `CO_FAST_HIDDEN`.
+
+A `SETUP_FINALLY` / `COMPILE_FBLOCK_INLINED_COMPREHENSION` handler
+restores those slots if the comprehension raises, so an enclosing `except`
+or `finally` sees the original values. 
+
+Runtime
+-------
+
+An inlined comprehension cell can share a localsplus name with an
+enclosing free variable (for example `[lambda: x for x in x]` inside a
+nested function). `FrameLocalsProxy` keys, values, items, and `len`
+keep the first slot of each name so they agree with `getitem`.
+
+Source
+------
+
+* [`Python/symtable.c`](../Python/symtable.c):
+  `symtable_handle_comprehension()`, `analyze_block()`,
+  `finalize_inlined_comprehension()`, `is_free_in_any_child()`
+* [`Python/compile.c`](../Python/compile.c):
+  `compiler_resolve_inlined_free()`,
+  `_PyCompile_EnterInlinedComprehensionScope()`,
+  `compiler_cellvars()`
+* [`Python/codegen.c`](../Python/codegen.c):
+  `codegen_comprehension()`,
+  `push_inlined_comprehension_state()`,
+  `codegen_push_inlined_comprehension_locals()`
+* [`Include/internal/pycore_symtable.h`](../Include/internal/pycore_symtable.h):
+  `InlinedComprehensionBlock`
+* [`Include/internal/pycore_compile.h`](../Include/internal/pycore_compile.h):
+  `_PyCompile_InlinedComprehensionState`
+* [`Objects/frameobject.c`](../Objects/frameobject.c):
+  `FrameLocalsProxy` duplicate-name handling

Lib/symtable.py

@@ -7,7 +7,7 @@
     DEF_NONLOCAL, DEF_LOCAL,
     DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS,
     DEF_IMPORT, DEF_BOUND, DEF_ANNOT,
-    DEF_COMP_ITER, DEF_COMP_CELL,
+    DEF_COMP_ITER,
     SCOPE_OFF, SCOPE_MASK,
     FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL
 )
@@ -56,6 +56,7 @@ class SymbolTableType(StrEnum):
     TYPE_ALIAS = "type alias"
     TYPE_PARAMETERS = "type parameters"
     TYPE_VARIABLE = "type variable"
+    INLINED_COMPREHENSION = "inlined comprehension"
 
 
 class SymbolTable:
@@ -98,6 +99,8 @@ def get_type(self):
             return SymbolTableType.TYPE_PARAMETERS
         if self._table.type == _symtable.TYPE_TYPE_VARIABLE:
             return SymbolTableType.TYPE_VARIABLE
+        if self._table.type == _symtable.TYPE_INLINED_COMPREHENSION:
+            return SymbolTableType.INLINED_COMPREHENSION
         assert False, f"unexpected type: {self._table.type}"
 
     def get_id(self):
@@ -151,8 +154,10 @@ def lookup(self, name):
             flags = self._table.symbols[name]
             namespaces = self.__check_children(name)
             module_scope = (self._table.name == "top")
+            inlined = (self._table.type == _symtable.TYPE_INLINED_COMPREHENSION)
             sym = self._symbols[name] = Symbol(name, flags, namespaces,
-                                               module_scope=module_scope)
+                                               module_scope=module_scope,
+                                               inlined_comprehension=inlined)
         return sym
 
     def get_symbols(self):
@@ -246,12 +251,14 @@ class Class(SymbolTable):
 
 class Symbol:
 
-    def __init__(self, name, flags, namespaces=None, *, module_scope=False):
+    def __init__(self, name, flags, namespaces=None, *, module_scope=False,
+                 inlined_comprehension=False):
         self.__name = name
         self.__flags = flags
         self.__scope = _get_scope(flags)
         self.__namespaces = namespaces or ()
         self.__module_scope = module_scope
+        self.__inlined_comprehension = inlined_comprehension
 
     def __repr__(self):
         flags_str = '|'.join(self._flags_str())
@@ -345,7 +352,7 @@ def is_comp_iter(self):
     def is_comp_cell(self):
         """Return *True* if the symbol is a cell in an inlined comprehension.
         """
-        return bool(self.__flags & DEF_COMP_CELL)
+        return self.is_cell() and self.__inlined_comprehension
 
     def is_namespace(self):
         """Returns *True* if name binding introduces new namespace.

Lib/test/test_compiler_assemble.py

@@ -17,8 +17,9 @@ def complete_metadata(self, metadata, filename="myfile.py"):
             metadata.setdefault(key, key)
         for key in ['consts']:
             metadata.setdefault(key, [])
-        for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']:
+        for key in ['names', 'varnames', 'cellvars', 'freevars']:
             metadata.setdefault(key, {})
+        metadata.setdefault('fasthidden', None)
         for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']:
             metadata.setdefault(key, 0)
         metadata.setdefault('firstlineno', 1)