[3.14] gh-155525: Fix quadratic f-string tokenization (GH-156756) (#157829)

* [ 3.14 ] gh-155525: Avoid quadratic f-string tokenization

Adapt the original expression-span fix to the release tokenizer and retain interactive buffers while a formatted string is open.

Co-authored-by: gwosti <322952417+gwosti@users.noreply.github.com>

* gh-155525: Cover quadratic f-string tokenization regression (GH-156756)

* gh-155525: Avoid quadratic f-string tokenization

* fixup! gh-155525: Avoid quadratic f-string tokenization

---------
(cherry picked from commit c1df684)

Co-authored-by: gwosti <322952417+gwosti@users.noreply.github.com>
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>

---------

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
Co-authored-by: gwosti <322952417+gwosti@users.noreply.github.com>

Author: 3 people

Lib/test/test_fstring.py

@@ -14,11 +14,13 @@
 import re
 import types
 import decimal
+import subprocess
 import unittest
 import warnings
 from test import support
 from test.support.os_helper import temp_cwd
-from test.support.script_helper import assert_python_failure, assert_python_ok
+from test.support.script_helper import (
+    assert_python_failure, assert_python_ok, spawn_python)
 
 a_global = 'global variable'
 
@@ -822,6 +824,18 @@ def build_fstr(n, extra=''):
         s = "f'{1}' 'x' 'y'" * 1024
         self.assertEqual(eval(s), '1xy' * 1024)
 
+    @support.requires_resource('cpu')
+    def test_many_fstrings_in_module(self):
+        fields = ''.join(f'{{x{i}}}' for i in range(100))
+        source = ''.join(
+            f"value_{i} = f'{fields}'\n" for i in range(1_000)
+        )
+        namespace = {f'x{i}': str(i) for i in range(100)}
+        expected = ''.join(str(i) for i in range(100))
+        exec(source, namespace)
+        self.assertEqual(namespace['value_0'], expected)
+        self.assertEqual(namespace['value_999'], expected)
+
     def test_format_specifier_expressions(self):
         width = 10
         precision = 4
@@ -1338,6 +1352,9 @@ def test_not_equal(self):
         self.assertEqual(f'{3!=4:}', 'True')
         self.assertEqual(f'{3!=4!s}', 'True')
         self.assertEqual(f'{3!=4!s:.3}', 'Tru')
+        a = 3
+        b = 4
+        self.assertEqual(f'{a!=b=:>10}', 'a!=b=         1')
 
     def test_equal_equal(self):
         # Because an expression ending in = has special meaning,
@@ -1789,6 +1806,32 @@ def test_debug_in_file(self):
         self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'),
                          "3\n=3")
 
+    @support.requires_subprocess()
+    def test_expression_in_interactive_after_buffer_resize(self):
+        expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
+        source = (
+            f"result = f'''{{{expression}=}}'''\n"
+            "print(repr(result))\n"
+        )
+        with spawn_python('-i', '-q', stderr=subprocess.PIPE) as process:
+            stdout, stderr = process.communicate(
+                source.encode(), timeout=support.SHORT_TIMEOUT)
+        self.assertEqual(process.returncode, 0, stderr)
+        self.assertEqual(stdout.decode().strip(), repr(expression + "=1"))
+
+    def test_debug_in_file_after_buffer_resize(self):
+        expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
+        expected = expression + "=1"
+        with temp_cwd():
+            script = 'script.py'
+            source = (
+                f"result = f'''{{{expression}=}}'''\n"
+                f"assert result == {expected!r}\n"
+            )
+            with open(script, 'w') as f:
+                f.write(source)
+            assert_python_ok(script)
+
     def test_syntax_warning_infinite_recursion_in_file(self):
         with temp_cwd():
             script = 'script.py'

Lib/test/test_tstring.py

@@ -1,5 +1,9 @@
+import subprocess
 import unittest
 
+from test import support
+from test.support.os_helper import temp_cwd
+from test.support.script_helper import assert_python_ok, spawn_python
 from test.test_string._support import TStringBaseCase, fstring
 
 
@@ -79,6 +83,44 @@ def upper(self):
         )
         self.assertEqual(fstring(t), "Name: Bob, Age: 30")
 
+    @support.requires_subprocess()
+    def test_expression_in_interactive_after_buffer_resize(self):
+        expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
+        source = (
+            f"result = t'''{{{expression}}}'''\n"
+            "print(repr(result.interpolations[0].expression))\n"
+        )
+        with spawn_python('-i', '-q', stderr=subprocess.PIPE) as process:
+            stdout, stderr = process.communicate(
+                source.encode(), timeout=support.SHORT_TIMEOUT)
+        self.assertEqual(process.returncode, 0, stderr)
+        self.assertEqual(stdout.decode().strip(), repr(expression))
+
+    def test_interpolation_expression_in_file_after_buffer_resize(self):
+        expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)"
+        with temp_cwd():
+            script = 'script.py'
+            source = (
+                f"template = t'''{{{expression}}}'''\n"
+                "interpolation = template.interpolations[0]\n"
+                f"assert interpolation.expression == {expression!r}\n"
+            )
+            with open(script, 'w') as f:
+                f.write(source)
+            assert_python_ok(script)
+
+    @support.requires_resource('cpu')
+    def test_many_tstrings_in_module(self):
+        fields = ''.join(f'{{x{i}}}' for i in range(100))
+        source = ''.join(
+            f"value_{i} = t'{fields}'\n" for i in range(1_000)
+        )
+        namespace = {f'x{i}': str(i) for i in range(100)}
+        expected = ''.join(str(i) for i in range(100))
+        exec(source, namespace)
+        self.assertEqual(fstring(namespace['value_0']), expected)
+        self.assertEqual(fstring(namespace['value_999']), expected)
+
     def test_format_specifiers(self):
         # Test basic format specifiers
         value = 3.14159
@@ -88,6 +130,14 @@ def test_format_specifiers(self):
         )
         self.assertEqual(fstring(t), "Pi: 3.14")
 
+        a = 3
+        b = 4
+        t = t"{a!=b:>10}"
+        self.assertTStringEqual(
+            t, ("", ""), [(a != b, "a!=b", None, ">10")]
+        )
+        self.assertEqual(fstring(t), "         1")
+
     def test_conversions(self):
         # Test !s conversion (str)
         obj = object()

Misc/NEWS.d/next/Core_and_Builtins/2026-09-01-07-15-20.gh-issue-155525.A7kP2m.rst

@@ -0,0 +1,2 @@
+Fix quadratic-time tokenization of modules containing many f-strings or
+t-strings.

Parser/lexer/buffer.c

@@ -15,6 +15,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok)
         mode = &(tok->tok_mode_stack[index]);
         mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
         mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
+        mode->last_expr_start_offset = mode->last_expr_start == NULL
+            ? -1 : mode->last_expr_start - tok->buf;
     }
 }
 
@@ -29,6 +31,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok)
         mode = &(tok->tok_mode_stack[index]);
         mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
         mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
+        mode->last_expr_start = mode->last_expr_start_offset < 0
+            ? NULL : tok->buf + mode->last_expr_start_offset;
     }
 }
 

Parser/lexer/lexer.c

@@ -119,15 +119,19 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
         return 0;
     }
+    const char *expression = tok_mode->last_expr_start;
+    assert(expression != NULL);
+    assert(expression <= tok->start);
+    Py_ssize_t expression_size = tok->start - expression;
     PyObject *res = NULL;
 
     // Look for a # character outside of string literals
     int hash_detected = 0;
     int in_string = 0;
     char quote_char = 0;
 
-    for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
-        char ch = tok_mode->last_expr_buffer[i];
+    for (Py_ssize_t i = 0; i < expression_size; i++) {
+        char ch = expression[i];
 
         // Skip escaped characters
         if (ch == '\\') {
@@ -163,7 +167,7 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     // If we found a # character in the expression, we need to handle comments
     if (hash_detected) {
         // Allocate buffer for processed result
-        char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
+        char *result = (char *)PyMem_Malloc((expression_size + 1) * sizeof(char));
         if (!result) {
             return -1;
         }
@@ -174,16 +178,16 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
         quote_char = 0;    // Current string quote char
 
         // Process each character
-        while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-            char ch = tok_mode->last_expr_buffer[i];
+        while (i < expression_size) {
+            char ch = expression[i];
 
             // Copy escaped characters without interpreting the escaped
             // character as a quote or comment marker.
             if (ch == '\\') {
                 result[j++] = ch;
                 i++;
-                if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-                    result[j++] = tok_mode->last_expr_buffer[i];
+                if (i < expression_size) {
+                    result[j++] = expression[i];
                 }
             }
             // Handle string quotes
@@ -199,11 +203,11 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
             }
             // Skip comments
             else if (ch == '#' && !in_string) {
-                while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
-                       tok_mode->last_expr_buffer[i] != '\n') {
+                while (i < expression_size &&
+                       expression[i] != '\n') {
                     i++;
                 }
-                if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
+                if (i < expression_size) {
                     result[j++] = '\n';
                 }
             }
@@ -219,8 +223,8 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
         PyMem_Free(result);
     } else {
         res = PyUnicode_DecodeUTF8(
-            tok_mode->last_expr_buffer,
-            tok_mode->last_expr_size - tok_mode->last_expr_end,
+            expression,
+            expression_size,
             NULL
         );
     }
@@ -232,61 +236,6 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     return 0;
 }
 
-int
-_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
-{
-    assert(tok->cur != NULL);
-
-    Py_ssize_t size = strlen(tok->cur);
-    tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
-
-    switch (cur) {
-       case 0:
-            if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
-                return 1;
-            }
-            char *new_buffer = PyMem_Realloc(
-                tok_mode->last_expr_buffer,
-                tok_mode->last_expr_size + size
-            );
-            if (new_buffer == NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-                goto error;
-            }
-            tok_mode->last_expr_buffer = new_buffer;
-            strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
-            tok_mode->last_expr_size += size;
-            break;
-        case '{':
-            if (tok_mode->last_expr_buffer != NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-            }
-            tok_mode->last_expr_buffer = PyMem_Malloc(size);
-            if (tok_mode->last_expr_buffer == NULL) {
-                goto error;
-            }
-            tok_mode->last_expr_size = size;
-            tok_mode->last_expr_end = -1;
-            strncpy(tok_mode->last_expr_buffer, tok->cur, size);
-            break;
-        case '}':
-        case '!':
-            tok_mode->last_expr_end = strlen(tok->start);
-            break;
-        case ':':
-            if (tok_mode->last_expr_end == -1) {
-               tok_mode->last_expr_end = strlen(tok->start);
-            }
-            break;
-        default:
-            Py_UNREACHABLE();
-    }
-    return 1;
-error:
-    tok->done = E_NOMEM;
-    return 0;
-}
-
 static int
 lookahead(struct tok_state *tok, const char *test)
 {
@@ -1103,9 +1052,8 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
         the_current_tok->first_line = tok->lineno;
         the_current_tok->start_offset = -1;
         the_current_tok->multi_line_start_offset = -1;
-        the_current_tok->last_expr_buffer = NULL;
-        the_current_tok->last_expr_size = 0;
-        the_current_tok->last_expr_end = -1;
+        the_current_tok->last_expr_start = NULL;
+        the_current_tok->last_expr_start_offset = -1;
         the_current_tok->in_format_spec = 0;
         the_current_tok->in_debug = 0;
 
@@ -1270,9 +1218,6 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
          int cursor_in_format_with_debug =
              cursor == 1 && (current_tok->in_debug || in_format_spec);
          int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
-            return MAKE_TOKEN(ENDMARKER);
-        }
         if ((cursor_valid) && c != '{' && set_ftstring_expr(tok, token, c)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
@@ -1416,6 +1361,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
     if (start_char == '{') {
         int peek1 = tok_nextc(tok);
         tok_backup(tok, peek1);
+        if (peek1 != '{') {
+            current_tok->last_expr_start = tok->cur;
+        }
         tok_backup(tok, start_char);
         if (peek1 != '{') {
             current_tok->curly_bracket_expr_start_depth++;
@@ -1440,13 +1388,6 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
         }
     }
 
-    if (current_tok->last_expr_buffer != NULL) {
-        PyMem_Free(current_tok->last_expr_buffer);
-        current_tok->last_expr_buffer = NULL;
-        current_tok->last_expr_size = 0;
-        current_tok->last_expr_end = -1;
-    }
-
     p_start = tok->start;
     p_end = tok->cur;
     tok->tok_mode_stack_index--;
@@ -1531,12 +1472,10 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
         }
 
         if (c == '{') {
-            if (!_PyLexer_update_ftstring_expr(tok, c)) {
-                return MAKE_TOKEN(ENDMARKER);
-            }
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
+                current_tok->last_expr_start = tok->cur;
                 tok_backup(tok, c);
                 current_tok->curly_bracket_expr_start_depth++;
                 if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {

Parser/lexer/lexer.h

@@ -3,8 +3,6 @@
 
 #include "state.h"
 
-int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
 #endif