Skip to content

Commit 2a96282

Browse files
pablogsalpablogsal
authored
gh-153569: consolidate tokenizer input, layout, and diagnostics (#157055)
* gh-153569: move tokenizer input state and relocation into the reader * gh-153569: borrow diagnostic lines through the source API * gh-153569: group indentation and logical-line state * gh-153569: remove unused tokenizer cursor and source lookup APIs * gh-153569: report tokenizer diagnostics without rewinding the scanner * gh-153569: use tokenizer views and remove obsolete API remnants
1 parent bee3031 commit 2a96282

32 files changed

Lines changed: 510 additions & 1072 deletions

‎Lib/test/test_capi/test_tokenizer.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,6 @@ def test_source(self):
1212
def test_source_discard(self):
1313
_testinternalcapi.test_tokenizer_source_discard()
1414

15-
def test_cursor(self):
16-
_testinternalcapi.test_tokenizer_cursor()
17-
1815

1916
if __name__ == "__main__":
2017
unittest.main()

‎Lib/test/test_codeop.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,17 @@ def test_valid(self, compiler):
113113
av("def f():\n pass\n#foo\n")
114114
av("@a.b.c\ndef f():\n pass\n")
115115

116+
@subTests('symbol', ('single', 'exec'))
117+
@subTests('prefix', ('', 'f', 't'))
118+
def test_incomplete_string_diagnostics(self, symbol, prefix):
119+
opening = f' á = {prefix}"""first\n'
120+
source = 'if True:\n' + opening + 'second'
121+
with self.assertRaises(_IncompleteInputError) as cm:
122+
Compile()(source, '<input>', symbol)
123+
text = opening + 'second' + ('\n' if symbol == 'exec' else '')
124+
self.assertEqual(cm.exception.args, (
125+
'incomplete input', ('<input>', 2, 9, text, 2, -1)))
126+
116127
@subTests('compiler', COMPILERS)
117128
def test_incomplete(self, compiler):
118129
ai = functools.partial(self.assertIncomplete, compiler=compiler)

‎Lib/test/test_repl.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,9 +185,8 @@ def read_until(marker, start=0):
185185

186186
@cpython_only
187187
def test_lexer_buffer_realloc_with_null_start(self):
188-
# gh-144759: NULL pointer arithmetic in the lexer when start and
189-
# multi_line_start are NULL (uninitialized in tok_mode_stack[0])
190-
# and the lexer buffer is reallocated while parsing long input.
188+
# gh-144759: NULL pointer arithmetic when the lexer buffer grows
189+
# while parsing long input.
191190
long_value = "a" * 2000
192191
user_input = dedent(f"""\
193192
x = f'{{{long_value!r}}}'

‎Lib/test/test_source_encoding.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import unittest
44
from test import support
55
from test.support import script_helper
6-
from test.support.os_helper import TESTFN, unlink, rmtree
7-
from test.support.import_helper import unload
6+
from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
7+
from test.support.import_helper import import_module, unload
88
import importlib
99
import os
1010
import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
8383
self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
8484

8585
def test_invalid_utf8_offset_after_non_ascii(self):
86+
for name in ('é', 'éé', '𝒜'):
87+
with self.subTest(name=name):
88+
source = ('x = ' + name).encode() + b'\xff\n'
89+
with self.assertRaises(SyntaxError) as caught:
90+
compile(source, '<test>', 'exec')
91+
error = caught.exception
92+
self.assertEqual(
93+
(error.lineno, error.offset, error.end_lineno, error.end_offset),
94+
(1, 5 + len(name), 1, 5 + len(name)),
95+
)
96+
97+
@support.cpython_only
98+
def test_invalid_utf8_file_offset_after_non_ascii(self):
99+
_testcapi = import_module('_testcapi')
100+
self.addCleanup(unlink, TESTFN_ASCII)
101+
with open(TESTFN_ASCII, 'wb') as f:
102+
f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
86103
with self.assertRaises(SyntaxError) as caught:
87-
compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
104+
_testcapi.run_file(
105+
os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
88106
error = caught.exception
89107
self.assertEqual(
90108
(error.lineno, error.offset, error.end_lineno, error.end_offset),
91-
(1, 6, 1, 6),
109+
(2, 7, 2, 7),
92110
)
93111

94112
def test_long_bom_conflict_message_is_not_truncated(self):

‎Lib/test/test_tstring.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,8 @@ def test_nested_templates(self):
323323

324324
def test_syntax_errors(self):
325325
for case, err in (
326+
('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"),
327+
('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"),
326328
("t'", "unterminated t-string literal"),
327329
("t'''", "unterminated triple-quoted t-string literal"),
328330
("t''''", "unterminated triple-quoted t-string literal"),

‎Makefile.pre.in‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -395,10 +395,10 @@ PEGEN_OBJS= \
395395

396396
TOKENIZER_OBJS= \
397397
Parser/lexer/lexer.o \
398+
Parser/lexer/layout.o \
398399
Parser/lexer/number.o \
399400
Parser/lexer/state.o \
400401
Parser/lexer/string.o \
401-
Parser/tokenizer/cursor.o \
402402
Parser/tokenizer/decoder.o \
403403
Parser/tokenizer/api.o \
404404
Parser/tokenizer/reader.o \
@@ -411,10 +411,8 @@ PEGEN_HEADERS= \
411411
$(srcdir)/Parser/string_parser.h
412412

413413
TOKENIZER_HEADERS= \
414-
Parser/lexer/lexer.h \
415414
Parser/lexer/lexer_internal.h \
416415
Parser/lexer/state.h \
417-
Parser/tokenizer/cursor.h \
418416
Parser/tokenizer/reader.h \
419417
Parser/tokenizer/reader_internal.h \
420418
Parser/tokenizer/source.h \
@@ -3471,7 +3469,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
34713469
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
34723470
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
34733471
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
3474-
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
3472+
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
34753473
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
34763474
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
34773475

0 commit comments

Comments
 (0)