gh-103089: Colorize only the beginning of very long lines in IDLE (#157647)

Adding a tag to a Tk text line takes time proportional to the number
of tags already in the line, so a line with hundreds of thousands of
tokens took hours to colorize.  Now only the first 2000 characters of
a line are colorized, tag positions are relative to the line start,
and tags are added from the end to the start.

Author: serhiy-storchaka

Lib/idlelib/colorizer.py

@@ -8,6 +8,11 @@
 
 DEBUG = False
 
+# Adding a tag to a line takes time proportional to the number of tags
+# already in the line, so only the beginning of a line is colorized;
+# the rest is usually not visible anyway (gh-103089).
+MAX_COLORIZED_LINE = 2000
+
 
 def any(name, alternates):
     "Return a named group pattern matching list of alternates."
@@ -350,16 +355,39 @@ def _add_tags_in_section(self, chars, head):
         `chars` is a string with the text to parse and to which
         highlighting is to be applied.
 
-            `head` is the index in the text widget where the text is found.
+        `head` is the index in the text widget where the text is found.
         """
-        for m in self.prog.finditer(chars):
+        # Positions are relative to the start of the current line, so that
+        # Tk does not resolve them through the previous lines.
+        line = int(head.split('.')[0])
+        line_start = 0  # Offset of the current line in chars.
+        tags = []
+        pos = 0
+        while True:
+            m = self.prog.search(chars, pos)
+            if m is None:
+                break
             for name, matched_text in matched_named_groups(m):
                 a, b = m.span(name)
-                self._add_tag(a, b, head, name)
+                tags.append((a - line_start, b - line_start, head, name))
                 if matched_text in ("def", "class"):
                     if m1 := self.idprog.match(chars, b):
                         a, b = m1.span(1)
-                        self._add_tag(a, b, head, "DEFINITION")
+                        tags.append((a - line_start, b - line_start,
+                                     head, "DEFINITION"))
+            pos = m.end()
+            if '\n' in m[0]:
+                line += m[0].count('\n')
+                line_start = m.start() + m[0].rindex('\n') + 1
+                head = f"{line}.0"
+            elif pos - line_start >= MAX_COLORIZED_LINE:
+                # The rest of a long line is not colorized.
+                pos = chars.find('\n', pos)
+                if pos < 0:
+                    break
+        # Adding a tag is faster if there are no tags after it.
+        for args in reversed(tags):
+            self._add_tag(*args)
 
     def removecolors(self):
         "Remove all colorizing tags."

Lib/idlelib/idle_test/test_colorizer.py

@@ -573,6 +573,18 @@ def test_long_multiline_string(self):
             e"""
             ''')
         self._assert_highlighting(source, {'STRING': [('1.0', '5.4')]})
+        source = '"""a\nb""" + str\n'
+        self._assert_highlighting(source, {'STRING': [('1.0', '2.4')],
+                                           'BUILTIN': [('2.7', '2.10')]})
+
+    def test_long_line(self):
+        # gh-103089: only the first MAX_COLORIZED_LINE characters of a line
+        # are colorized.
+        n = colorizer.MAX_COLORIZED_LINE
+        source = f"pass\n{'x' * (n - 3)}'a', 'b'\n'c'\n"
+        self._assert_highlighting(source, {'KEYWORD': [('1.0', '1.4')],
+                                           'STRING': [(f'2.{n-3}', f'2.{n}'),
+                                                      ('3.0', '3.3')]})
 
     @run_in_tk_mainloop(delay=50)
     def test_incremental_editing(self):

Misc/NEWS.d/next/IDLE/2026-09-17-02-00-00.gh-issue-103089.longline.rst

@@ -0,0 +1,2 @@
+IDLE no longer hangs when opening a file with very long lines.
+Only the first 2,000 characters of a line are colorized.