👌 Add mapped token subtype for fence and HTML blocks

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+* 👌 Expose mapped token types for fenced code and HTML blocks, with non-optional source maps, in [#430](https://github.com/executablebooks/markdown-it-py/issues/430).
 * ✨ Add `--enable-tables` to the CLI for file, standard input and interactive parsing in [#422](https://github.com/executablebooks/markdown-it-py/pull/422)
 * 🐛 Fix CLI interactive mode joining input lines with an extra newline, which split every line into its own paragraph and broke hard line breaks, in [#172](https://github.com/executablebooks/markdown-it-py/issues/172)
 * 📚 Document the Python renderer constructor contract.

docs/using.md

@@ -43,6 +43,21 @@ for token in md.parse("some *text*"):
     print()
 ```
 
+The built-in fenced-code and HTML-block rules emit `MappedToken` instances.
+Unlike a general `Token`, a `MappedToken` has a non-optional source line map:
+
+```python
+from markdown_it.token import MappedToken
+
+for token in MarkdownIt().parse("```python\npass\n```\n"):
+    if isinstance(token, MappedToken) and token.type == "fence":
+        start_line, end_line = token.map
+```
+
+Plugins can create these tokens using `StateBlock.push_mapped(..., map=[start, end])`.
+Checking only `token.type` does not narrow its map type: plugins can emit
+arbitrary token types, and a general `Token` may have no map.
+
 ## The Parser
 
 +++

markdown_it/rules_block/fence.py

@@ -131,11 +131,10 @@ def _fence_rule(
 
         state.line = nextLine + (1 if haveEndMarker else 0)
 
-        token = state.push(token_type, "code", 0)
+        token = state.push_mapped(token_type, "code", 0, map=[startLine, state.line])
         token.info = params
         token.content = state.getLines(startLine + 1, nextLine, length, True)
         token.markup = markup
-        token.map = [startLine, state.line]
 
         return True
 

markdown_it/rules_block/html_block.py

@@ -86,8 +86,7 @@ def html_block(state: StateBlock, startLine: int, endLine: int, silent: bool) ->
 
     state.line = nextLine
 
-    token = state.push("html_block", "", 0)
-    token.map = [startLine, nextLine]
+    token = state.push_mapped("html_block", "", 0, map=[startLine, nextLine])
     token.content = state.getLines(startLine, nextLine, state.blkIndent, True)
 
     return True

markdown_it/rules_block/state_block.py

@@ -4,7 +4,7 @@
 
 from ..common.utils import isStrSpace
 from ..ruler import StateBase
-from ..token import Token
+from ..token import MappedToken, Token
 from ..utils import EnvType
 
 if TYPE_CHECKING:
@@ -128,6 +128,20 @@ def push(self, ttype: str, tag: str, nesting: Literal[-1, 0, 1]) -> Token:
         self.tokens.append(token)
         return token
 
+    def push_mapped(
+        self, ttype: str, tag: str, nesting: Literal[-1, 0, 1], *, map: list[int]
+    ) -> MappedToken:
+        """Push a block token whose source map is known."""
+        token = MappedToken(ttype, tag, nesting, map=map)
+        token.block = True
+        if nesting < 0:
+            self.level -= 1  # closing tag
+        token.level = self.level
+        if nesting > 0:
+            self.level += 1  # opening tag
+        self.tokens.append(token)
+        return token
+
     def isEmpty(self, line: int) -> bool:
         """."""
         return (self.bMarks[line] + self.tShift[line]) >= self.eMarks[line]

markdown_it/token.py

@@ -176,3 +176,40 @@ def from_dict(cls, dct: MutableMapping[str, Any]) -> Token:
         if token.children:
             token.children = [cls.from_dict(c) for c in token.children]  # type: ignore[arg-type]
         return token
+
+
+@dc.dataclass(slots=True, eq=False)
+class MappedToken(Token):
+    """A token with source line information available at construction."""
+
+    map: list[int] = dc.field(kw_only=True)
+
+    def __post_init__(self) -> None:
+        Token.__post_init__(self)
+        if self.map is None:
+            raise TypeError("MappedToken requires a source map")
+
+    def __eq__(self, other: object) -> bool:
+        """Retain value equality with existing ``Token`` instances."""
+        if not isinstance(other, Token):
+            return NotImplemented
+        return all(
+            getattr(self, field.name) == getattr(other, field.name)
+            for field in dc.fields(Token)
+        )
+
+    def copy(self, **changes: Any) -> MappedToken:
+        """Return a shallow copy that retains the mapped token type."""
+        return dc.replace(self, **changes)
+
+    @classmethod
+    def from_dict(cls, dct: MutableMapping[str, Any]) -> MappedToken:
+        """Restore a mapped token and its potentially unmapped children."""
+        children = dct.get("children")
+        if isinstance(children, list):
+            dct = dict(dct)
+            dct["children"] = [
+                Token.from_dict(child) if isinstance(child, MutableMapping) else child
+                for child in children
+            ]
+        return cls(**dct)

tests/test_api/test_make_fence_rule.py

@@ -4,6 +4,7 @@
 
 from markdown_it import MarkdownIt
 from markdown_it.rules_block.fence import make_fence_rule
+from markdown_it.token import MappedToken
 
 
 def _make_colon_fence_md() -> MarkdownIt:
@@ -30,6 +31,7 @@ def test_basic(self):
         md = _make_colon_fence_md()
         tokens = md.parse(":::\nfoo\n:::\n")
         assert len(tokens) == 1
+        assert isinstance(tokens[0], MappedToken)
         assert tokens[0].type == "colon_fence"
         assert tokens[0].content == "foo\n"
         assert tokens[0].markup == ":::"

tests/test_api/test_token.py

@@ -1,6 +1,20 @@
+from typing import TYPE_CHECKING
 import warnings
 
-from markdown_it.token import Token
+import pytest
+
+from markdown_it import MarkdownIt
+from markdown_it.rules_block.state_block import StateBlock
+from markdown_it.token import MappedToken, Token
+
+if TYPE_CHECKING:
+    from typing_extensions import assert_type
+
+    def check_mapped_token_type(source: str) -> None:
+        for token in MarkdownIt().parse(source):
+            if isinstance(token, MappedToken):
+                assert_type(token.map, list[int])
+                assert_type(token.copy(), MappedToken)
 
 
 def test_token():
@@ -36,3 +50,55 @@ def test_token():
 def test_serialization():
     token = Token("name", "tag", 0, children=[Token("other", "tag2", 0)])
     assert token == Token.from_dict(token.as_dict())
+
+
+@pytest.mark.parametrize(
+    ("source", "token_type", "source_map"),
+    [
+        ("```python\npass\n```\n", "fence", [0, 3]),
+        ("<!-- comment -->\n", "html_block", [0, 1]),
+    ],
+)
+def test_builtin_mapped_tokens(source, token_type, source_map):
+    token = MarkdownIt().parse(source)[0]
+    assert isinstance(token, MappedToken)
+    assert token.type == token_type
+    assert token.map == source_map
+    assert token == Token(
+        token_type,
+        token.tag,
+        token.nesting,
+        map=source_map,
+        content=token.content,
+        markup=token.markup,
+        info=token.info,
+        block=True,
+    )
+
+
+def test_mapped_token_requires_map():
+    with pytest.raises(TypeError):
+        MappedToken.from_dict({"type": "fence", "tag": "code", "nesting": 0})
+    with pytest.raises(TypeError, match="requires a source map"):
+        MappedToken.from_dict(
+            {"type": "fence", "tag": "code", "nesting": 0, "map": None}
+        )
+
+
+def test_mapped_token_serialization_with_unmapped_child():
+    token = MappedToken("custom", "", 0, map=[0, 1], children=[Token("text", "", 0)])
+    assert MappedToken.from_dict(token.as_dict()) == token
+    assert Token.from_dict(token.as_dict()) == token
+    copied = token.copy()
+    assert isinstance(copied, MappedToken)
+    assert copied == token
+    assert copied is not token
+
+
+def test_push_mapped_preserves_block_nesting():
+    tokens: list[Token] = []
+    state = StateBlock("", MarkdownIt(), {}, tokens)
+    opening = state.push_mapped("custom_open", "div", 1, map=[0, 1])
+    closing = state.push_mapped("custom_close", "div", -1, map=[0, 1])
+    assert opening.level == closing.level == state.level == 0
+    assert tokens == [opening, closing]