diff --git a/CHANGELOG.md b/CHANGELOG.md index e574b3cd..7bb22b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +* 👌 Expose mapped token types for fenced code and HTML blocks, with non-optional source maps, in [#430](https://github.com/executablebooks/markdown-it-py/issues/430). * ✨ Add `--enable-tables` to the CLI for file, standard input and interactive parsing in [#422](https://github.com/executablebooks/markdown-it-py/pull/422) * 🐛 Fix CLI interactive mode joining input lines with an extra newline, which split every line into its own paragraph and broke hard line breaks, in [#172](https://github.com/executablebooks/markdown-it-py/issues/172) * 📚 Document the Python renderer constructor contract. diff --git a/docs/using.md b/docs/using.md index aae3e858..54fa661a 100644 --- a/docs/using.md +++ b/docs/using.md @@ -43,6 +43,21 @@ for token in md.parse("some *text*"): print() ``` +The built-in fenced-code and HTML-block rules emit `MappedToken` instances. +Unlike a general `Token`, a `MappedToken` has a non-optional source line map: + +```python +from markdown_it.token import MappedToken + +for token in MarkdownIt().parse("```python\npass\n```\n"): + if isinstance(token, MappedToken) and token.type == "fence": + start_line, end_line = token.map +``` + +Plugins can create these tokens using `StateBlock.push_mapped(..., map=[start, end])`. +Checking only `token.type` does not narrow its map type: plugins can emit +arbitrary token types, and a general `Token` may have no map. + ## The Parser +++ diff --git a/markdown_it/rules_block/fence.py b/markdown_it/rules_block/fence.py index 621924b5..f2d28969 100644 --- a/markdown_it/rules_block/fence.py +++ b/markdown_it/rules_block/fence.py @@ -131,11 +131,10 @@ def _fence_rule( state.line = nextLine + (1 if haveEndMarker else 0) - token = state.push(token_type, "code", 0) + token = state.push_mapped(token_type, "code", 0, map=[startLine, state.line]) token.info = params token.content = state.getLines(startLine + 1, nextLine, length, True) token.markup = markup - token.map = [startLine, state.line] return True diff --git a/markdown_it/rules_block/html_block.py b/markdown_it/rules_block/html_block.py index fe7e464e..9f280771 100644 --- a/markdown_it/rules_block/html_block.py +++ b/markdown_it/rules_block/html_block.py @@ -86,8 +86,7 @@ def html_block(state: StateBlock, startLine: int, endLine: int, silent: bool) -> state.line = nextLine - token = state.push("html_block", "", 0) - token.map = [startLine, nextLine] + token = state.push_mapped("html_block", "", 0, map=[startLine, nextLine]) token.content = state.getLines(startLine, nextLine, state.blkIndent, True) return True diff --git a/markdown_it/rules_block/state_block.py b/markdown_it/rules_block/state_block.py index 445ad265..ec8ef66b 100644 --- a/markdown_it/rules_block/state_block.py +++ b/markdown_it/rules_block/state_block.py @@ -4,7 +4,7 @@ from ..common.utils import isStrSpace from ..ruler import StateBase -from ..token import Token +from ..token import MappedToken, Token from ..utils import EnvType if TYPE_CHECKING: @@ -128,6 +128,20 @@ def push(self, ttype: str, tag: str, nesting: Literal[-1, 0, 1]) -> Token: self.tokens.append(token) return token + def push_mapped( + self, ttype: str, tag: str, nesting: Literal[-1, 0, 1], *, map: list[int] + ) -> MappedToken: + """Push a block token whose source map is known.""" + token = MappedToken(ttype, tag, nesting, map=map) + token.block = True + if nesting < 0: + self.level -= 1 # closing tag + token.level = self.level + if nesting > 0: + self.level += 1 # opening tag + self.tokens.append(token) + return token + def isEmpty(self, line: int) -> bool: """.""" return (self.bMarks[line] + self.tShift[line]) >= self.eMarks[line] diff --git a/markdown_it/token.py b/markdown_it/token.py index 309bb96f..e9243f0e 100644 --- a/markdown_it/token.py +++ b/markdown_it/token.py @@ -176,3 +176,40 @@ def from_dict(cls, dct: MutableMapping[str, Any]) -> Token: if token.children: token.children = [cls.from_dict(c) for c in token.children] # type: ignore[arg-type] return token + + +@dc.dataclass(slots=True, eq=False) +class MappedToken(Token): + """A token with source line information available at construction.""" + + map: list[int] = dc.field(kw_only=True) + + def __post_init__(self) -> None: + Token.__post_init__(self) + if self.map is None: + raise TypeError("MappedToken requires a source map") + + def __eq__(self, other: object) -> bool: + """Retain value equality with existing ``Token`` instances.""" + if not isinstance(other, Token): + return NotImplemented + return all( + getattr(self, field.name) == getattr(other, field.name) + for field in dc.fields(Token) + ) + + def copy(self, **changes: Any) -> MappedToken: + """Return a shallow copy that retains the mapped token type.""" + return dc.replace(self, **changes) + + @classmethod + def from_dict(cls, dct: MutableMapping[str, Any]) -> MappedToken: + """Restore a mapped token and its potentially unmapped children.""" + children = dct.get("children") + if isinstance(children, list): + dct = dict(dct) + dct["children"] = [ + Token.from_dict(child) if isinstance(child, MutableMapping) else child + for child in children + ] + return cls(**dct) diff --git a/tests/test_api/test_make_fence_rule.py b/tests/test_api/test_make_fence_rule.py index 7160cc54..e4c4aa7e 100644 --- a/tests/test_api/test_make_fence_rule.py +++ b/tests/test_api/test_make_fence_rule.py @@ -4,6 +4,7 @@ from markdown_it import MarkdownIt from markdown_it.rules_block.fence import make_fence_rule +from markdown_it.token import MappedToken def _make_colon_fence_md() -> MarkdownIt: @@ -30,6 +31,7 @@ def test_basic(self): md = _make_colon_fence_md() tokens = md.parse(":::\nfoo\n:::\n") assert len(tokens) == 1 + assert isinstance(tokens[0], MappedToken) assert tokens[0].type == "colon_fence" assert tokens[0].content == "foo\n" assert tokens[0].markup == ":::" diff --git a/tests/test_api/test_token.py b/tests/test_api/test_token.py index 44035981..8b8b6fc3 100644 --- a/tests/test_api/test_token.py +++ b/tests/test_api/test_token.py @@ -1,6 +1,20 @@ +from typing import TYPE_CHECKING import warnings -from markdown_it.token import Token +import pytest + +from markdown_it import MarkdownIt +from markdown_it.rules_block.state_block import StateBlock +from markdown_it.token import MappedToken, Token + +if TYPE_CHECKING: + from typing_extensions import assert_type + + def check_mapped_token_type(source: str) -> None: + for token in MarkdownIt().parse(source): + if isinstance(token, MappedToken): + assert_type(token.map, list[int]) + assert_type(token.copy(), MappedToken) def test_token(): @@ -36,3 +50,55 @@ def test_token(): def test_serialization(): token = Token("name", "tag", 0, children=[Token("other", "tag2", 0)]) assert token == Token.from_dict(token.as_dict()) + + +@pytest.mark.parametrize( + ("source", "token_type", "source_map"), + [ + ("```python\npass\n```\n", "fence", [0, 3]), + ("\n", "html_block", [0, 1]), + ], +) +def test_builtin_mapped_tokens(source, token_type, source_map): + token = MarkdownIt().parse(source)[0] + assert isinstance(token, MappedToken) + assert token.type == token_type + assert token.map == source_map + assert token == Token( + token_type, + token.tag, + token.nesting, + map=source_map, + content=token.content, + markup=token.markup, + info=token.info, + block=True, + ) + + +def test_mapped_token_requires_map(): + with pytest.raises(TypeError): + MappedToken.from_dict({"type": "fence", "tag": "code", "nesting": 0}) + with pytest.raises(TypeError, match="requires a source map"): + MappedToken.from_dict( + {"type": "fence", "tag": "code", "nesting": 0, "map": None} + ) + + +def test_mapped_token_serialization_with_unmapped_child(): + token = MappedToken("custom", "", 0, map=[0, 1], children=[Token("text", "", 0)]) + assert MappedToken.from_dict(token.as_dict()) == token + assert Token.from_dict(token.as_dict()) == token + copied = token.copy() + assert isinstance(copied, MappedToken) + assert copied == token + assert copied is not token + + +def test_push_mapped_preserves_block_nesting(): + tokens: list[Token] = [] + state = StateBlock("", MarkdownIt(), {}, tokens) + opening = state.push_mapped("custom_open", "div", 1, map=[0, 1]) + closing = state.push_mapped("custom_close", "div", -1, map=[0, 1]) + assert opening.level == closing.level == state.level == 0 + assert tokens == [opening, closing]