From 4305d2cff8ce50ab23c945c77bac5cfd64c4a059 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Jul 2026 23:06:07 +0000 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20document-relative=20`?= =?UTF-8?q?content=5Foffset`=20for=20directives?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Directives rendered by MyST now receive a document-relative `content_offset` (the 0-based absolute line of the first content line, == 1-based first content line minus 1), and `content.items` carrying each line's 0-based absolute document line -- exactly the convention docutils' own rST parser provides. Previously `content_offset` was the body-relative `body_offset` (0, 1, 2... from the fence) and items were relative 0, 1, 2..., so extensions computing `content_offset + N` (e.g. sphinx-jinja2) mis-attributed reporter warnings. This is coupled with `MockState.nested_parse`, whose `input_offset` is now an absolute document line (dropping the `self._lineno +` shift). For the standard `nested_parse(self.content, self.content_offset, node)` idiom the rendered line is `position + body_offset` either way, so every nested node line is byte-identical -- no rendering fixtures change. Free line-attribution fixes made correct while here: - `{epigraph}` attribution `.line` was one too small (latent off-by-one); it now equals the attribution's true document line, matching docutils' `parse_attribution` (`lineno = 1 + line_offset`). - `{parsed-literal}` `.line` (docutils sets `content_offset + 1`) is now the true absolute content line instead of a hardcoded 1. Internal docs directives (`_docs.py`) that passed a literal `0` to `nested_parse` now pass `self.content_offset`, preserving their render-at-the-directive placement under the new absolute semantics. Behaviour note for extension authors: `MockState.nested_parse` now interprets `input_offset` as an absolute document line (docutils-consistent); callers that passed a literal directive-relative offset should pass an absolute one (the common `self.content_offset` idiom is unaffected). `DirectiveParsingResult.body_offset` docstring clarified (relative to the line after the opening line; `0` = next line, `-1` = merged); its semantics are unchanged. Tests: new `tests/test_directive_line_mapping.py` pins the contract across fence shapes (bare, merged, options, YAML, nested, list-embedded, colon, argument) at multiple document positions, against docutils rST ground truth, plus the eval-rst pin, the epigraph/parsed-literal fixes, nested-line invariance, the `reporter.warning(line=content_offset + N)` idiom, and a `{role}` smoke test. The existing `test_directive_block_text_rst_parity` asserted the old body-relative `content_offset` (3); updated to the document-relative value (4) -- `body line` is on document line 5. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- myst_parser/_docs.py | 6 +- myst_parser/mdit_to_docutils/base.py | 22 +- myst_parser/mocking.py | 13 +- myst_parser/parsers/directives.py | 9 +- tests/test_directive_line_mapping.py | 379 +++++++++++++++++++++++++++ tests/test_docutils.py | 6 +- 6 files changed, 419 insertions(+), 16 deletions(-) create mode 100644 tests/test_directive_line_mapping.py diff --git a/myst_parser/_docs.py b/myst_parser/_docs.py index 95c8314e..cb8601ee 100644 --- a/myst_parser/_docs.py +++ b/myst_parser/_docs.py @@ -161,7 +161,7 @@ def run(self): text.append("```````") node = nodes.Element() - self.state.nested_parse(text, 0, node) + self.state.nested_parse(text, self.content_offset, node) return node.children @@ -219,7 +219,7 @@ def run(self): for key, func in klass.option_spec.items(): text += f" {key} | {convert_opt(name, func)}\n" node = nodes.Element() - self.state.nested_parse(text.splitlines(), 0, node) + self.state.nested_parse(text.splitlines(), self.content_offset, node) return node.children @@ -276,7 +276,7 @@ def run(self): ] text = [f"- `myst.{name}`: {' '.join(doc)}" for name, doc in warning_names] node = nodes.Element() - self.state.nested_parse(text, 0, node) + self.state.nested_parse(text, self.content_offset, node) return node.children diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 68cfae72..9465b684 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1897,6 +1897,15 @@ def run_directive( lineno=position, ) else: + source = self.document["source"] + # ``position`` is the 1-based document line of the opening fence, and + # ``parsed.body_offset`` is relative to the line after it, so their sum + # is the 0-based document line of the first content line, i.e. the + # ``content_offset`` docutils' rST parser provides (== 1-based first + # content line minus 1). The per-line ``items`` likewise carry the + # 0-based *absolute* document line of each content line, matching + # docutils rather than the historical 0, 1, 2, ... relative offsets. + content_offset = position + parsed.body_offset state_machine = MockStateMachine(self, position) state = MockState(self, state_machine, position) directive_instance = directive_class( @@ -1906,11 +1915,18 @@ def run_directive( # a dictionary mapping option names to values options=parsed.options, # the directive content line by line - content=StringList(parsed.body, self.document["source"]), + content=StringList( + parsed.body, + source, + items=[ + (source, content_offset + i) for i in range(len(parsed.body)) + ], + ), # the absolute line number of the first line of the directive lineno=position, - # the line offset of the first line of the content - content_offset=parsed.body_offset, + # the 0-based document line of the first content line + # (docutils convention, document-relative) + content_offset=content_offset, # a string containing the entire directive block_text=content if block_text is None else block_text, state=state, diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index b65888e7..9d419b38 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -161,8 +161,9 @@ def nested_parse( """Perform a nested parse of the input block, with ``node`` as the parent. :param block: The block of lines to parse. - :param input_offset: The offset of the first line of block, - to the starting line of the state (i.e. directive). + :param input_offset: The 0-based absolute line offset of the first line of + ``block`` within the document (docutils' convention, i.e. what a + directive's ``content_offset`` provides). :param node: The parent node to attach the parsed content to. :param match_titles: Whether to to allow the parsing of headings (normally this is false, @@ -172,7 +173,7 @@ def nested_parse( with self._renderer.current_node_context(node): self._renderer.nested_render_text( "\n".join(block), - self._lineno + input_offset, + input_offset, temp_root_node=node if match_titles else None, ) self.state_machine.match_titles = sm_match_titles @@ -246,7 +247,11 @@ def block_quote(self, lines: list[str], line_offset: int) -> list[nodes.Element] # parse attribution if attribution_lines: attribution_text = "\n".join(attribution_lines) - lineno = self._lineno + line_offset + (attribution_line_offset or 0) + # ``line_offset`` is the 0-based absolute document line of the first + # block-quote line, so the attribution's 1-based document line is + # ``line_offset + attribution_line_offset + 1`` (mirrors docutils' + # ``parse_attribution``: ``lineno = 1 + line_offset``). + lineno = line_offset + (attribution_line_offset or 0) + 1 textnodes, messages = self.inline_text(attribution_text, lineno) attribution = nodes.attribution(attribution_text, "", *textnodes) ( diff --git a/myst_parser/parsers/directives.py b/myst_parser/parsers/directives.py index 61c4d524..f40106b8 100644 --- a/myst_parser/parsers/directives.py +++ b/myst_parser/parsers/directives.py @@ -75,11 +75,12 @@ class DirectiveParsingResult: body: list[str] """The lines of body content""" body_offset: int - """The number of lines from the directive's opening line - to the first line of the body content. + """The offset of the first body line, relative to the line + directly after the directive's opening line. - This is ``-1`` when body content starts on the opening line itself - (only possible for directives that take no arguments). + So ``0`` means the body content starts on the line immediately following + the opening line, and ``-1`` means it starts (merged) on the opening line + itself (only possible for directives that take no arguments). """ warnings: list[ParseWarnings] """List of non-fatal errors encountered during parsing. diff --git a/tests/test_directive_line_mapping.py b/tests/test_directive_line_mapping.py new file mode 100644 index 00000000..dda5ca98 --- /dev/null +++ b/tests/test_directive_line_mapping.py @@ -0,0 +1,379 @@ +"""Tests for document-relative directive line mapping (docutils convention). + +A directive at any document position must receive a ``content_offset`` such +that ``content_offset + N`` maps its Nth (1-based) content line to the true +1-based document line. Equivalently, ``content_offset`` is the 0-based +*absolute* document line of the first content line (``(1-based line) - 1``), +and each entry of ``content.items`` carries the 0-based absolute document line +of its line. This mirrors what docutils' own rST parser provides, so that +extensions computing ``content_offset + N`` (e.g. sphinx-jinja2) report the +correct line under MyST too. +""" + +import contextlib +import io + +import pytest +from docutils import nodes +from docutils.core import publish_doctree +from docutils.parsers.rst import Directive +from docutils.parsers.rst import directives as rst_directives + +from myst_parser.parsers.docutils_ import Parser + +# module-level sink the capture directives append to (cleared per run) +_CAPTURED: list[dict] = [] + + +class _Capture(Directive): + """Record a directive's line-mapping data, then nested-parse its content.""" + + has_content = True + option_spec = { + "opt": rst_directives.unchanged, + "class": rst_directives.class_option, + "name": rst_directives.unchanged, + } + + def run(self): + _CAPTURED.append( + { + "lineno": self.lineno, + "content_offset": self.content_offset, + "content": list(self.content), + "items": list(self.content.items), + } + ) + # a standard nested parse, so nested node lines can be observed too + node = nodes.Element() + self.state.nested_parse(self.content, self.content_offset, node) + return list(node.children) + + +class _CaptureArg(_Capture): + """Capture variant that takes a single (whitespace) argument.""" + + optional_arguments = 1 + final_argument_whitespace = True + + +@contextlib.contextmanager +def _registered(**named_directives): + """Register directives for the duration of the block, then remove them.""" + for name, cls in named_directives.items(): + rst_directives.register_directive(name, cls) + try: + yield + finally: + for name in named_directives: + rst_directives._directives.pop(name, None) + + +def _run_myst(source, *, extensions=None): + """Parse ``source`` as MyST and return (captured, doctree).""" + _CAPTURED.clear() + overrides = {"warning_stream": io.StringIO()} + if extensions: + overrides["myst_enable_extensions"] = extensions + with _registered(cap=_Capture, caparg=_CaptureArg): + doctree = publish_doctree(source, parser=Parser(), settings_overrides=overrides) + return list(_CAPTURED), doctree + + +def _run_rst(source): + """Parse ``source`` with the plain docutils rST parser (ground truth).""" + _CAPTURED.clear() + with _registered(cap=_Capture, caparg=_CaptureArg): + doctree = publish_doctree( + source, settings_overrides={"warning_stream": io.StringIO()} + ) + return list(_CAPTURED), doctree + + +def _publish(source, *, extensions=None): + """Parse ``source`` as MyST and return (doctree, stripped warnings).""" + overrides = {"warning_stream": io.StringIO()} + if extensions: + overrides["myst_enable_extensions"] = extensions + doctree = publish_doctree(source, parser=Parser(), settings_overrides=overrides) + return doctree, overrides["warning_stream"].getvalue().strip() + + +def _assert_docutils_convention(cap): + """Assert a capture obeys the docutils ``content_offset``/``items`` contract.""" + length = len(cap["content"]) + if length: + source = cap["items"][0][0] + # items are one-per-line, each the 0-based absolute document line + assert cap["items"] == [ + (source, cap["content_offset"] + i) for i in range(length) + ] + # content_offset is the 0-based absolute line of the first content line + assert cap["content_offset"] == cap["items"][0][1] + else: + assert cap["items"] == [] + + +# Each case: (source, extensions, expected) where ``expected`` is a list of +# (lineno, first_content_line, content) per captured directive, in run order. +# ``first_content_line`` is the true 1-based document line of the first content +# line, so the expected ``content_offset`` is ``first_content_line - 1``. +_MYST_SHAPES = [ + pytest.param( + # 1:```{cap} 2:hello world 3:``` + "```{cap}\nhello world\n```\n", + None, + [(1, 2, ["hello world"])], + id="bare-pos1", + ), + pytest.param( + # 1:# Title 2: 3:para 4: 5:```{cap} 6:hello world 7:``` + "# Title\n\npara\n\n```{cap}\nhello world\n```\n", + None, + [(5, 6, ["hello world"])], + id="bare-pos2", + ), + pytest.param( + # 1:```{cap} hello world 2:``` + "```{cap} hello world\n```\n", + None, + [(1, 1, ["hello world"])], + id="merged-first-line", + ), + pytest.param( + # 1:```{caparg} myarg 2:content here 3:``` + "```{caparg} myarg\ncontent here\n```\n", + None, + [(1, 2, ["content here"])], + id="argument", + ), + pytest.param( + # 1:```{cap} 2::opt: xx 3::class: cc 4:content here 5:``` + "```{cap}\n:opt: xx\n:class: cc\ncontent here\n```\n", + None, + [(1, 4, ["content here"])], + id="options-pos1", + ), + pytest.param( + # 1:intro 2: 3:```{cap} 4::opt: xx 5::class: cc 6:body 7:``` + "intro\n\n```{cap}\n:opt: xx\n:class: cc\nbody\n```\n", + None, + [(3, 6, ["body"])], + id="options-pos2", + ), + pytest.param( + # 1:```{cap} 2::opt: xx 3::class: cc 4:(blank) 5:content here 6:``` + "```{cap}\n:opt: xx\n:class: cc\n\ncontent here\n```\n", + None, + [(1, 5, ["content here"])], + id="options-blank", + ), + pytest.param( + # 1:```{cap} 2:--- 3:opt: xx 4:class: cc 5:--- 6:content here 7:``` + "```{cap}\n---\nopt: xx\nclass: cc\n---\ncontent here\n```\n", + None, + [(1, 6, ["content here"])], + id="yaml-block", + ), + pytest.param( + # 1:before 2: 3::::{cap} 4:colon content 5:::: + "before\n\n:::{cap}\ncolon content\n:::\n", + ["colon_fence"], + [(3, 4, ["colon content"])], + id="colon-fence", + ), + pytest.param( + # 1:intro 2: 3:````{cap} 4:(blank) 5:```{cap} 6:inner content 7:``` 8:```` + "intro para\n\n````{cap}\n\n```{cap}\ninner content\n```\n````\n", + None, + [ + (3, 5, ["```{cap}", "inner content", "```"]), + (5, 6, ["inner content"]), + ], + id="nested-fence", + ), + pytest.param( + # 1:para before 2: 3:- list item 4: 5: ```{cap} 6: nested 7: ``` + "para before\n\n- list item text\n\n ```{cap}\n nested in list\n ```\n", + None, + [(5, 6, ["nested in list"])], + id="list-embedded", + ), +] + + +@pytest.mark.parametrize("source,extensions,expected", _MYST_SHAPES) +def test_myst_content_offset_document_relative(source, extensions, expected): + """``content_offset`` and ``content.items`` are document-relative (0-based).""" + caps, _doctree = _run_myst(source, extensions=extensions) + assert len(caps) == len(expected) + for cap, (lineno, first_content_line, content) in zip(caps, expected, strict=True): + assert cap["lineno"] == lineno + assert cap["content"] == content + # content_offset == (1-based first content line) - 1 + assert cap["content_offset"] == first_content_line - 1 + _assert_docutils_convention(cap) + + +@pytest.mark.parametrize( + "source,first_content_line", + [ + # 1:.. cap:: 2:(blank) 3: content here + (".. cap::\n\n content here\n", 3), + # 1:.. cap:: 2: :opt: 3: :class: 4:(blank) 5: content here + (".. cap::\n :opt: xx\n :class: cc\n\n content here\n", 5), + # 1:Title 2:===== 3: 4:para 5: 6:.. cap:: 7: 8: deep content + ("Title\n=====\n\npara\n\n.. cap::\n\n deep content\n", 8), + ], +) +def test_rst_ground_truth_convention(source, first_content_line): + """The plain docutils rST parser follows the same convention MyST now does. + + This pins the *contract* (``content_offset == first content line - 1`` and + absolute per-line ``items``) rather than frozen numbers, so MyST parity is + asserted against docutils itself. + """ + caps, _doctree = _run_rst(source) + assert len(caps) == 1 + cap = caps[0] + assert cap["content_offset"] == first_content_line - 1 + _assert_docutils_convention(cap) + + +def test_eval_rst_content_offset_pinned(): + """``eval-rst`` already uses the real rST state machine (document-relative).""" + source = ( + "# Heading\n\npara one\n\npara two\n\n" + "```{eval-rst}\n.. cap::\n\n rst inner content\n```\n" + ) + caps, _doctree = _run_myst(source) + assert len(caps) == 1 + cap = caps[0] + # eval-rst fence on line 7, ``.. cap::`` on line 8, content on line 10 + assert cap["lineno"] == 8 + assert cap["content_offset"] == 9 + assert cap["content"] == ["rst inner content"] + _assert_docutils_convention(cap) + + +class _WarnAtFirstContentLine(Directive): + """Mimic an extension reporting a problem on its first content line.""" + + has_content = True + option_spec = { + "opt": rst_directives.unchanged, + "class": rst_directives.class_option, + } + + def run(self): + # ``content_offset + N`` for the Nth (1-based) content line, i.e. N=1 + self.state_machine.reporter.warning("boom", line=self.content_offset + 1) + return [] + + +@pytest.mark.parametrize( + "source,true_line", + [ + # fence on line 1, content on line 2 + ("```{warnat}\nbody\n```\n", 2), + # fence on line 5, content on line 6 + ("a\n\nb\n\n```{warnat}\nbody\n```\n", 6), + # fence on line 1, options on 2-3, content on line 4 + ("```{warnat}\n:opt: x\n:class: c\nbody\n```\n", 4), + ], +) +def test_downstream_reporter_warning_line(source, true_line): + """A ``reporter.warning(line=content_offset + N)`` cites the true document line. + + This is the definition of done: the sphinx-jinja2 style idiom must now map + to ``:TRUELINE:`` for the offending content line. + """ + stream = io.StringIO() + with _registered(warnat=_WarnAtFirstContentLine): + publish_doctree( + source, parser=Parser(), settings_overrides={"warning_stream": stream} + ) + assert f":{true_line}:" in stream.getvalue() + + +@pytest.mark.parametrize( + "source,note_body_line", + [ + # note fence on line 1, body on line 2 + ("```{note}\nnote body\n```\n", 2), + # note fence on line 5, body on line 6 + ("# Title\n\npara\n\n```{note}\nnote body\n```\n", 6), + ], +) +def test_nested_content_line_unchanged(source, note_body_line): + """Nested rendering keeps its (already correct) line numbers. + + The document-relative ``content_offset`` change must not shift the lines of + nested content -- the ``nested_parse`` identity holds for the standard idiom. + """ + doctree, _warnings = _publish(source) + para = next( + p for p in doctree.findall(nodes.paragraph) if p.astext() == "note body" + ) + assert para.line == note_body_line + + +def test_epigraph_attribution_line_docutils_parity(): + """``{epigraph}`` attribution ``.line`` matches the docutils rST convention. + + Previously the attribution line was one too small (a latent off-by-one); + it now equals the attribution's true document line, exactly as the plain + rST parser reports it. + """ + # fence on line 3, quoted line 4, blank 5, attribution on document line 6 + source = "before\n\n```{epigraph}\nquoted line\n\n-- Attribution Author\n```\n" + doctree, _warnings = _publish(source) + attribution = next(doctree.findall(nodes.attribution)) + quoted = next( + p for p in doctree.findall(nodes.paragraph) if p.astext() == "quoted line" + ) + assert attribution.line == 6 # true document line of the attribution + assert quoted.line == 4 # nested body line is unchanged + + # docutils ground truth: attribution .line == its own document line (here 5) + rst = ".. epigraph::\n\n quoted line\n\n -- Attribution Author\n" + rst_doctree = publish_doctree( + rst, settings_overrides={"warning_stream": io.StringIO()} + ) + rst_attribution = next(rst_doctree.findall(nodes.attribution)) + assert rst_attribution.line == 5 + + +def test_parsed_literal_line_absolute(): + """``{parsed-literal}`` node ``.line`` is now the correct absolute line. + + docutils sets ``node.line = content_offset + 1``; with a document-relative + ``content_offset`` this is the true content line (previously a hardcoded 1). + """ + # fence on line 3, content on line 4 + source = "before\n\n```{parsed-literal}\nliteral content\n```\n" + doctree, _warnings = _publish(source) + literal = next(doctree.findall(nodes.literal_block)) + assert literal.line == 4 + + +def test_role_directive_smoke(): + """``{role}`` definitions still register cleanly (no regression). + + The docutils ``Role`` directive guards on ``content_offset > lineno``; with + the merged role spec this never trips, so plain and ``:class:`` forms both + register and remain usable. + """ + # plain custom role registers and is usable + doctree, warnings = _publish("```{role} myrole(emphasis)\n```\n\n{myrole}`hi`\n") + assert warnings == "" + assert len(list(doctree.findall(nodes.emphasis))) == 1 + + # custom role carrying a :class: option also registers without regression + doctree, warnings = _publish( + "```{role} myrole2(emphasis)\n:class: myclass\n```\n\n{myrole2}`hi`\n" + ) + assert warnings == "" + emphasis = list(doctree.findall(nodes.emphasis)) + assert len(emphasis) == 1 + assert emphasis[0]["classes"] == ["myclass"] diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 890badbc..7404d491 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -445,7 +445,9 @@ def test_directive_block_text_rst_parity(): """A fenced directive's ``self.block_text`` mirrors the rST full source. The block text spans the opening fence, options block, body and closing - fence, while ``self.content``/``self.content_offset`` stay body-relative. + fence, while ``self.content`` holds only the body lines and + ``self.content_offset`` is the body's 0-based document line + (docutils convention: here ``body line`` is on document line 5, so 4). """ from docutils.parsers.rst import Directive from docutils.parsers.rst import directives as rst_directives @@ -475,7 +477,7 @@ def run(self): "```{blocktextecho}\n---\nclass: tip\n---\nbody line\n```\n" ) assert captured["content"] == ["body line"] - assert captured["content_offset"] == 3 + assert captured["content_offset"] == 4 def test_directive_block_text_fallback(): From 86e18602da4a7258c8815c749f91c2715dc699af Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Jul 2026 23:29:20 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=E2=9C=A8=20NEW:=20source-attributed=20nest?= =?UTF-8?q?ed=20renders=20and=20`insert=5Finput`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a supported mechanism for extensions to render generated content (an included file, a template's output, ...) at the current document position, attributing both the rendered nodes and any warnings to a caller-specified source path. `DocutilsRenderer.nested_render_text` gains a `source` keyword. While it is set, the document/reporter are pointed at that path (so node `source` stamping and docutils reporter warnings attribute there) and the path is pushed onto a new renderer attribution stack consulted by `create_warning` (so the sphinx logger attributes there too). The override is restored on exit -- including after an exception -- and nests re-entrantly, so a source render may itself contain further source renders (e.g. nested includes). The method is now documented as supported API, with its `lineno` line-space semantics and the MyST-not-rST parsing caveat spelled out. `warnings_.create_warning` gains a matching `source` keyword: - sphinx branch: an explicit `node` still wins; otherwise `source` produces a location *string* `"source:line"` (or `"source:"` when there is no line). The colon is required -- sphinx passes a colon-containing string through verbatim, whereas a colon-less string (or a `(source, line)` tuple) is run through `env.doc2path`, which would mangle an explicit source path. The default `(env.docname, line)` path is byte-identical to before. - docutils branch: `source` (with `line`) is passed straight to `reporter.warning`, which docutils honours as an explicit override. The two modes are therefore symmetric: a warning inside generated content is reported against the generating source in both. Consumers wired in-repo: - `MockIncludeDirective.run` now delegates the document/reporter dance to `nested_render_text(..., source=str(path))`, and renders at `startline` rather than `startline + 1`. This fixes the include line off-by-one: a warning on line 3 of an included file was reported at line 4. In sphinx mode it was additionally attributed to the *parent* document (a side effect of the logger switching to `env.docname`); it now cites the included file. The relative-images/relative-docs md_env handling stays in the directive, as it resolves against the containing document's directory. - `MockStateMachine.insert_input(input_lines, source=None)` renders generated MyST at the directive's position with a docutils-compatible signature; with `source` it attributes to `source:1..n`, otherwise it renders in the document's line-space. Content is parsed as MyST (not rST) and rendered immediately into the current node (before any nodes the directive returns -- identical to docutils for the common `return []` pattern). - `MockState.renderer` / `MockStateMachine.renderer` read-only properties bless the existing `_renderer` as the supported way for extensions to reach these APIs from a directive. Known limitation (documented): deferred document-level warnings (e.g. duplicate reference definitions, emitted in `_render_finalise`) escape a render-scoped override and attribute to the containing document; inline tokens carry no per-line maps. Also fixes a stale docs URL in `MockState.__getattr__` (kept with the code). Tests: new `tests/test_renderers/test_nested_render.py` pins include attribution (sphinx via `sphinx_doctree`, docutils via `publish_doctree` + `tmp_path`), the source override in both modes (warnings + node `.source` stamping), `insert_input` (ordering, source attribution, document-space fallback, and the `renderer` properties), and restoration (nested override-in-override attributing innermost then restoring outward, restoration after a directive raises, and restoration when the render itself raises mid-way). It lives under `tests/test_renderers/` so its sphinx builds run after the docutils role fixtures (the same ordering the existing `test_fixtures_docutils`/`test_fixtures_sphinx` split relies on). Fixture regenerated for the off-by-one fix (hand-verified against the source): - `tests/test_renderers/fixtures/mock_include_errors.md`: the unknown-role warning for the single-line `bad.md` moves from `bad.md:2` to `bad.md:1` (its true line); the `bad.md` attribution itself is unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- myst_parser/mdit_to_docutils/base.py | 84 +++- myst_parser/mocking.py | 77 +++- myst_parser/warnings_.py | 31 +- .../fixtures/mock_include_errors.md | 2 +- tests/test_renderers/test_nested_render.py | 387 ++++++++++++++++++ 5 files changed, 553 insertions(+), 28 deletions(-) create mode 100644 tests/test_renderers/test_nested_render.py diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 9465b684..70175a96 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -113,6 +113,10 @@ def __init__(self, parser: MarkdownIt) -> None: } # these are lazy loaded, when needed self._inventories: None | dict[str, inventory.InventoryType] = None + # a stack of source paths that nested renders attribute their warnings to + # (the top entry, if any, overrides the logged location of a warning); + # see ``nested_render_text``'s ``source`` parameter + self._attribution_sources: list[str] = [] def __getattr__(self, name: str): """Warn when the renderer has not been setup yet.""" @@ -178,6 +182,10 @@ def create_warning( If the warning type is listed in the ``suppress_warnings`` configuration, then ``None`` will be returned and no warning logged. + + While a source-attributed nested render is active (see + :meth:`nested_render_text`), the warning is attributed to that source + path rather than the containing document. """ return create_warning( self.document, @@ -186,6 +194,7 @@ def create_warning( wtype=wtype, line=line, append_to=append_to, + source=self._attribution_sources[-1] if self._attribution_sources else None, ) def _render_tokens(self, tokens: list[Token]) -> None: @@ -302,14 +311,37 @@ def nested_render_text( inline: bool = False, temp_root_node: None | nodes.Element = None, heading_offset: int = 0, + source: str | None = None, ) -> None: - """Render unparsed text (appending to the current node). - - :param text: the text to render - :param lineno: the starting line number of the text, within the full source - :param inline: whether the text is inline or block - :param temp_root_node: If set, allow sections to be created as children of this node - :param heading_offset: offset heading levels by this amount + """Render a string of unparsed text, appending to the current node. + + This is the supported mechanism for extensions to render generated + content (for example an included file, or the output of a template) at + the current position in the document. + + .. important:: + + The text is parsed as **MyST Markdown**, not reStructuredText. A + directive whose content is written in rST (rather than MyST) should + wrap it in an ``{eval-rst}`` block instead of rendering it here. + + :param text: the text to render. + :param lineno: a 0-based line shift added to every rendered node's line, + i.e. it establishes the line-space of ``text``. A directive rendering + document-relative content passes its ``content_offset``; content that + is relative to an external file passes ``0`` (so the file's 1-based + line ``N`` is reported as line ``N``). + :param inline: whether to parse the text as inline or block content. + :param temp_root_node: if set, allow sections to be created as children + of this node (used when parsing content that may contain headings). + :param heading_offset: offset heading levels by this amount. + :param source: if given, attribute both the rendered nodes' ``source`` + and any warnings emitted during the render to this path (rather than + the containing document), for the duration of the render. Typically + an absolute path to the file or template the ``text`` originates from. + The override is restored afterwards and nests correctly, so a source + render may itself contain further source renders (e.g. nested + includes). """ tokens = ( self.md.parseInline(text, self.md_env) @@ -344,9 +376,45 @@ def _restore(): self.md_env["temp_root_node"] = current_root_node self._level_to_section = current_level_to_section - with _restore(): + with self._attribute_to_source(source), _restore(): self._render_tokens(tokens) + @contextmanager + def _attribute_to_source(self, source: str | None) -> Iterator[None]: + """Temporarily attribute rendered nodes and warnings to ``source``. + + For the duration of the context the document/reporter are pointed at + ``source`` (so node ``source`` stamping and docutils reporter warnings + attribute to it) and ``source`` is pushed onto the attribution stack + consulted by :meth:`create_warning` (so the sphinx logger attributes to + it too). Everything is restored on exit, including after an exception, + and nests re-entrantly. A ``source`` of ``None`` is a no-op. + """ + if source is None: + yield + return + reporter = self.reporter + prev_document_source = self.document["source"] + prev_reporter_source = reporter.source + # ``get_source_and_line`` may not pre-exist (it is normally installed by + # the rST state machine), so record whether to restore or delete it. + had_get_source_and_line = hasattr(reporter, "get_source_and_line") + prev_get_source_and_line = getattr(reporter, "get_source_and_line", None) + self.document["source"] = source + reporter.source = source + reporter.get_source_and_line = lambda li=None: (source, li) + self._attribution_sources.append(source) + try: + yield + finally: + self._attribution_sources.pop() + self.document["source"] = prev_document_source + reporter.source = prev_reporter_source + if had_get_source_and_line: + reporter.get_source_and_line = prev_get_source_and_line + else: + del reporter.get_source_and_line + @contextmanager def current_node_context( self, node: nodes.Element, append: bool = False diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index 9d419b38..534d564f 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -7,6 +7,7 @@ import os import re import sys +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING, Any @@ -123,6 +124,17 @@ class Struct: self.memo = Struct + @property + def renderer(self) -> DocutilsRenderer: + """The MyST renderer that this mock state is rendering into. + + This is the supported access point for extension authors to reach the + MyST rendering APIs from within a directive (via ``self.state.renderer``), + for example :meth:`.DocutilsRenderer.nested_render_text` to render + generated MyST content at the directive's position. + """ + return self._renderer + def parse_directive_block( self, content: StringList, @@ -305,7 +317,7 @@ def __getattr__(self, name: str): msg = ( f"{cls} has not yet implemented attribute '{name}'. " "You can parse RST directly via the `{{eval-rst}}` directive: " - "https://myst-parser.readthedocs.io/en/latest/syntax/syntax.html#how-directives-parse-content" + "https://myst-parser.readthedocs.io/en/latest/syntax/roles-and-directives.html#how-directives-parse-content" if hasattr(Body, name) else f"{cls} has no attribute '{name}'" ) @@ -327,6 +339,44 @@ def __init__(self, renderer: DocutilsRenderer, lineno: int): self.node: nodes.Element = renderer.current_node self.match_titles: bool = True + @property + def renderer(self) -> DocutilsRenderer: + """The MyST renderer that this mock state machine is rendering into. + + This is the supported access point for extension authors to reach the + MyST rendering APIs from within a directive (via + ``self.state_machine.renderer``). + """ + return self._renderer + + def insert_input( + self, input_lines: Sequence[str], source: str | None = None + ) -> None: + """Render generated text immediately at the directive's position. + + This mirrors the signature of docutils' + ``RSTStateMachine.insert_input``, so directives written for docutils + keep working, but with two behavioural differences to be aware of: + + - the text is parsed as **MyST Markdown**, not reStructuredText; + - it is rendered *immediately* into the current node (appearing before + any nodes the directive itself returns), rather than being spliced + back into the input stream. For the common ``return []`` pattern the + result is identical to docutils. + + :param input_lines: the lines to render; a list of strings or a + docutils ``StringList``. + :param source: if given, attribute the rendered nodes and any warnings + to this path (reported as ``source:1`` onwards -- matching docutils' + handling of inserted external text). Otherwise the text is rendered + in the document's line-space, just after the directive. + """ + text = "\n".join(input_lines) + if source is not None: + self._renderer.nested_render_text(text, 0, source=source) + else: + self._renderer.nested_render_text(text, self._lineno) + def get_source(self, lineno: int | None = None): """Return document source path.""" return self.document["source"] @@ -500,15 +550,15 @@ def run(self) -> list[nodes.Element]: ) return codeblock.run() - # Here we perform a nested render, but temporarily setup the document/reporter - # with the correct document path and lineno for the included file. - source = self.renderer.document["source"] - rsource = self.renderer.reporter.source - line_func = getattr(self.renderer.reporter, "get_source_and_line", None) + # Perform a nested render of the included file. ``nested_render_text`` + # (via its ``source`` argument) temporarily points the document/reporter + # at the included file's path, so its nodes and warnings attribute there. + # ``startline`` is a 0-based line shift, so a warning on the file's + # 1-based line ``N`` is reported at ``N`` (previously ``startline + 1`` + # reported it one line too far -- the include off-by-one). + # The relative-images/relative-docs md_env handling stays here, as it is + # resolved against the *containing* document's directory (``source_dir``). try: - self.renderer.document["source"] = str(path) - self.renderer.reporter.source = str(path) - self.renderer.reporter.get_source_and_line = lambda li: (str(path), li) if "relative-images" in self.options: self.renderer.md_env["relative-images"] = os.path.relpath( path.parent, source_dir @@ -521,18 +571,13 @@ def run(self) -> list[nodes.Element]: ) self.renderer.nested_render_text( file_content, - startline + 1, + startline, + source=str(path), heading_offset=self.options.get("heading-offset", 0), ) finally: - self.renderer.document["source"] = source - self.renderer.reporter.source = rsource self.renderer.md_env.pop("relative-images", None) self.renderer.md_env.pop("relative-docs", None) - if line_func is not None: - self.renderer.reporter.get_source_and_line = line_func - else: - del self.renderer.reporter.get_source_and_line return [] def add_name(self, node: nodes.Element): diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py index 99b9072f..386dcdb2 100644 --- a/myst_parser/warnings_.py +++ b/myst_parser/warnings_.py @@ -102,11 +102,20 @@ def create_warning( node: nodes.Element | None = None, line: int | None = None, append_to: nodes.Element | None = None, + source: str | None = None, ) -> nodes.system_message | None: """Generate a warning, logging if it is necessary. If the warning type is listed in the ``suppress_warnings`` configuration, then ``None`` will be returned and no warning logged. + + :param node: if given, the warning is located at this node (takes priority + over ``source``/``line``). + :param line: the (1-based) line number the warning is located at. + :param source: if given (and ``node`` is not), attribute the warning to this + source path rather than the containing document. This is used to + attribute warnings emitted during a nested render (e.g. an included + file) to the file they originate from. """ # In general we want to both create a warning node within the document AST, # and also log the warning to output it in the CLI etc. @@ -124,13 +133,22 @@ def create_warning( from sphinx.util.logging import getLogger logger = getLogger(__name__) + if node is not None: + location = node + elif source is not None: + # A location *string* containing a colon is passed through by sphinx + # verbatim; a colon-less string (or a ``(source, line)`` tuple) would + # instead be run through ``env.doc2path``, mangling an explicit + # source path -- hence the required trailing colon when there is no + # line number. + location = f"{source}:{line}" if line is not None else f"{source}:" + else: + location = (document.settings.env.docname, line) logger.warning( message, type=type_str, subtype=subtype_str, - location=node - if node is not None - else (document.settings.env.docname, line), + location=location, ) if _is_suppressed_warning( type_str, subtype_str, document.settings.env.config.suppress_warnings @@ -138,6 +156,8 @@ def create_warning( return None if node is not None: _source, _line = utils.get_source_line(node) + elif source is not None: + _source, _line = source, line else: _source, _line = document["source"], line msg_node = _create_warning_node(message_with_type, _source, _line) @@ -150,6 +170,11 @@ def create_warning( kwargs = {} if node is not None: kwargs["base_node"] = node + elif source is not None: + # docutils honours an explicit ``source`` (with ``line``) override + kwargs["source"] = source + if line is not None: + kwargs["line"] = line elif line is not None: kwargs["line"] = line msg_node = document.reporter.warning(message_with_type, **kwargs) diff --git a/tests/test_renderers/fixtures/mock_include_errors.md b/tests/test_renderers/fixtures/mock_include_errors.md index 6e443cb6..a8d7171e 100644 --- a/tests/test_renderers/fixtures/mock_include_errors.md +++ b/tests/test_renderers/fixtures/mock_include_errors.md @@ -19,5 +19,5 @@ Error in include file: ```{include} bad.md ``` . -tmpdir/bad.md:2: (WARNING/2) Unknown interpreted text role "a". [myst.role_unknown] +tmpdir/bad.md:1: (WARNING/2) Unknown interpreted text role "a". [myst.role_unknown] . diff --git a/tests/test_renderers/test_nested_render.py b/tests/test_renderers/test_nested_render.py new file mode 100644 index 00000000..bba6af8e --- /dev/null +++ b/tests/test_renderers/test_nested_render.py @@ -0,0 +1,387 @@ +"""Tests for source-attributed nested renders and ``insert_input``. + +These cover the supported extension-author API for rendering generated content: + +- :meth:`.DocutilsRenderer.nested_render_text` with ``source=`` attributes both + the rendered nodes' ``source`` and any warnings emitted during the render to a + caller-specified path (rather than the containing document), and restores the + previous attribution afterwards (re-entrantly); +- :meth:`.MockStateMachine.insert_input` renders generated MyST at the + directive's position, with a docutils-compatible signature; +- ``MockState.renderer`` / ``MockStateMachine.renderer`` expose the renderer; +- the ``include`` directive attributes warnings to the included file at the + correct (off-by-one-fixed) line, in both docutils and sphinx modes. + +Docutils-mode tests use ``publish_doctree`` with a ``StringIO`` warning stream +(and ``tmp_path`` where real files are needed); sphinx-mode tests use the +``sphinx_doctree`` fixture from ``sphinx-pytest`` (whose ``.warnings`` normalises +the source directory to ````). +""" + +import contextlib +import io + +import pytest +from docutils import nodes +from docutils.core import publish_doctree +from docutils.parsers.rst import Directive +from docutils.parsers.rst import directives as rst_directives +from sphinx.util.console import strip_colors +from sphinx_pytest.plugin import CreateDoctree + +from myst_parser.config.main import MdParserConfig +from myst_parser.mdit_to_docutils.base import DocutilsRenderer, make_document +from myst_parser.parsers.docutils_ import Parser +from myst_parser.parsers.mdit import create_md_parser + + +@contextlib.contextmanager +def _registered(**named_directives): + """Register directives for the duration of the block, then remove them.""" + for name, cls in named_directives.items(): + rst_directives.register_directive(name, cls) + try: + yield + finally: + for name in named_directives: + rst_directives._directives.pop(name, None) + + +def _publish(source, **overrides): + """Parse ``source`` as MyST, returning (doctree, warning text).""" + stream = io.StringIO() + doctree = publish_doctree( + source, + parser=Parser(), + settings_overrides={"warning_stream": stream, **overrides}, + ) + return doctree, stream.getvalue() + + +# --- test directives --------------------------------------------------------- + + +class _SourceRender(Directive): + """Render generated MyST attributed to a fixed fake template path.""" + + has_content = False + + def run(self): + self.state.renderer.nested_render_text( + "{unknownrole}`x`\n\nmore *text*", + 0, + source="/fake/template.j2", + ) + return [] + + +class _InsertInput(Directive): + """Insert generated MyST at the directive position, attributed to a source.""" + + has_content = False + + def run(self): + # the renderer accessors are the supported entry point, and identical + assert self.state.renderer is self.state_machine.renderer + self.state_machine.insert_input(["hello *world*"], source="/fake/gen.txt") + return [] + + +class _InsertInputNoSource(Directive): + """Insert generated MyST rendered in the document's own line-space.""" + + has_content = False + + def run(self): + self.state_machine.insert_input(["plain *text*"]) + return [] + + +class _InsertWarn(Directive): + """Insert generated MyST that emits a warning (attributed to the source).""" + + has_content = False + + def run(self): + self.state_machine.insert_input(["{unknownrole}`x`"], source="/fake/gen.txt") + return [] + + +class _Outer(Directive): + """Nested source render: renders content that itself renders a source.""" + + has_content = False + + def run(self): + self.state.renderer.nested_render_text( + "outer para\n\n```{inner}\n```\n\n{unknownrole}`after-inner`", + 0, + source="/fake/outer.txt", + ) + return [] + + +class _Inner(Directive): + has_content = False + + def run(self): + self.state.renderer.nested_render_text( + "{unknownrole}`in-inner`", 0, source="/fake/inner.txt" + ) + return [] + + +class _RaiseAfterRender(Directive): + """A misbehaving directive that raises after a source render.""" + + has_content = False + + def run(self): + self.state.renderer.nested_render_text("ok text", 0, source="/fake/boom.j2") + raise ValueError("directive boom") + + +# ============================================================================= +# 1 + 2. include attribution (attribution + off-by-one fix) +# ============================================================================= + + +def test_include_attribution_docutils(tmp_path): + """A warning inside an included file cites that file, at its true line. + + The unknown directive is on line 3 of ``inc.md``; previously the include + reported it one line too far (``inc.md:4``). + """ + tmp_path.joinpath("inc.md").write_text("para\n\n```{unknowndir}\n```\n") + stream = io.StringIO() + publish_doctree( + "# Title\n\n```{include} inc.md\n```\n", + source_path=str(tmp_path / "index.md"), + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + warnings = stream.getvalue().replace(str(tmp_path) + "/", "") + assert "inc.md:3: (WARNING/2) Unknown directive type: 'unknowndir'" in warnings + # not attributed to the containing document, nor the old off-by-one line + assert "index.md" not in warnings + assert "inc.md:4" not in warnings + + +def test_include_attribution_sphinx(sphinx_doctree: CreateDoctree): + """Sphinx logs the *included* file (not the parent) at the true line. + + ``inc.md`` is excluded from the build (via ``exclude_patterns``) so it is not + additionally built as a standalone document, which would emit its own + (coincidentally identical) warning and mask a mis-attribution. + """ + sphinx_doctree.srcdir.joinpath("inc.md").write_text( + "para\n\n```{unknowndir}\n```\n" + ) + sphinx_doctree.set_conf( + {"extensions": ["myst_parser"], "exclude_patterns": ["inc.md"]} + ) + result = sphinx_doctree("# Title\n\n```{include} inc.md\n```\n", "index.md") + warnings = strip_colors(result.warnings) + assert "/inc.md:3: WARNING: Unknown directive type: 'unknowndir'" in warnings + # not the containing document / old off-by-one line + assert "index.md:4" not in warnings + + +# ============================================================================= +# 3. extension-style source override (both modes) +# ============================================================================= + + +def test_source_override_docutils(): + """``nested_render_text(source=...)`` attributes warnings and nodes.""" + with _registered(sourcerender=_SourceRender): + doctree, warnings = _publish("# T\n\n```{sourcerender}\n```\n") + assert ( + '/fake/template.j2:1: (WARNING/2) Unknown interpreted text role "unknownrole".' + in warnings + ) + # node stamping: the rendered paragraph carries the overridden source + para = next( + p for p in doctree.findall(nodes.paragraph) if p.astext() == "more text" + ) + assert para.source == "/fake/template.j2" + assert para.line == 3 # third line of the rendered text + + +def test_source_override_sphinx(sphinx_doctree: CreateDoctree): + """The sphinx logger path attributes to the source string verbatim. + + The location is a colon-containing string, so sphinx does not run it through + ``doc2path`` (which would resolve the relative template path against the + source dir). + """ + sphinx_doctree.set_conf({"extensions": ["myst_parser"]}) + with _registered(sourcerender=_SourceRender): + result = sphinx_doctree("# T\n\n```{sourcerender}\n```\n", "index.md") + warnings = strip_colors(result.warnings) + assert ( + '/fake/template.j2:1: WARNING: Unknown interpreted text role "unknownrole".' + in warnings + ) + doctree = result.doctrees["index"] + para = next( + p for p in doctree.findall(nodes.paragraph) if p.astext() == "more text" + ) + assert para.source == "/fake/template.j2" + + +# ============================================================================= +# 4. insert_input +# ============================================================================= + + +def test_insert_input_renders_at_position(): + """Inserted content renders at the directive's position, source-attributed.""" + with _registered(insertinput=_InsertInput): + doctree, warnings = _publish( + "# T\n\nbefore\n\n```{insertinput}\n```\n\nafter\n" + ) + assert not warnings.strip() + # content is spliced in document order, before any returned nodes + paras = [p.astext() for p in doctree.findall(nodes.paragraph)] + assert paras == ["before", "hello world", "after"] + # the inserted paragraph is attributed to the given source + hello = next( + p for p in doctree.findall(nodes.paragraph) if p.astext() == "hello world" + ) + assert hello.source == "/fake/gen.txt" + emphasis = next(e for e in doctree.findall(nodes.emphasis)) + assert emphasis.astext() == "world" + + +def test_insert_input_warning_attribution(): + """A warning from inserted content attributes to ``source:1``.""" + with _registered(insertwarn=_InsertWarn): + _doctree, warnings = _publish("# T\n\n```{insertwarn}\n```\n") + assert ( + '/fake/gen.txt:1: (WARNING/2) Unknown interpreted text role "unknownrole".' + in warnings + ) + + +def test_insert_input_no_source_document_space(): + """Without ``source``, inserted content renders in the document's space.""" + with _registered(insertnosource=_InsertInputNoSource): + doctree, warnings = _publish("# T\n\n```{insertnosource}\n```\n") + assert not warnings.strip() + para = next( + p for p in doctree.findall(nodes.paragraph) if p.astext() == "plain text" + ) + # attributed to the document itself (no override), not a separate source + assert para.source == "" + + +def test_renderer_properties_are_the_renderer(): + """``MockState.renderer`` and ``MockStateMachine.renderer`` are the renderer.""" + captured = {} + + class _Capture(Directive): + has_content = False + + def run(self): + captured["state"] = self.state.renderer + captured["machine"] = self.state_machine.renderer + captured["is_renderer"] = isinstance(self.state.renderer, DocutilsRenderer) + return [] + + with _registered(capturerenderer=_Capture): + _publish("```{capturerenderer}\n```\n") + assert captured["is_renderer"] + assert captured["state"] is captured["machine"] + + +# ============================================================================= +# 5. restoration (success path, nesting, and exception path) +# ============================================================================= + + +def test_nested_source_render_attributes_innermost_and_restores(): + """Nested source renders attribute innermost, and restore outward. + + An ``{outer}`` render (source ``outer.txt``) contains an ``{inner}`` render + (source ``inner.txt``); the inner warning cites ``inner.txt`` and, after it + returns, the outer render's warning cites ``outer.txt`` again. After the + whole directive a document-level warning attributes to the document. + """ + source = "# T\n\n```{outer}\n```\n\n{unknownrole}`doc-level`\n" + with _registered(outer=_Outer, inner=_Inner): + _doctree, warnings = _publish(source) + # innermost attributed to inner.txt (line 1 of the inner render) + assert "/fake/inner.txt:1: (WARNING/2) Unknown interpreted text role" in warnings + # restored outward to outer.txt for content after the inner render (line 6) + assert "/fake/outer.txt:6: (WARNING/2) Unknown interpreted text role" in warnings + # restored to the document for a warning after the directive (line 6) + assert ':6: (WARNING/2) Unknown interpreted text role "unknownrole".' in ( + warnings + ) + + +def test_source_restored_after_directive_raises(): + """A directive raising after a source render leaves the document restored. + + The source override lives inside ``nested_render_text`` (which completes + before the raise), so a subsequent document warning attributes to the + document, and ``document["source"]`` is the original. + """ + + class _Guard(Directive): + has_content = False + + def run(self): + try: + self.state.renderer.nested_render_text( + "ok text", 0, source="/fake/boom.j2" + ) + finally: + # capture state immediately after the source render returns + _Guard.doc_source_after = self.state.renderer.document["source"] + _Guard.stack_after = list(self.state.renderer._attribution_sources) + return [] + + with _registered(guard=_Guard): + _doctree, warnings = _publish( + "# T\n\n```{guard}\n```\n\n{unknownrole}`later`\n" + ) + assert _Guard.doc_source_after == "" + assert _Guard.stack_after == [] + # the later document warning attributes to the document, not the source + assert ':6: (WARNING/2) Unknown interpreted text role "unknownrole".' in ( + warnings + ) + assert "/fake/boom.j2" not in warnings + + +def test_nested_render_text_restores_on_exception(monkeypatch): + """The source override is restored even if the render raises mid-way.""" + md = create_md_parser(MdParserConfig(), DocutilsRenderer) + document = make_document("orig.md") + md.options["document"] = document + renderer = md.renderer + renderer.setup_render(md.options, {}) + + orig_source = renderer.document["source"] + orig_reporter_source = renderer.reporter.source + orig_has_gsl = hasattr(renderer.reporter, "get_source_and_line") + + def boom(_tokens): + # the override must be active while rendering... + assert renderer.document["source"] == "/fake/s.txt" + assert renderer.reporter.source == "/fake/s.txt" + assert renderer._attribution_sources == ["/fake/s.txt"] + raise RuntimeError("render boom") + + monkeypatch.setattr(renderer, "_render_tokens", boom) + + with pytest.raises(RuntimeError, match="render boom"): + renderer.nested_render_text("x", 0, source="/fake/s.txt") + + # ...and fully restored afterwards + assert renderer.document["source"] == orig_source + assert renderer.reporter.source == orig_reporter_source + assert hasattr(renderer.reporter, "get_source_and_line") == orig_has_gsl + assert renderer._attribution_sources == [] From 00f47be74dc3d5e4334ba6a5d71fda0fb7c8e52d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 19 Jul 2026 23:35:41 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20extension-author=20g?= =?UTF-8?q?uide=20for=20nested=20rendering=20APIs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `docs/develop/extending.md` (in the developer toctree after `architecture.md`), documenting the supported API for rendering generated content from a directive: - reaching the renderer via `self.state.renderer` / `self.state_machine.renderer`; - `nested_render_text` (its `lineno` line-space semantics, with a worked example, and `source=` attribution); - `insert_input` (the docutils-compatible signature, the MyST-not-rST caveat, and the immediate-render ordering note); - what `content_offset` now means (document-relative; the `content_offset + N` idiom reports true document lines); - documented limitations (deferred document-level warnings attribute to the containing document; inline tokens carry no per-line maps). The page is narrative and cross-references the API objects with `{py:meth}`/ `{py:obj}`/`{py:func}` roles; autodoc2 renders the docstrings, which are the source of truth. Add an `## Unreleased` CHANGELOG section (✨ New Features / 🐛 Bug Fixes / 📚 Documentation) covering the document-relative `content_offset` fix (with the epigraph/parsed-literal corrections and the absolute `nested_parse` offset note), the include attribution + off-by-one fix, the `source=` nested-render API, `insert_input`, the `renderer` properties, and this page. The stale `MockState` docs URL fix ships with the code in the preceding commit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- CHANGELOG.md | 17 +++++ docs/develop/extending.md | 144 ++++++++++++++++++++++++++++++++++++++ docs/develop/index.md | 1 + 3 files changed, 162 insertions(+) create mode 100644 docs/develop/extending.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ff2d904f..3d86e7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## Unreleased + +### ✨ New Features + +- ✨ Add a supported `source=` argument to `DocutilsRenderer.nested_render_text`, so extensions that render generated content (an included file, a template's output) can attribute its nodes and warnings to the originating file or template, in both docutils and Sphinx builds +- ✨ Add `MockStateMachine.insert_input`, a docutils-compatible way for a directive to render generated MyST at its position +- ✨ Expose the active renderer to directives via read-only `MockState.renderer` and `MockStateMachine.renderer` properties + +### 🐛 Bug Fixes + +- 🐛 Make a directive's `content_offset` document-relative (the 0-based absolute line of its first content line), matching docutils' own reStructuredText parser, so extensions computing `content_offset + N` (e.g. sphinx-jinja2) report the correct line; `MockState.nested_parse` correspondingly treats its `input_offset` as an absolute document line. While here, `{epigraph}` attribution and `{parsed-literal}` node lines are corrected to their true document lines +- 🐛 Attribute warnings raised inside an `{include}`d file to that file rather than the parent document (in Sphinx), and fix an off-by-one so the reported line matches the included file + +### 📚 Documentation + +- 📚 Add a developer guide for rendering generated content from a directive, covering the renderer accessors, `nested_render_text`, `insert_input` and `content_offset` semantics + ## 5.1.0 - 2026-05-13 ### ✨ New Features diff --git a/docs/develop/extending.md b/docs/develop/extending.md new file mode 100644 index 00000000..8bfd1f90 --- /dev/null +++ b/docs/develop/extending.md @@ -0,0 +1,144 @@ +(develop/extending)= + +# Extending the parser: rendering generated content + +Some Sphinx/docutils directives do not just wrap their literal content, but +*generate* new content to render -- for example an included file, or markup +produced from a template. This page describes the supported API for doing that +from a directive under MyST, so that the generated content is rendered as MyST +Markdown and its warnings and nodes are attributed to the right source. + +:::{important} +Content rendered through these APIs is parsed as **MyST Markdown**, not +reStructuredText. A directive that generates rST should wrap it in an +[`{eval-rst}`](syntax/directives/parsing) block instead. +::: + +## Reaching the renderer from a directive + +When MyST runs a directive it passes mock ``state`` and ``state_machine`` +objects (rather than the docutils reStructuredText ones). Both expose the +active MyST renderer through a read-only ``renderer`` property, which is the +supported entry point for the APIs below: + +```python +from docutils.parsers.rst import Directive + + +class MyDirective(Directive): + has_content = False + + def run(self): + renderer = self.state.renderer # or self.state_machine.renderer + ... + return [] +``` + +See {py:obj}`myst_parser.mocking.MockState.renderer` and +{py:obj}`myst_parser.mocking.MockStateMachine.renderer`. + +## Rendering generated text + +{py:meth}`myst_parser.mdit_to_docutils.base.DocutilsRenderer.nested_render_text` +renders a string of MyST at the current position (appending to the current +node): + +```python +self.state.renderer.nested_render_text(text, lineno) +``` + +The ``lineno`` argument is a **0-based line shift** added to every rendered +node's line: it establishes the *line-space* of ``text``. There are two common +choices: + +- **Document-relative content** -- pass the directive's ``content_offset`` (see + [below](content-offset)), so the generated lines map onto the document. +- **File- or template-relative content** -- pass ``0``, so line ``N`` of the + generated text is reported as line ``N`` (of that file/template, when combined + with ``source`` below). + +For example, generated text whose warnings should read as lines ``1, 2, 3, …`` +of a template is rendered with ``lineno=0``. + +### Attributing to a source + +Pass ``source`` to attribute both the rendered nodes' ``source`` and any +warnings emitted during the render to a specific path (typically the absolute +path of the file or template the text came from), rather than the containing +document: + +```python +self.state.renderer.nested_render_text(template_output, 0, source="/abs/template.j2") +``` + +A warning on the first line of ``template_output`` is then reported as +``/abs/template.j2:1`` in both docutils and Sphinx builds, and the resulting +nodes carry ``source="/abs/template.j2"``. The override is restored when the +render finishes (even on error) and nests correctly, so a source render may +itself contain further source renders -- this is exactly how nested +``{include}`` directives attribute each file. + +:::{note} +In Sphinx the location is passed to the logger as a ``"source:line"`` *string*. +This is deliberate: Sphinx passes a colon-containing string through verbatim, +whereas a plain path would be resolved against the project via ``doc2path``. +::: + +## Inserting text at the directive's position + +{py:meth}`myst_parser.mocking.MockStateMachine.insert_input` mirrors the +signature of docutils' ``RSTStateMachine.insert_input``, so directives written +for docutils keep working: + +```python +self.state_machine.insert_input(generated_lines, source="/abs/gen.txt") +``` + +Two behavioural differences are worth knowing: + +- the lines are parsed as **MyST Markdown**, not reStructuredText; +- they are rendered *immediately* into the current node -- appearing **before** + any nodes the directive itself returns -- rather than being spliced back into + the input stream. For the common ``return []`` pattern the outcome is + identical to docutils. + +Without ``source`` the text is rendered in the document's own line-space, just +after the directive. + +(content-offset)= +## What ``content_offset`` means + +A directive's ``self.content_offset`` is **document-relative**: it is the +0-based absolute line of the directive's first content line (equivalently, its +1-based document line minus one), exactly as docutils' own reStructuredText +parser provides. The standard idiom therefore reports true document lines: + +```python +# the Nth (1-based) content line is document line content_offset + N +self.state_machine.reporter.warning("...", line=self.content_offset + n) +``` + +and each entry of ``self.content.items`` carries the absolute document line of +its line. A plain nested parse of the content needs nothing special: + +```python +self.state.nested_parse(self.content, self.content_offset, node) +``` + +## Documented limitations + +- **Deferred, document-level warnings** are *not* covered by a render-scoped + ``source``. For instance, a duplicate Markdown reference definition is + reported at the end of the whole-document render, so it attributes to the + containing document (with the in-file line) even when the definition came from + an included file. +- **Inline tokens carry no per-line maps**, so a warning about inline content is + attributed to the line of its containing block, not the exact inline position. + +## API reference + +- {py:obj}`myst_parser.mocking.MockState.renderer` / + {py:obj}`myst_parser.mocking.MockStateMachine.renderer` +- {py:meth}`myst_parser.mdit_to_docutils.base.DocutilsRenderer.nested_render_text` +- {py:meth}`myst_parser.mocking.MockStateMachine.insert_input` +- {py:func}`myst_parser.warnings_.create_warning` diff --git a/docs/develop/index.md b/docs/develop/index.md index f3a3f977..95196ccf 100644 --- a/docs/develop/index.md +++ b/docs/develop/index.md @@ -6,6 +6,7 @@ codebase, and some guidelines for how you can contribute. ```{toctree} contributing.md architecture.md +extending.md test_infrastructure.md ``` From 2bd301d10f9b11163a1f1eb76291f4ca9c84d7b1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 00:07:28 +0000 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20character-based=20lin?= =?UTF-8?q?e=20numbers=20for=20`:start-after:`=20includes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In `MockIncludeDirective.run`, the `:start-after:` handling advanced the included file's `startline` by `split_index + len(split_on)` -- a *character* index into the file content -- rather than a line count. Because `startline` is then used as the 0-based line shift for `nested_render_text`, a warning inside the retained segment was reported far past the end of the file (e.g. a directive truly on line 4 of a 5-line included file warned at `inc.md:19`). Advance by the number of newlines in the consumed prefix instead: startline += file_content.count("\n", 0, split_index + len(split_on)) The remaining text's first (possibly partial) line still lies ON the marker's line, so counting the consumed newlines makes `nested_render_text` report the TRUE file lines -- consistent with the `:start-line:` path. This deliberately differs from docutils, whose `:start-after:` reports lines relative to the retained segment; MyST reports true file lines. `:end-before:` only truncates `file_content` and does not touch `startline`, so it is unaffected. Tests (`tests/test_renderers/test_nested_render.py`): a `:start-after:` include whose warning-triggering line sits at a known true file line asserts the true line is reported and the old character-index-derived line is absent, plus a combined `:start-after:` + `:end-before:` case. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- CHANGELOG.md | 2 +- myst_parser/mocking.py | 11 ++++- tests/test_renderers/test_nested_render.py | 48 ++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d86e7c0..e2fbd203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ ### 🐛 Bug Fixes - 🐛 Make a directive's `content_offset` document-relative (the 0-based absolute line of its first content line), matching docutils' own reStructuredText parser, so extensions computing `content_offset + N` (e.g. sphinx-jinja2) report the correct line; `MockState.nested_parse` correspondingly treats its `input_offset` as an absolute document line. While here, `{epigraph}` attribution and `{parsed-literal}` node lines are corrected to their true document lines -- 🐛 Attribute warnings raised inside an `{include}`d file to that file rather than the parent document (in Sphinx), and fix an off-by-one so the reported line matches the included file +- 🐛 Attribute warnings raised inside an `{include}`d file to that file rather than the parent document (in Sphinx), and fix line-number reporting so it matches the included file: an off-by-one in the base case, and a `:start-after:` bug that derived the line from a *character* index (so warnings landed past the end of the file). MyST reports true file lines here, unlike docutils' `:start-after:` which reports lines relative to the retained segment ### 📚 Documentation diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index 534d564f..d324564b 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -497,7 +497,16 @@ def run(self) -> list[nodes.Element]: f'Directive "{self.name}"; option "{split_on_type}": text not found "{split_on}".', ) if split_on_type == "start-after": - startline += split_index + len(split_on) + # ``split_index`` is a *character* index into ``file_content``, + # so add the number of newlines in the consumed prefix (not the + # character count) to keep ``startline`` a line number. The + # remaining text's first (possibly partial) line still lies ON + # the marker's line, so counting the consumed newlines makes + # ``nested_render_text(file_content, startline)`` report the TRUE + # file lines -- consistent with the ``:start-line:`` path. (This + # differs from docutils, whose ``:start-after:`` reports lines + # relative to the retained segment; MyST reports true file lines.) + startline += file_content.count("\n", 0, split_index + len(split_on)) file_content = file_content[split_index + len(split_on) :] else: file_content = file_content[:split_index] diff --git a/tests/test_renderers/test_nested_render.py b/tests/test_renderers/test_nested_render.py index bba6af8e..e06ad339 100644 --- a/tests/test_renderers/test_nested_render.py +++ b/tests/test_renderers/test_nested_render.py @@ -187,6 +187,54 @@ def test_include_attribution_sphinx(sphinx_doctree: CreateDoctree): assert "index.md:4" not in warnings +def test_include_start_after_true_file_line(tmp_path): + """``:start-after:`` reports the warning at its TRUE file line. + + The included file's ``{unknowndir}`` fence is on file line 5. Previously + ``startline`` was advanced by the *character* index of the marker, so the + warning was reported far past the end of the file (here ``inc.md:32``); + the fix advances by the number of consumed newlines, giving ``inc.md:5``. + """ + tmp_path.joinpath("inc.md").write_text( + "first line\nsecond START-HERE\nthird line\n\n```{unknowndir}\n```\n" + ) + stream = io.StringIO() + publish_doctree( + "# Title\n\n```{include} inc.md\n:start-after: START-HERE\n```\n", + source_path=str(tmp_path / "index.md"), + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + warnings = stream.getvalue().replace(str(tmp_path) + "/", "") + assert "inc.md:5: (WARNING/2) Unknown directive type: 'unknowndir'" in warnings + # not the old character-index-derived line (past EOF) + assert "inc.md:32" not in warnings + + +def test_include_start_after_end_before_true_file_line(tmp_path): + """Combined ``:start-after:`` + ``:end-before:`` reports the TRUE file line. + + The ``{unknowndir}`` fence is on file line 4, between the ``START`` and + ``END`` markers. ``:end-before:`` does not touch ``startline``, so only the + (now newline-counted) ``:start-after:`` shift applies -- giving ``inc.md:4`` + rather than the old character-index-derived ``inc.md:15``. + """ + tmp_path.joinpath("inc.md").write_text( + "header\nSTART\n\n```{unknowndir}\n```\n\nEND\ntrailing\n" + ) + stream = io.StringIO() + publish_doctree( + "# Title\n\n```{include} inc.md\n:start-after: START\n:end-before: END\n```\n", + source_path=str(tmp_path / "index.md"), + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + warnings = stream.getvalue().replace(str(tmp_path) + "/", "") + assert "inc.md:4: (WARNING/2) Unknown directive type: 'unknowndir'" in warnings + # not the old character-index-derived line + assert "inc.md:15" not in warnings + + # ============================================================================= # 3. extension-style source override (both modes) # ============================================================================= From a7ec25c85ec35ce2fd0c39fb979d9b1c7983e72c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 00:15:44 +0000 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20strengthen=20attr?= =?UTF-8?q?ibution=20tests=20and=20API=20caveats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the review of the nested-render/attribution work. No runtime behaviour changes -- tests, docs, docstrings and a changelog note only. - Tests: make `create_warning`'s `source` kwarg standalone-load-bearing. The existing extension-style tests always run with the renderer's ambient reporter override active, which masks the docutils-branch `source` kwarg (dropping it survives the suite). Add `test_create_warning_source_docutils_arm` (calls `create_warning` directly on an override-free `new_document`, asserting the `source`/`source:line` attribution and that the document's own source does not leak), and `test_create_warning_source_sphinx_arm_no_line` (sphinx build, `line=None` -> logged `source:` trailing-colon form, not doc2path-mangled). - Tests: parametrize `test_insert_input_renders_at_position` over `list[str]` and a docutils `StringList` input, pinning that `insert_input` accepts both. - Tests: `test_downstream_reporter_warning_line` now also asserts the old, body-relative `:{body_offset + 1}:` location is absent (guarded to skip if it would coincide with the true line -- it does not for these cases). - Changelog: add a migration note for extension authors under the `content_offset` bullet -- `MockState.nested_parse`'s `input_offset` is now an absolute document offset; the standard idiom is unchanged, but literal relative offsets (e.g. `0`) now place docutils-identically (autodoc/autosummary `offset=0` renders at the document top), and `self.lineno + self.content_offset` now double-counts (as it always did under reStructuredText). - Docs/docstring: `insert_input` -- state plainly that docutils requires `source` positionally while the mock makes it optional; that inserted content lands before the directive's returned nodes (docutils splices after; identical only for `return []`); and that a `StringList`'s per-line `(source, offset)` items are not preserved (joined and attributed uniformly to `source`). - Docs: fix the `{eval-rst}` cross-reference to the `#`-prefixed label form used by sibling docs, and note in the limitations that `document["source"]`/the reporter are repointed at the override during a source-attributed render (extensions resolving paths against `document["source"]` mid-render see it -- the same behaviour `{include}` has always had). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- CHANGELOG.md | 1 + docs/develop/extending.md | 22 ++++- myst_parser/mocking.py | 25 +++-- tests/test_directive_line_mapping.py | 26 ++++-- tests/test_renderers/test_nested_render.py | 104 ++++++++++++++++++++- 5 files changed, 153 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2fbd203..c8370a34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### 🐛 Bug Fixes - 🐛 Make a directive's `content_offset` document-relative (the 0-based absolute line of its first content line), matching docutils' own reStructuredText parser, so extensions computing `content_offset + N` (e.g. sphinx-jinja2) report the correct line; `MockState.nested_parse` correspondingly treats its `input_offset` as an absolute document line. While here, `{epigraph}` attribution and `{parsed-literal}` node lines are corrected to their true document lines + - _Migration note for extension authors:_ `MockState.nested_parse`'s `input_offset` is now an absolute document offset (docutils-consistent). The standard `nested_parse(self.content, self.content_offset, node)` idiom is unchanged. However, a directive passing a literal *relative* offset (e.g. `0`) now gets docutils-identical placement — under Sphinx, autodoc/autosummary's default `offset=0` relocates rendered content to the top of the document, exactly as under reStructuredText — and arithmetic such as `self.lineno + self.content_offset` now double-counts the position (as it always did under reStructuredText) - 🐛 Attribute warnings raised inside an `{include}`d file to that file rather than the parent document (in Sphinx), and fix line-number reporting so it matches the included file: an off-by-one in the base case, and a `:start-after:` bug that derived the line from a *character* index (so warnings landed past the end of the file). MyST reports true file lines here, unlike docutils' `:start-after:` which reports lines relative to the retained segment ### 📚 Documentation diff --git a/docs/develop/extending.md b/docs/develop/extending.md index 8bfd1f90..61fac725 100644 --- a/docs/develop/extending.md +++ b/docs/develop/extending.md @@ -11,7 +11,7 @@ Markdown and its warnings and nodes are attributed to the right source. :::{important} Content rendered through these APIs is parsed as **MyST Markdown**, not reStructuredText. A directive that generates rST should wrap it in an -[`{eval-rst}`](syntax/directives/parsing) block instead. +[`{eval-rst}`](#syntax/directives/parsing) block instead. ::: ## Reaching the renderer from a directive @@ -94,13 +94,19 @@ for docutils keep working: self.state_machine.insert_input(generated_lines, source="/abs/gen.txt") ``` -Two behavioural differences are worth knowing: +Several behavioural differences are worth knowing: +- **``source`` is optional.** docutils' ``insert_input(input_lines, source)`` + takes ``source`` as a *required* positional argument; here it is optional (see + the last paragraph). - the lines are parsed as **MyST Markdown**, not reStructuredText; - they are rendered *immediately* into the current node -- appearing **before** - any nodes the directive itself returns -- rather than being spliced back into - the input stream. For the common ``return []`` pattern the outcome is - identical to docutils. + any nodes the directive itself returns -- whereas docutils splices the input + back into the state machine, so it is processed **after** the directive's + returned nodes. The two match only for the common ``return []`` pattern; +- if you pass a docutils ``StringList``, its per-line ``(source, offset)`` items + are **not** preserved: the lines are joined and attributed uniformly to + ``source``. Without ``source`` the text is rendered in the document's own line-space, just after the directive. @@ -134,6 +140,12 @@ self.state.nested_parse(self.content, self.content_offset, node) an included file. - **Inline tokens carry no per-line maps**, so a warning about inline content is attributed to the line of its containing block, not the exact inline position. +- **``document["source"]`` is repointed during a source-attributed render.** + While a ``source`` render is in progress, ``document["source"]`` and the + reporter are temporarily pointed at the override, so an extension that resolves + filesystem paths against ``document["source"]`` *mid-render* will see the + override rather than the containing document -- the same behaviour the + ``{include}`` directive has always had. ## API reference diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index d324564b..3487abb3 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -356,16 +356,25 @@ def insert_input( This mirrors the signature of docutils' ``RSTStateMachine.insert_input``, so directives written for docutils - keep working, but with two behavioural differences to be aware of: - - - the text is parsed as **MyST Markdown**, not reStructuredText; - - it is rendered *immediately* into the current node (appearing before - any nodes the directive itself returns), rather than being spliced - back into the input stream. For the common ``return []`` pattern the - result is identical to docutils. + keep working, but with several behavioural differences to be aware of: + + - **``source`` is optional here.** docutils' + ``insert_input(input_lines, source)`` takes ``source`` as a *required* + positional argument; this mock makes it optional (see ``source`` + below). + - **The text is parsed as MyST Markdown**, not reStructuredText. + - **Ordering differs.** The content is rendered *immediately* into the + current node, so it lands **before** any nodes the directive itself + returns. docutils instead splices the input back into the state + machine, so it is processed **after** the directive's returned nodes. + The two are identical only for the common ``return []`` pattern. + - **Per-line source info is not preserved.** If ``input_lines`` is a + docutils ``StringList``, its per-line ``(source, offset)`` items are + *discarded*: the lines are joined and attributed uniformly to + ``source``. :param input_lines: the lines to render; a list of strings or a - docutils ``StringList``. + docutils ``StringList`` (whose per-line source items are not kept). :param source: if given, attribute the rendered nodes and any warnings to this path (reported as ``source:1`` onwards -- matching docutils' handling of inserted external text). Otherwise the text is rendered diff --git a/tests/test_directive_line_mapping.py b/tests/test_directive_line_mapping.py index dda5ca98..a4d60a41 100644 --- a/tests/test_directive_line_mapping.py +++ b/tests/test_directive_line_mapping.py @@ -272,28 +272,36 @@ def run(self): @pytest.mark.parametrize( - "source,true_line", + "source,true_line,old_wrong_line", [ - # fence on line 1, content on line 2 - ("```{warnat}\nbody\n```\n", 2), - # fence on line 5, content on line 6 - ("a\n\nb\n\n```{warnat}\nbody\n```\n", 6), + # fence on line 1, content on line 2 (old body-relative offset 0 -> line 1) + ("```{warnat}\nbody\n```\n", 2, 1), + # fence on line 5, content on line 6 (old body-relative offset 0 -> line 1) + ("a\n\nb\n\n```{warnat}\nbody\n```\n", 6, 1), # fence on line 1, options on 2-3, content on line 4 - ("```{warnat}\n:opt: x\n:class: c\nbody\n```\n", 4), + # (old body-relative offset 2 -> line 3) + ("```{warnat}\n:opt: x\n:class: c\nbody\n```\n", 4, 3), ], ) -def test_downstream_reporter_warning_line(source, true_line): +def test_downstream_reporter_warning_line(source, true_line, old_wrong_line): """A ``reporter.warning(line=content_offset + N)`` cites the true document line. This is the definition of done: the sphinx-jinja2 style idiom must now map - to ``:TRUELINE:`` for the offending content line. + to ``:TRUELINE:`` for the offending content line. It must *also* + no longer emit the old, body-relative ``:{body_offset + 1}:`` line + (when ``content_offset`` was the relative ``body_offset``). """ stream = io.StringIO() with _registered(warnat=_WarnAtFirstContentLine): publish_doctree( source, parser=Parser(), settings_overrides={"warning_stream": stream} ) - assert f":{true_line}:" in stream.getvalue() + warnings = stream.getvalue() + assert f":{true_line}:" in warnings + # the old body-relative location must be gone (skip if it coincides with the + # true line -- it does not for any case here, but guard against it anyway) + if old_wrong_line != true_line: + assert f":{old_wrong_line}:" not in warnings @pytest.mark.parametrize( diff --git a/tests/test_renderers/test_nested_render.py b/tests/test_renderers/test_nested_render.py index e06ad339..d9e857d3 100644 --- a/tests/test_renderers/test_nested_render.py +++ b/tests/test_renderers/test_nested_render.py @@ -20,12 +20,17 @@ import contextlib import io +from collections.abc import Sequence import pytest from docutils import nodes from docutils.core import publish_doctree +from docutils.frontend import get_default_settings from docutils.parsers.rst import Directive +from docutils.parsers.rst import Parser as RSTParser from docutils.parsers.rst import directives as rst_directives +from docutils.statemachine import StringList +from docutils.utils import new_document from sphinx.util.console import strip_colors from sphinx_pytest.plugin import CreateDoctree @@ -33,6 +38,7 @@ from myst_parser.mdit_to_docutils.base import DocutilsRenderer, make_document from myst_parser.parsers.docutils_ import Parser from myst_parser.parsers.mdit import create_md_parser +from myst_parser.warnings_ import MystWarnings, create_warning @contextlib.contextmanager @@ -79,11 +85,13 @@ class _InsertInput(Directive): """Insert generated MyST at the directive position, attributed to a source.""" has_content = False + # overridable by tests to exercise different input container types + input_lines: Sequence[str] = ["hello *world*"] def run(self): # the renderer accessors are the supported entry point, and identical assert self.state.renderer is self.state_machine.renderer - self.state_machine.insert_input(["hello *world*"], source="/fake/gen.txt") + self.state_machine.insert_input(self.input_lines, source="/fake/gen.txt") return [] @@ -283,8 +291,23 @@ def test_source_override_sphinx(sphinx_doctree: CreateDoctree): # ============================================================================= -def test_insert_input_renders_at_position(): - """Inserted content renders at the directive's position, source-attributed.""" +@pytest.mark.parametrize( + "make_input", + [ + pytest.param(lambda: ["hello *world*"], id="list"), + pytest.param( + lambda: StringList(["hello *world*"], source="/fake/gen.txt"), + id="stringlist", + ), + ], +) +def test_insert_input_renders_at_position(make_input, monkeypatch): + """Inserted content renders at the directive's position, source-attributed. + + ``insert_input`` accepts either a plain ``list[str]`` or a docutils + ``StringList`` (whose lines are joined), so both produce identical output. + """ + monkeypatch.setattr(_InsertInput, "input_lines", make_input()) with _registered(insertinput=_InsertInput): doctree, warnings = _publish( "# T\n\nbefore\n\n```{insertinput}\n```\n\nafter\n" @@ -433,3 +456,78 @@ def boom(_tokens): assert renderer.reporter.source == orig_reporter_source assert hasattr(renderer.reporter, "get_source_and_line") == orig_has_gsl assert renderer._attribution_sources == [] + + +# ============================================================================= +# 6. create_warning ``source`` kwarg is standalone-load-bearing (override-free) +# ============================================================================= +# +# The extension-style tests above always run with the renderer's ambient +# reporter override active (set by ``nested_render_text(source=...)``), which +# would mask ``create_warning``'s own ``source`` handling. These call +# ``create_warning`` directly, with no override in play, so the ``source`` +# kwarg is the only thing that can move the attribution. + + +def _override_free_document(stream): + """A docutils-mode document with a fresh reporter and no source override.""" + settings = get_default_settings(RSTParser) + settings.warning_stream = stream + settings.myst_suppress_warnings = [] + return new_document("mydoc.md", settings) + + +@pytest.mark.parametrize( + "line,expected", + [(3, "/fake/x.txt:3:"), (None, "/fake/x.txt:")], + ids=["with-line", "no-line"], +) +def test_create_warning_source_docutils_arm(line, expected): + """``create_warning(source=...)`` attributes via the docutils reporter arm. + + There is no renderer (hence no ambient reporter override), so the ``source`` + kwarg passed through to ``reporter.warning`` is the only thing that can move + the attribution off the document's own source -- i.e. it is + standalone-load-bearing. + """ + stream = io.StringIO() + document = _override_free_document(stream) + create_warning( + document, "msg", MystWarnings.RENDER_METHOD, source="/fake/x.txt", line=line + ) + warnings = stream.getvalue() + assert expected in warnings + # the document's own source must not leak in (proves the kwarg is honoured) + assert "mydoc.md" not in warnings + + +class _SphinxSourceWarnNoLine(Directive): + """Emit a source-attributed, line-less warning through ``create_warning``.""" + + has_content = False + + def run(self): + create_warning( + self.state.document, + "no-line msg", + MystWarnings.RENDER_METHOD, + source="/fake/x.txt", + line=None, + ) + return [] + + +def test_create_warning_source_sphinx_arm_no_line(sphinx_doctree: CreateDoctree): + """The sphinx arm logs ``source:`` (trailing colon) and skips ``doc2path``. + + With ``line=None`` the location is the string ``"/fake/x.txt:"``; the + trailing colon makes sphinx pass it through verbatim rather than resolving + it as a docname (which would append a source suffix such as ``.rst``). + """ + sphinx_doctree.set_conf({"extensions": ["myst_parser"]}) + with _registered(swarn=_SphinxSourceWarnNoLine): + result = sphinx_doctree("# T\n\n```{swarn}\n```\n", "index.md") + warnings = strip_colors(result.warnings) + assert "/fake/x.txt:: WARNING: no-line msg" in warnings + # not doc2path-mangled into a docname with a source suffix + assert ".rst" not in warnings From 4a9f0a5b559f0e2727b7e413999ca9e61b2cd6b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 00:45:22 +0000 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20stability=20promi?= =?UTF-8?q?se,=20eval-rst=20docs,=20review=20polish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docs/develop/extending.md: add a "Stability" subsection promising the page's APIs (`nested_render_text`/`source`, `insert_input`, the `renderer` properties, the document-relative `content_offset` contract) are supported public API. - docs/develop/extending.md: document `{eval-rst}` semantics (real docutils state machine; document-absolute `lineno`/`content_offset`; `content_offset + N` works identically), pinned by `test_eval_rst_content_offset_pinned`. - CHANGELOG.md (Unreleased): tighten to house-bullet style keeping every fact, and correct the autodoc/autosummary offset note (rST-consistent offset value, not byte-identical `StringList` attribution). - Code polish: `_attribute_to_source` guards empty-string source (`if not source`); `create_warning` docstring now documents all parameters. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- CHANGELOG.md | 8 ++++---- docs/develop/extending.md | 20 ++++++++++++++++++++ myst_parser/mdit_to_docutils/base.py | 5 +++-- myst_parser/warnings_.py | 10 ++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8370a34..7d5738a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,15 @@ ### ✨ New Features -- ✨ Add a supported `source=` argument to `DocutilsRenderer.nested_render_text`, so extensions that render generated content (an included file, a template's output) can attribute its nodes and warnings to the originating file or template, in both docutils and Sphinx builds +- ✨ Add a `source=` argument to `DocutilsRenderer.nested_render_text`, so extensions rendering generated content (an included file, a template's output) can attribute its nodes and warnings to the originating file or template, in both docutils and Sphinx builds - ✨ Add `MockStateMachine.insert_input`, a docutils-compatible way for a directive to render generated MyST at its position - ✨ Expose the active renderer to directives via read-only `MockState.renderer` and `MockStateMachine.renderer` properties ### 🐛 Bug Fixes -- 🐛 Make a directive's `content_offset` document-relative (the 0-based absolute line of its first content line), matching docutils' own reStructuredText parser, so extensions computing `content_offset + N` (e.g. sphinx-jinja2) report the correct line; `MockState.nested_parse` correspondingly treats its `input_offset` as an absolute document line. While here, `{epigraph}` attribution and `{parsed-literal}` node lines are corrected to their true document lines - - _Migration note for extension authors:_ `MockState.nested_parse`'s `input_offset` is now an absolute document offset (docutils-consistent). The standard `nested_parse(self.content, self.content_offset, node)` idiom is unchanged. However, a directive passing a literal *relative* offset (e.g. `0`) now gets docutils-identical placement — under Sphinx, autodoc/autosummary's default `offset=0` relocates rendered content to the top of the document, exactly as under reStructuredText — and arithmetic such as `self.lineno + self.content_offset` now double-counts the position (as it always did under reStructuredText) -- 🐛 Attribute warnings raised inside an `{include}`d file to that file rather than the parent document (in Sphinx), and fix line-number reporting so it matches the included file: an off-by-one in the base case, and a `:start-after:` bug that derived the line from a *character* index (so warnings landed past the end of the file). MyST reports true file lines here, unlike docutils' `:start-after:` which reports lines relative to the retained segment +- 🐛 Make a directive's `content_offset` document-relative (the 0-based absolute line of its first content line), matching docutils' reStructuredText parser, so extensions computing `content_offset + N` (e.g. sphinx-jinja2) report the correct line; `MockState.nested_parse` correspondingly treats its `input_offset` as an absolute document line. Also corrects `{epigraph}` attribution and `{parsed-literal}` node lines to their true document lines + - _Migration note for extension authors:_ the standard `nested_parse(self.content, self.content_offset, node)` idiom is unchanged, but a directive passing a literal *relative* offset (e.g. `0`) now gets docutils-identical placement. Under Sphinx, autodoc/autosummary's default `offset=0` relocates rendered content to the top of the document — rST-consistent in the offset value, though rST additionally attributes such content via per-line `StringList` source items (which the MyST mock does not model), so the attribution is not byte-identical. Arithmetic such as `self.lineno + self.content_offset` now double-counts the position (as it always did under reStructuredText) +- 🐛 Attribute warnings raised inside an `{include}`d file to that file rather than the parent document (in Sphinx), and fix line reporting to match the included file: an off-by-one in the base case, and a `:start-after:` bug that derived the line from a *character* index (so warnings landed past the file's end). MyST reports true file lines here, unlike docutils' `:start-after:`, which reports lines relative to the retained segment ### 📚 Documentation diff --git a/docs/develop/extending.md b/docs/develop/extending.md index 61fac725..40f3344e 100644 --- a/docs/develop/extending.md +++ b/docs/develop/extending.md @@ -14,6 +14,16 @@ reStructuredText. A directive that generates rST should wrap it in an [`{eval-rst}`](#syntax/directives/parsing) block instead. ::: +## Stability + +The APIs documented on this page are **supported, public API** as of the release +that includes this page: ``DocutilsRenderer.nested_render_text`` (including its +``source`` parameter), ``MockStateMachine.insert_input``, the +``MockState.renderer`` / ``MockStateMachine.renderer`` properties, and the +document-relative ``content_offset`` contract. Third-party extensions may rely +on them: any breaking change will be announced in the changelog and, where +practicable, go through a deprecation cycle rather than changing silently. + ## Reaching the renderer from a directive When MyST runs a directive it passes mock ``state`` and ``state_machine`` @@ -131,6 +141,16 @@ its line. A plain nested parse of the content needs nothing special: self.state.nested_parse(self.content, self.content_offset, node) ``` +### Directives inside `{eval-rst}` blocks + +Directives written in an [`{eval-rst}`](#syntax/directives/parsing) block run +under the **real** docutils reStructuredText state machine, not the mocks used +for MyST-native directives. Their ``lineno`` and ``content_offset`` are still +document-absolute with respect to the containing ``.md`` file -- the same +convention as MyST-native fences -- so the ``content_offset + N`` idiom reports +true document lines identically in both. This equivalence is pinned by +``tests/test_directive_line_mapping.py::test_eval_rst_content_offset_pinned``. + ## Documented limitations - **Deferred, document-level warnings** are *not* covered by a render-scoped diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 70175a96..d4619800 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -388,9 +388,10 @@ def _attribute_to_source(self, source: str | None) -> Iterator[None]: attribute to it) and ``source`` is pushed onto the attribution stack consulted by :meth:`create_warning` (so the sphinx logger attributes to it too). Everything is restored on exit, including after an exception, - and nests re-entrantly. A ``source`` of ``None`` is a no-op. + and nests re-entrantly. A ``source`` of ``None`` (or an empty string) + is a no-op. """ - if source is None: + if not source: yield return reporter = self.reporter diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py index 386dcdb2..7515a100 100644 --- a/myst_parser/warnings_.py +++ b/myst_parser/warnings_.py @@ -109,13 +109,23 @@ def create_warning( If the warning type is listed in the ``suppress_warnings`` configuration, then ``None`` will be returned and no warning logged. + :param document: the document the warning is raised against; supplies the + reporter/environment and the default source and line. + :param message: the warning message. + :param subtype: the warning subtype (a ``MystWarnings`` member or its string + value), combined with ``wtype`` to form the warning category. + :param wtype: the warning type; defaults to ``"myst"``, giving categories + such as ``myst.substitution``. :param node: if given, the warning is located at this node (takes priority over ``source``/``line``). :param line: the (1-based) line number the warning is located at. + :param append_to: if given, the created warning node is appended to this + element. :param source: if given (and ``node`` is not), attribute the warning to this source path rather than the containing document. This is used to attribute warnings emitted during a nested render (e.g. an included file) to the file they originate from. + :return: the created ``system_message`` node, or ``None`` if suppressed. """ # In general we want to both create a warning node within the document AST, # and also log the warning to output it in the CLI etc. From 2939f54c37c130b3d3b282f9a6dcc20bd20ca424 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 12:44:14 +0000 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20normalise=20path=20s?= =?UTF-8?q?eparators=20in=20sphinx=20include-attribution=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assertion in test_include_attribution_sphinx compared against a forward-slash /inc.md path, but on Windows the include path is joined with a backslash (\inc.md), failing both windows-latest CI jobs. The CI logs show the feature itself behaves correctly on Windows (true file, true line); only the assertion was separator-naive. Normalise backslashes in the captured warnings before asserting. The other sphinx-mode tests assert verbatim caller-supplied paths and the docutils-mode tests use prefix-independent substring matches, so this was the only separator-sensitive assertion (all other tests passed on windows-latest). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBANSW6TYQxwWRqGhxhdHw --- tests/test_renderers/test_nested_render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_renderers/test_nested_render.py b/tests/test_renderers/test_nested_render.py index d9e857d3..d67c3285 100644 --- a/tests/test_renderers/test_nested_render.py +++ b/tests/test_renderers/test_nested_render.py @@ -189,7 +189,8 @@ def test_include_attribution_sphinx(sphinx_doctree: CreateDoctree): {"extensions": ["myst_parser"], "exclude_patterns": ["inc.md"]} ) result = sphinx_doctree("# Title\n\n```{include} inc.md\n```\n", "index.md") - warnings = strip_colors(result.warnings) + # normalise path separators: on Windows the include path is ``\inc.md`` + warnings = strip_colors(result.warnings).replace("\\", "/") assert "/inc.md:3: WARNING: Unknown directive type: 'unknowndir'" in warnings # not the containing document / old off-by-one line assert "index.md:4" not in warnings