🐛 FIX: Source line reporting for nested parsing

chrisjsewell · chrisjsewell · commit 06e409d94a8e · 2022-01-04T18:29:23.000+01:00
Previously, nested parsing of content within directives
would not correctly propagate the starting line number
of the content.
diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py
@@ -262,10 +262,19 @@ def add_document_wordcount(self) -> None:
             self.document.note_substitution_def(substitution_node, f"wordcount-{key}")
 
     def nested_render_text(self, text: str, lineno: int) -> None:
-        """Render unparsed text."""
+        """Render unparsed text.
+
+        :param text: the text to render
+        :param lineno: the starting line number of the text, within the full source
+        """
 
         tokens = self.md.parse(text + "\n", self.md_env)
 
+        # update the line numbers
+        for token in tokens:
+            if token.map:
+                token.map = [token.map[0] + lineno, token.map[1] + lineno]
+
         # remove front matter
         if tokens and tokens[0].type == "front_matter":
             tokens.pop(0)
@@ -1139,7 +1148,7 @@ def run_directive(
             directive_class.option_spec["relative-docs"] = directives.path
 
         try:
-            arguments, options, body_lines = parse_directive_text(
+            arguments, options, body_lines, content_offset = parse_directive_text(
                 directive_class, first_line, content
             )
         except DirectiveParsingError as error:
@@ -1175,7 +1184,7 @@ def run_directive(
                 # the absolute line number of the first line of the directive
                 lineno=position,
                 # the line offset of the first line of the content
-                content_offset=0,  # TODO get content offset from `parse_directive_text`
+                content_offset=content_offset,
                 # a string containing the entire directive
                 block_text="\n".join(body_lines),
                 state=state,
diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py
@@ -114,14 +114,14 @@ def parse_directive_block(
         if option_presets:
             raise MockingError("parse_directive_block: option_presets not implemented")
         # TODO should argument_str always be ""?
-        arguments, options, body_lines = parse_directive_text(
+        arguments, options, body_lines, content_offset = parse_directive_text(
             directive, "", "\n".join(content)
         )
         return (
             arguments,
             options,
             StringList(body_lines, source=content.source),
-            line_offset + len(content) - len(body_lines),
+            line_offset + content_offset,
         )
 
     def nested_parse(
diff --git a/myst_parser/parse_directives.py b/myst_parser/parse_directives.py
@@ -36,7 +36,7 @@
 import datetime
 import re
 from textwrap import dedent
-from typing import Any, Callable, Dict, Type
+from typing import Any, Callable, Dict, List, Tuple, Type
 
 import yaml
 from docutils.parsers.rst import Directive
@@ -54,25 +54,27 @@ def parse_directive_text(
     first_line: str,
     content: str,
     validate_options: bool = True,
-):
+) -> Tuple[List[str], dict, List[str], int]:
     """Parse (and validate) the full directive text.
 
     :param first_line: The text on the same line as the directive name.
         May be an argument or body text, dependent on the directive
     :param content: All text after the first line. Can include options.
     :param validate_options: Whether to validate the values of options
 
+    :returns: (arguments, options, body_lines, content_offset)
     """
     if directive_class.option_spec:
         body, options = parse_directive_options(
             content, directive_class, validate=validate_options
         )
+        body_lines = body.splitlines()
+        content_offset = len(content.splitlines()) - len(body_lines)
     else:
         # If there are no possible options, we do not look for a YAML block
         options = {}
-        body = content
-
-    body_lines = body.splitlines()
+        body_lines = content.splitlines()
+        content_offset = 0
 
     if not (
         directive_class.required_arguments
@@ -91,12 +93,13 @@ def parse_directive_text(
     # this is to allow space between the options and the content
     if body_lines and not body_lines[0].strip():
         body_lines = body_lines[1:]
+        content_offset += 1
 
     # check for body content
     if body_lines and not directive_class.has_content:
         raise DirectiveParsingError("No content permitted")
 
-    return arguments, options, body_lines
+    return arguments, options, body_lines, content_offset
 
 
 def parse_directive_options(
diff --git a/tests/test_renderers/fixtures/mock_include_errors.md b/tests/test_renderers/fixtures/mock_include_errors.md
@@ -20,5 +20,5 @@ Error in include file:
 ```{include} bad.md
 ```
 .
-tmpdir/bad.md:1: (ERROR/3) Unknown interpreted text role "a".
+tmpdir/bad.md:2: (ERROR/3) Unknown interpreted text role "a".
 .
diff --git a/tests/test_renderers/fixtures/reporter_warnings.md b/tests/test_renderers/fixtures/reporter_warnings.md
@@ -136,5 +136,19 @@ header nested in admonition
 # Header
 ```
 .
-<string>:1: (WARNING/2) Header nested in this element can lead to unexpected outcomes
+<string>:2: (WARNING/2) Header nested in this element can lead to unexpected outcomes
+.
+
+nested parse warning
+.
+Paragraph
+
+```{note}
+:class: abc
+:name: xyz
+
+{unknown}`a`
+```
+.
+<string>:7: (ERROR/3) Unknown interpreted text role "unknown".
 .
diff --git a/tests/test_renderers/test_parse_directives.py b/tests/test_renderers/test_parse_directives.py
@@ -6,11 +6,21 @@
 from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text
 
 
-@pytest.mark.parametrize("klass,arguments,content", [(Note, "", "a"), (Note, "a", "")])
+@pytest.mark.parametrize(
+    "klass,arguments,content",
+    [(Note, "", "a"), (Note, "a", ""), (Note, "", ":class: name\n\na")],
+)
 def test_parsing(klass, arguments, content, data_regression):
-    arguments, options, body_lines = parse_directive_text(klass, arguments, content)
+    arguments, options, body_lines, content_offset = parse_directive_text(
+        klass, arguments, content
+    )
     data_regression.check(
-        {"arguments": arguments, "options": options, "body": body_lines}
+        {
+            "arguments": arguments,
+            "options": options,
+            "body": body_lines,
+            "content_offset": content_offset,
+        }
     )
 
 
diff --git a/tests/test_renderers/test_parse_directives/test_parsing_Note___class__name_n_na_.yml b/tests/test_renderers/test_parse_directives/test_parsing_Note___class__name_n_na_.yml
@@ -0,0 +1,7 @@
+arguments: []
+body:
+- a
+content_offset: 2
+options:
+  class:
+  - name
diff --git a/tests/test_renderers/test_parse_directives/test_parsing_Note__a_.yml b/tests/test_renderers/test_parse_directives/test_parsing_Note__a_.yml
@@ -1,4 +1,5 @@
 arguments: []
 body:
 - a
+content_offset: 0
 options: {}
diff --git a/tests/test_renderers/test_parse_directives/test_parsing_Note_a__.yml b/tests/test_renderers/test_parse_directives/test_parsing_Note_a__.yml
@@ -1,4 +1,5 @@
 arguments: []
 body:
 - a
+content_offset: 0
 options: {}

Original file line number	Diff line number	Diff line change
`@@ -20,5 +20,5 @@ Error in include file:`
`20`	`20`	```{include} bad.md
`21`	`21`	```
`22`	`22`	`.`
`23`		`-tmpdir/bad.md:1: (ERROR/3) Unknown interpreted text role "a".`
	`23`	`+tmpdir/bad.md:2: (ERROR/3) Unknown interpreted text role "a".`
`24`	`24`	`.`