Fixed rendering of underscores in multi-line text blocks

doc/changes/changes_4.11.0.md

@@ -14,6 +14,10 @@ Fix typo and syntax in agent skills.
 
 - #612: Add support for Visual Basic in the source tag importer
 
+## Bugfixes
+
+- #574: Fixed invalid HTML when inline code contains underscores
+
 ## Documentation
 
 - #604: Fixed link to Product Lifecycle in SECURITY.md

doc/user_guide/use_cases/writing_a_specification.md

@@ -177,6 +177,24 @@ is functionally equivalent to
 
 Tags are described in detail later in this document, see section [Distributing the Detailing Work](distributing_the_detailing_work.md#distributing-the-detailing-work).
 
+#### Notation in Multiline Text Blocks
+
+Multiline text blocks within specification items—specifically the `Description` (or implicit description text), `Rationale`, and `Comment`—support a subset of Markdown notation. When OpenFastTrace processes and renders these blocks (such as in HTML reports), the following markup elements are supported:
+
+* **Paragraphs**: Blocks of text are grouped into paragraphs separated by blank lines.
+* **Unordered Lists**: Lines starting with a bullet marker (`-`, `*`, or `+`) with up to three leading spaces create bulleted lists. Multi-line list items are supported by continuing text on following lines without an empty line in between.
+* **Ordered Lists**: Lines starting with numbers followed by a period (e.g., `1.`, `2.`) create numbered lists.
+* **Preformatted Blocks / Code**: Lines indented by at least four spaces are treated as preformatted text and rendered in `<pre>` blocks.
+* **Inline Code**: Text enclosed in backticks (`` `code` ``) is rendered as inline monospace code (`<code>`).
+* **Hyperlinks**: Markdown links in the format `[Link Text](URL)` are rendered as clickable links.
+* **Emphasis (Italics)**: Text surrounded by single asterisks (`*text*`) or single underscores (`_text_`) is formatted with emphasis (`<em>`).
+* **Strong (Bold)**: Text surrounded by double asterisks (`**text**`) or double underscores (`__text__`) is formatted in bold (`<strong>`).
+* **HTML Escaping**: Angle brackets (`<` and `>`) are automatically escaped (as `&lt;` and `&gt;`) so that code snippets, XML/HTML tags, or type parameters (such as `<T>`) are safely rendered without breaking the document structure.
+
+##### Constraints on Multiline Text Blocks
+
+* **No Zero Characters**: Original multiline texts must not contain zero characters (NUL / `\0`). Any zero characters present in the text are removed before rendering.
+
 
 ### Quality Scenarios
 

reporter/html/src/main/java/org/itsallcode/openfasttrace/report/html/view/html/MarkdownSpanConverter.java

@@ -1,10 +1,14 @@
 package org.itsallcode.openfasttrace.report.html.view.html;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 final class MarkdownSpanConverter
 {
+    private static final String ZERO_SEPARATOR = "\0";
+    private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile(ZERO_SEPARATOR + "(\\d+)" + ZERO_SEPARATOR);
     private static final RegexReplacement INDENTED_CODE = RegexReplacement.create("(    .*[\n])+", "<pre>$1</pre>");
     private static final RegexReplacement BACKTICK_QUOTED_CODE = RegexReplacement.create("`(.*?)`", "<code>$1</code>");
     private static final RegexReplacement LINK = RegexReplacement.create("\\[([^]]*?)\\]\\(([^)].*?)\\)",
@@ -14,8 +18,9 @@ final class MarkdownSpanConverter
     private static final RegexReplacement EMPHASIZED_TEXT = RegexReplacement.create("([_*])(\\p{L}(?:.*\\p{L}))\\1",
             "<em>$2</em>");
 
-    private static final List<RegexReplacement> ALL_MARKDOWN_REPLACEMENTS = List.of(INDENTED_CODE, BACKTICK_QUOTED_CODE,
-            LINK, BOLD_TEXT, EMPHASIZED_TEXT);
+    private static final List<RegexReplacement> CODE_REPLACEMENTS = List.of(INDENTED_CODE, BACKTICK_QUOTED_CODE);
+    private static final List<RegexReplacement> INLINE_MARKDOWN_REPLACEMENTS = List.of(LINK, BOLD_TEXT,
+            EMPHASIZED_TEXT);
 
     // Prevent instantiation
     private MarkdownSpanConverter()
@@ -25,12 +30,46 @@ private MarkdownSpanConverter()
     // [impl->dsn~reporting.html.escape-html~1]
     static String convertLineContent(final String input)
     {
-        String text = escapeHtml(input);
-        for (final RegexReplacement replacement : ALL_MARKDOWN_REPLACEMENTS)
+        String text = escapeHtml(input).replace(ZERO_SEPARATOR, "");
+        final List<String> codeSpans = new ArrayList<>();
+        for (final RegexReplacement codeReplacement : CODE_REPLACEMENTS)
+        {
+            text = extractCodeSpans(codeReplacement, text, codeSpans);
+        }
+        for (final RegexReplacement replacement : INLINE_MARKDOWN_REPLACEMENTS)
         {
             text = replacement.apply(text);
         }
-        return text;
+        return restoreCodeSpans(text, codeSpans);
+    }
+
+    private static String extractCodeSpans(final RegexReplacement codeReplacement, final String input,
+            final List<String> codeSpans)
+    {
+        final Matcher matcher = codeReplacement.pattern.matcher(input);
+        final StringBuilder builder = new StringBuilder();
+        while (matcher.find())
+        {
+            final String placeholder = ZERO_SEPARATOR + codeSpans.size() + ZERO_SEPARATOR;
+            final String converted = codeReplacement.apply(matcher.group());
+            codeSpans.add(converted);
+            matcher.appendReplacement(builder, Matcher.quoteReplacement(placeholder));
+        }
+        matcher.appendTail(builder);
+        return builder.toString();
+    }
+
+    private static String restoreCodeSpans(final String text, final List<String> codeSpans)
+    {
+        final Matcher matcher = PLACEHOLDER_PATTERN.matcher(text);
+        final StringBuilder builder = new StringBuilder();
+        while (matcher.find())
+        {
+            final int index = Integer.parseInt(matcher.group(1));
+            matcher.appendReplacement(builder, Matcher.quoteReplacement(codeSpans.get(index)));
+        }
+        matcher.appendTail(builder);
+        return builder.toString();
     }
 
     static String escapeHtml(String text)

reporter/html/src/test/java/org/itsallcode/openfasttrace/report/html/view/html/MarkdownSpanConverterTest.java

@@ -3,18 +3,36 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
 
 class MarkdownSpanConverterTest
 {
     @ParameterizedTest(name = "Line ''{0}'' converted to HTML ''{1}''")
     @CsvSource(
     {
             "'    code\n', '<pre>    code\n</pre>'",
+            "'    code_with_underscore\n', '<pre>    code_with_underscore\n</pre>'",
+            "'    code with `backticks` and _underscores_\n', '<pre>    code with `backticks` and _underscores_\n</pre>'",
             "'     code\n', '<pre>     code\n</pre>'",
             "'    code', '    code'",
             "`code`, <code>code</code>",
+            "`pull_request_target`, <code>pull_request_target</code>",
+            "`GITHUB_OUTPUT`, <code>GITHUB_OUTPUT</code>",
+            "`pull_request_target` and `GITHUB_OUTPUT`, <code>pull_request_target</code> and <code>GITHUB_OUTPUT</code>",
+            "`_text_`, <code>_text_</code>",
+            "`__text__`, <code>__text__</code>",
+            "`*text*`, <code>*text*</code>",
+            "`**text**`, <code>**text**</code>",
+            "`$foo_bar`, <code>$foo_bar</code>",
+            "`\\d+_test`, <code>\\d+_test</code>",
+            "`<xml_tag>`, <code>&lt;xml_tag&gt;</code>",
+            "`[text](https://example.com)`, <code>[text](https://example.com)</code>",
+            "`pull_request_target` and _emphasis_ and **bold**, <code>pull_request_target</code> and <em>emphasis</em> and <strong>bold</strong>",
+            "_emphasis_ `code_with_underscore` *emphasis*, <em>emphasis</em> <code>code_with_underscore</code> <em>emphasis</em>",
             "[text](https://example.com), <a href=\"https://example.com\">text</a>",
             "[ text ](https://example.com), <a href=\"https://example.com\"> text </a>",
             "prefix [text](https://example.com) suffix, prefix <a href=\"https://example.com\">text</a> suffix",
@@ -47,7 +65,32 @@ void assertConverted(final String inputLine, final String expected)
         assertThat(MarkdownSpanConverter.convertLineContent(inputLine), equalTo(expected));
     }
 
+    @Test
+    void testZeroCharactersStrippedBeforeRendering()
+    {
+        assertThat(MarkdownSpanConverter.convertLineContent("x \u00000\u0000 `y`"),
+                equalTo("x 0 <code>y</code>"));
+        assertThat(MarkdownSpanConverter.convertLineContent("`a` \u00001\u0000 `b`"),
+                equalTo("<code>a</code> 1 <code>b</code>"));
+    }
+
     // [utest->dsn~reporting.html.escape-html~1]
+    @ParameterizedTest(name = "Line ''{0}'' converted to HTML ''{1}''")
+    @MethodSource("provideSpecialTestCases")
+    void assertConvertedSpecial(final String inputLine, final String expected)
+    {
+        assertThat(MarkdownSpanConverter.convertLineContent(inputLine), equalTo(expected));
+    }
+
+    private static java.util.stream.Stream<Arguments> provideSpecialTestCases()
+    {
+        return java.util.stream.Stream.of(
+            Arguments.of("`a`, `b`0`c`", "<code>a</code>, <code>b</code>0<code>c</code>"),
+            Arguments.of("`x` `y`0`z`", "<code>x</code> <code>y</code>0<code>z</code>"),
+            Arguments.of("    indented\n0`backtick`", "<pre>    indented\n</pre>0<code>backtick</code>")
+        );
+    }
+
     @ParameterizedTest(name = "Line ''{0}'' escaped as HTML ''{1}''")
     @CsvSource(
     {

reporter/html/src/test/java/org/itsallcode/openfasttrace/report/html/view/html/TestMarkdownConverter.java

@@ -149,6 +149,13 @@ void testConvertCode()
                 "<p>This text <code>contains code </code> and regular text</p>");
     }
 
+    @Test
+    void testConvertCodeWithUnderscores()
+    {
+        assertConverted("Workflows that run in `pull_request_target` context must not write `GITHUB_OUTPUT`.",
+                "<p>Workflows that run in <code>pull_request_target</code> context must not write <code>GITHUB_OUTPUT</code>.</p>");
+    }
+
     @Test
     void testConvertLink()
     {