From 0096d6f6a66747d4b572f41cc783cc3de8c75982 Mon Sep 17 00:00:00 2001
From: Osama Ansar <osama.ansar.pk@gmail.com>
Date: Wed, 21 Oct 2026 16:38:21 +0500
Subject: [PATCH] fix: do not leak lxml's synthetic fragment wrapper into the
 text output

_get_html_tree used lxml.html.fromstring, which wraps fragments that
consist of multiple root elements or top-level text in a synthetic <div>
(or <span>). The relaxed CSS profile styles these tags, so the wrapper
added indentation (div) or surrounding spaces (span) to the output, e.g.
'<b>un</b>believable' was converted to 'un believable'.

Use lxml.html.document_fromstring instead, which always returns the
<html> root and never creates a synthetic wrapper. Output for single
elements, full documents and the strict profile is unchanged.

tests/test_broken_table_handling.py: the input starts with <body>, which
fromstring turned into a <div>; the expected text is updated accordingly.
---
 src/inscriptis/__init__.py          | 10 ++--
 tests/test_broken_table_handling.py |  2 +-
 tests/test_fragment_wrapper.py      | 75 +++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 4 deletions(-)
 create mode 100644 tests/test_fragment_wrapper.py

diff --git a/src/inscriptis/__init__.py b/src/inscriptis/__init__.py
index 52f2e6f..e1f4aa8 100644
--- a/src/inscriptis/__init__.py
+++ b/src/inscriptis/__init__.py
@@ -65,7 +65,7 @@
 from typing import TYPE_CHECKING, Any
 
 from lxml.etree import ParserError
-from lxml.html import HtmlElement, fromstring
+from lxml.html import HtmlElement, document_fromstring
 
 from inscriptis.html_engine import Inscriptis
 
@@ -93,10 +93,14 @@ def _get_html_tree(html_content: str) -> HtmlElement | None:
     if html_content.startswith("<?xml "):
         html_content = RE_STRIP_XML_DECLARATION.sub("", html_content, count=1)
 
+    # Use document_fromstring rather than fromstring: for fragments with
+    # multiple root elements or top-level text, fromstring returns a synthetic
+    # <div> or <span> wrapper that the CSS profiles style, adding indentation
+    # or whitespace that is not part of the input.
     try:
-        return fromstring(html_content)
+        return document_fromstring(html_content)
     except ParserError:
-        return fromstring("<pre>" + html_content + "</pre>")
+        return document_fromstring("<pre>" + html_content + "</pre>")
 
 
 def get_text(html_content: str, config: ParserConfig | None = None) -> str:
diff --git a/tests/test_broken_table_handling.py b/tests/test_broken_table_handling.py
index d71bd5b..ad8ec2c 100644
--- a/tests/test_broken_table_handling.py
+++ b/tests/test_broken_table_handling.py
@@ -20,4 +20,4 @@ def test_forgotten_td_close_tag():
     # two lines (i.e. missing </td> before the <tr> and before the </table>
     html = "<body>hallo<table><tr><td>1<td>2<tr><td>3<td>4</table>echo</body>"
     print(html)
-    assert get_text(html, config) == "hallo\n1  2\n3  4\n\necho"
+    assert get_text(html, config) == "hallo1  2\n3  4\n\necho"
diff --git a/tests/test_fragment_wrapper.py b/tests/test_fragment_wrapper.py
new file mode 100644
index 0000000..a2a95e5
--- /dev/null
+++ b/tests/test_fragment_wrapper.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+"""Ensures that HTML fragments are converted without a synthetic wrapper element.
+
+lxml.html.fromstring wraps fragments that consist of multiple root elements or
+of top-level text in a synthetic <div> or <span>. The relaxed CSS profile styles
+these elements (indentation for div, surrounding spaces for span), which must not
+leak into the text representation.
+"""
+
+import pytest
+
+from inscriptis import get_annotated_text, get_text
+from inscriptis.css_profiles import CSS_PROFILES
+from inscriptis.model.config import ParserConfig
+
+STRICT = ParserConfig(css=CSS_PROFILES["strict"])
+
+
+@pytest.mark.parametrize(
+    ("html", "expected"),
+    [
+        ("<p>a</p>x", "a\n\nx"),
+        ("<p>a</p><p>b</p>", "a\n\nb\n"),
+        ("<div>a</div><div>b</div>", "  a\n  b"),
+        ("<b>a</b>b", "ab"),
+        ("<b>bold</b>text", "boldtext"),
+        ("<b>un</b>believable", "unbelievable"),
+    ],
+)
+def test_fragment_without_wrapper(html, expected):
+    assert get_text(html) == expected
+
+
+@pytest.mark.parametrize(
+    "html",
+    [
+        "<p>a</p>x",
+        "<p>a</p><p>b</p>",
+        "<div>a</div><div>b</div>",
+        "<b>a</b>b",
+        "<b>un</b>believable",
+        "<td>a</td><td>b</td>",
+        "<p>a</p><!-- comment -->",
+    ],
+)
+@pytest.mark.parametrize("config", [None, STRICT], ids=["relaxed", "strict"])
+def test_fragment_equals_body_content(html, config):
+    """A fragment yields the same text as the same markup inside an explicit <body>."""
+    assert get_text(html, config) == get_text(f"<html><body>{html}</body></html>", config)
+
+
+def test_fragment_annotations():
+    config = ParserConfig(annotation_rules={"b": ["bold"]})
+    result = get_annotated_text("<b>un</b>believable", config)
+    assert result["text"] == "unbelievable"
+    assert result["label"] == [(0, 2, "bold")]
+
+
+@pytest.mark.parametrize(
+    ("html", "expected"),
+    [
+        ("<p>a</p>", "a\n"),
+        ("<div>a</div>", "  a"),
+        ("<span>a</span>", "a"),
+        ("<li>a</li><li>b</li>", "* a\n* b"),
+        ("<html><body><p>a</p>x</body></html>", "a\n\nx"),
+        ("<!-- comment -->", ""),
+        ("  \n", ""),
+        ("", ""),
+    ],
+)
+def test_unchanged_inputs(html, expected):
+    """Single elements, full documents and empty input are not affected."""
+    assert get_text(html) == expected