diff --git a/src/inscriptis/__init__.py b/src/inscriptis/__init__.py index 52f2e6f..e1f4aa8 100644 --- a/src/inscriptis/__init__.py +++ b/src/inscriptis/__init__.py @@ -65,7 +65,7 @@ from typing import TYPE_CHECKING, Any from lxml.etree import ParserError -from lxml.html import HtmlElement, fromstring +from lxml.html import HtmlElement, document_fromstring from inscriptis.html_engine import Inscriptis @@ -93,10 +93,14 @@ def _get_html_tree(html_content: str) -> HtmlElement | None: if html_content.startswith(" or wrapper that the CSS profiles style, adding indentation + # or whitespace that is not part of the input. try: - return fromstring(html_content) + return document_fromstring(html_content) except ParserError: - return fromstring("
" + html_content + "
") + return document_fromstring("
" + html_content + "
") def get_text(html_content: str, config: ParserConfig | None = None) -> str: diff --git a/tests/test_broken_table_handling.py b/tests/test_broken_table_handling.py index d71bd5b..ad8ec2c 100644 --- a/tests/test_broken_table_handling.py +++ b/tests/test_broken_table_handling.py @@ -20,4 +20,4 @@ def test_forgotten_td_close_tag(): # two lines (i.e. missing before the and before the html = "hallo
12
34
echo" print(html) - assert get_text(html, config) == "hallo\n1 2\n3 4\n\necho" + assert get_text(html, config) == "hallo1 2\n3 4\n\necho" diff --git a/tests/test_fragment_wrapper.py b/tests/test_fragment_wrapper.py new file mode 100644 index 0000000..a2a95e5 --- /dev/null +++ b/tests/test_fragment_wrapper.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +"""Ensures that HTML fragments are converted without a synthetic wrapper element. + +lxml.html.fromstring wraps fragments that consist of multiple root elements or +of top-level text in a synthetic
or . The relaxed CSS profile styles +these elements (indentation for div, surrounding spaces for span), which must not +leak into the text representation. +""" + +import pytest + +from inscriptis import get_annotated_text, get_text +from inscriptis.css_profiles import CSS_PROFILES +from inscriptis.model.config import ParserConfig + +STRICT = ParserConfig(css=CSS_PROFILES["strict"]) + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

a

x", "a\n\nx"), + ("

a

b

", "a\n\nb\n"), + ("
a
b
", " a\n b"), + ("ab", "ab"), + ("boldtext", "boldtext"), + ("unbelievable", "unbelievable"), + ], +) +def test_fragment_without_wrapper(html, expected): + assert get_text(html) == expected + + +@pytest.mark.parametrize( + "html", + [ + "

a

x", + "

a

b

", + "
a
b
", + "ab", + "unbelievable", + "ab", + "

a

", + ], +) +@pytest.mark.parametrize("config", [None, STRICT], ids=["relaxed", "strict"]) +def test_fragment_equals_body_content(html, config): + """A fragment yields the same text as the same markup inside an explicit .""" + assert get_text(html, config) == get_text(f"{html}", config) + + +def test_fragment_annotations(): + config = ParserConfig(annotation_rules={"b": ["bold"]}) + result = get_annotated_text("unbelievable", config) + assert result["text"] == "unbelievable" + assert result["label"] == [(0, 2, "bold")] + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ("

a

", "a\n"), + ("
a
", " a"), + ("a", "a"), + ("
  • a
  • b
  • ", "* a\n* b"), + ("

    a

    x", "a\n\nx"), + ("", ""), + (" \n", ""), + ("", ""), + ], +) +def test_unchanged_inputs(html, expected): + """Single elements, full documents and empty input are not affected.""" + assert get_text(html) == expected