From 40fffbb6c71c418f515356ec39ab468fe80ec039 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Wed, 23 Sep 2026 19:28:15 +0200 Subject: [PATCH 1/2] #2093 LLM text extractor: keep page markers out of the prompt and clean the reply replacePlaceholders() substituted the page HTML into the template with a plain String.replace. Jsoup writes script and style contents and comments out verbatim, so a page could carry the template's own marker lines, such as <|HTML_CONTENT_END|>, and open its own instruction section. The marker tokens of the form <|NAME|> found in the configured template are now removed from the HTML before it is substituted, which also covers custom templates. {REQUEST} is substituted before {HTML}, so a page containing {REQUEST} no longer pulls in the configured request. text() returned the reply as it came. It now takes the content of the envelope the default prompt asks for, or the whole reply if there is none, keeps only its text nodes so that no markup is returned, and truncates it to textextractor.llm.text.maxlength (default -1, no limit). The default prompt no longer tells the model to ignore its guidelines when a user instruction is present. An operator who wants that can say so in textextractor.llm.user_request or a custom template. --- docs/src/main/asciidoc/configuration.adoc | 3 +- external/ai/README.md | 7 + external/ai/ai-conf.yaml | 6 + .../ai/AbstractLLMTextExtractor.java | 98 +++++++++++- .../src/main/resources/llm-default-prompt.txt | 2 - .../ai/LLMTextExtractorPromptTest.java | 149 ++++++++++++++++++ 6 files changed, 258 insertions(+), 7 deletions(-) create mode 100644 external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index ec59cb4ab..1cfb193e3 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -612,8 +612,9 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/ai[ai mod | textextractor.llm.url | - | LLM API endpoint URL (e.g., OpenAI or Ollama endpoint). | textextractor.llm.model | - | Model name to use (e.g., "gpt-4", "llama2"). | textextractor.system.prompt | - | System prompt for the LLM (optional). -| textextractor.llm.prompt | - | User prompt template. Use `\{HTML}` and `\{REQUEST}` as placeholders (optional). +| textextractor.llm.prompt | - | User prompt template. Use `\{HTML}` and `\{REQUEST}` as placeholders (optional). Marker tokens of the form `<\|NAME\|>` in the template are removed from the page HTML before it is substituted. | textextractor.llm.user_request | - | Extra user request passed to the prompt template (optional). +| textextractor.llm.text.maxlength | -1 | Maximum number of characters of extracted text, -1 for no limit. The text is taken from the `` envelope of the reply if there is one, and any markup is removed. | textextractor.llm.listener.clazz | - | Listener class for tracking LLM response metrics (optional). |=== diff --git a/external/ai/README.md b/external/ai/README.md index a89fb6440..db65d456f 100644 --- a/external/ai/README.md +++ b/external/ai/README.md @@ -50,10 +50,17 @@ textextractor.llm.prompt: | # Optional: extra request passed into the user prompt textextractor.llm.user_request: "Only include body content relevant to articles." +# Optional: maximum number of characters of extracted text, -1 (default) for no limit +textextractor.llm.text.maxlength: 100000 + # Optional: listener class implementing LlmResponseListener to hook into success/failure of LLM response, i.e. for tracking usage metrics. textextractor.llm.listener.clazz: "" ``` +Marker tokens of the form `<|NAME|>` that appear in the prompt template, such as `<|HTML_CONTENT_END|>` in the default one, are removed from the page HTML before it is substituted, so that a page cannot open or close a section of the prompt. If your own template delimits its sections, use markers of that form. + +The text returned is the content of the `…` envelope that the default prompt asks for, or the whole reply if it has none. Any HTML markup left in it is removed, so the result contains only text, as with the default `TextExtractor`. + Note: You **must** set `textextractor.class` to use this extractor in a StormCrawler topology. The `LlmTextExtractor` does not support the following configuration options from the default `TextExtractor`: diff --git a/external/ai/ai-conf.yaml b/external/ai/ai-conf.yaml index 9177a01c1..2daf611a7 100644 --- a/external/ai/ai-conf.yaml +++ b/external/ai/ai-conf.yaml @@ -27,8 +27,14 @@ textextractor.llm.model: "" # Allows to define a custom prompt. {HTML} is replaced with the page html, {REQUEST} is replaced with the content of textextractor.llm.user_request #textextractor.llm.prompt: "see llm-default-prompt.txt - can be a multi line string with placeholders" +# Marker tokens of the form <|NAME|> found in the prompt are removed from the page html before it is substituted. + # Allows to configure a special user request which the LLM should honour. #textextractor.llm.user_request: "-" +# Maximum number of characters of extracted text, -1 for no limit. The text is taken from the envelope +# of the reply if there is one, with any markup removed, before it is truncated. +#textextractor.llm.text.maxlength: -1 + # Allows to define the listener class to have the possibility to hook in usage metrics (i.e. for payment related metrics) #textextractor.llm.listener.clazz: "org.apache.stormcrawler.ai.listener.NoOpListener" diff --git a/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java b/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java index c5ec1a187..24698f679 100644 --- a/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java +++ b/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java @@ -27,12 +27,18 @@ import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.stormcrawler.ai.listener.LlmResponseListener; import org.apache.stormcrawler.ai.listener.NoOpListener; import org.apache.stormcrawler.parse.TextExtractor; import org.apache.stormcrawler.util.ConfUtils; import org.jsoup.nodes.Element; +import org.jsoup.nodes.TextNode; +import org.jsoup.parser.Parser; /** * Abstract base class for LLM-based text extractors that use a {@link ChatModel} to convert HTML @@ -51,11 +57,20 @@ public abstract class AbstractLLMTextExtractor implements TextExtractor { public static final String USER_PROMPT = "textextractor.llm.prompt"; public static final String USER_REQUEST = "textextractor.llm.user_request"; public static final String LISTENER_CLASS = "textextractor.llm.listener.clazz"; + public static final String TEXT_MAX_LENGTH = "textextractor.llm.text.maxlength"; + + /** marker tokens such as {@code <|HTML_CONTENT_END|>} used to delimit sections of a prompt */ + private static final Pattern MARKER_PATTERN = Pattern.compile("<\\|[^|<>\\s]+\\|>"); + + private static final String CONTENT_START = ""; + private static final String CONTENT_END = ""; private final ChatModel model; private final SystemMessage systemMessage; private final String userMessage; private final String userRequest; + private final Set markers; + private final int textMaxLength; private final LlmResponseListener listener; /** @@ -76,6 +91,8 @@ public AbstractLLMTextExtractor(Map stormConf) { ConfUtils.getString( stormConf, USER_PROMPT, readFromClasspath("llm-default-prompt.txt")); this.userRequest = ConfUtils.getString(stormConf, USER_REQUEST, ""); + this.markers = findMarkers(userMessage); + this.textMaxLength = ConfUtils.getInt(stormConf, TEXT_MAX_LENGTH, -1); final String clazz = ConfUtils.getString(stormConf, LISTENER_CLASS, NoOpListener.class.getName()); try { @@ -123,6 +140,10 @@ protected String readFromClasspath(String resource) { /** * Extracts text from a given JSoup {@link Element} by sending a prompt to the LLM model. * + *

The reply is reduced to the content of its {@code } envelope when it has one, any + * markup left in it is removed and it is truncated to the length set with {@value + * #TEXT_MAX_LENGTH}, if any. + * * @param element an {@link Element} representing a portion of HTML * @return the LLM-extracted plain text or an empty string on failure */ @@ -139,7 +160,7 @@ public String text(Object element) { .build(); final ChatResponse response = model.chat(chatRequest); listener.onResponse(response); - return response.aiMessage().text(); + return cleanReply(response.aiMessage().text()); } catch (RuntimeException ex) { listener.onFailure(element, ex); } @@ -149,15 +170,84 @@ public String text(Object element) { /** * Replaces placeholders in the user message template with the actual HTML content and user - * request. + * request. Marker tokens of the template, such as {@code <|HTML_CONTENT_END|>}, are removed + * from the HTML first so that the page cannot close or open a section of the prompt. * * @param userMessage the original user message template * @param html the HTML string to insert * @return the updated user message string with placeholders replaced */ protected String replacePlaceholders(String userMessage, String html) { - userMessage = userMessage.replace("{HTML}", html); + // the request is substituted first so that a {REQUEST} in the page is left as it is userMessage = userMessage.replace("{REQUEST}", userRequest); - return userMessage; + return userMessage.replace("{HTML}", removeMarkers(html)); + } + + /** + * Returns the text of a reply: the content of its {@code } envelope if there is one, + * or the whole reply otherwise, without markup and truncated to the length set with {@value + * #TEXT_MAX_LENGTH}. + * + * @param reply the text returned by the model + * @return the extracted text + */ + protected String cleanReply(String reply) { + if (reply == null) { + return ""; + } + final int start = reply.indexOf(CONTENT_START); + if (start >= 0) { + final int end = reply.lastIndexOf(CONTENT_END); + final int from = start + CONTENT_START.length(); + reply = end >= from ? reply.substring(from, end) : reply.substring(from); + } + String text = stripMarkup(reply).strip(); + if (textMaxLength >= 0 && text.length() > textMaxLength) { + int cut = textMaxLength; + if (cut > 0 && Character.isHighSurrogate(text.charAt(cut - 1))) { + cut--; + } + text = text.substring(0, cut); + } + return text; + } + + /** keeps the text nodes of the input and their line breaks, dropping elements and comments */ + private static String stripMarkup(String text) { + final StringBuilder sb = new StringBuilder(text.length()); + Parser.htmlParser() + .parseInput(text, "") + .body() + .traverse( + (node, depth) -> { + if (node instanceof TextNode t) { + sb.append(t.getWholeText()); + } + }); + return sb.toString(); + } + + private String removeMarkers(String html) { + if (markers.isEmpty()) { + return html; + } + // repeat, as removing one marker can join the pieces of another + String previous; + do { + previous = html; + for (String marker : markers) { + html = html.replace(marker, ""); + } + } while (!html.equals(previous)); + return html; + } + + private static Set findMarkers(String template) { + final Set found = new LinkedHashSet<>(); + final Matcher m = MARKER_PATTERN.matcher(template); + while (m.find()) { + found.add(m.group()); + } + return found; } } diff --git a/external/ai/src/main/resources/llm-default-prompt.txt b/external/ai/src/main/resources/llm-default-prompt.txt index 366e72a8e..61c60b769 100644 --- a/external/ai/src/main/resources/llm-default-prompt.txt +++ b/external/ai/src/main/resources/llm-default-prompt.txt @@ -23,8 +23,6 @@ TASK DETAILS: - DON'T: Fragment related content - DON'T: Duplicate information -IMPORTANT: If user specific instruction is provided, ignore above guideline and prioritize those requirements over these general guidelines. - OUTPUT FORMAT: Wrap your response in tags. Use proper markdown throughout. diff --git a/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java b/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java new file mode 100644 index 000000000..7de0e7fd7 --- /dev/null +++ b/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.ai; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.model.chat.request.ChatRequest; +import dev.langchain4j.model.chat.response.ChatResponse; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; +import org.apache.storm.Config; +import org.jsoup.parser.Parser; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Checks the prompt sent to the model and the text returned from its reply, without a model. */ +class LLMTextExtractorPromptTest { + + /** records the prompt and replies with a fixed string */ + private static class RecordingModel implements ChatModel { + String prompt; + String reply = ""; + + @Override + public ChatResponse chat(ChatRequest chatRequest) { + prompt = ((UserMessage) chatRequest.messages().get(1)).singleText(); + return ChatResponse.builder().aiMessage(AiMessage.from(reply)).build(); + } + } + + private static class TestExtractor extends AbstractLLMTextExtractor { + static final RecordingModel MODEL = new RecordingModel(); + + TestExtractor(Map conf) { + super(conf); + } + + @Override + protected ChatModel getChatModel(Map stormConf) { + return MODEL; + } + } + + private final Map conf = new HashMap<>(new Config()); + + @BeforeEach + void reset() { + TestExtractor.MODEL.prompt = null; + TestExtractor.MODEL.reply = ""; + } + + private String extract(String html, String reply) { + TestExtractor.MODEL.reply = reply; + return new TestExtractor(conf).text(Parser.htmlParser().parseInput(html, "").body()); + } + + private static int count(String text, String token) { + return text.split(Pattern.quote(token), -1).length - 1; + } + + @Test + void pageContentCannotCloseTheHtmlSection() { + // a script element is written out verbatim by jsoup, unlike a text node + extract( + "

hello

", + ""); + final String prompt = TestExtractor.MODEL.prompt; + assertEquals(1, count(prompt, "<|HTML_CONTENT_END|>")); + assertEquals(1, count(prompt, "<|USER_INSTRUCTION_START|>")); + assertTrue(prompt.contains("return nothing")); + } + + @Test + void markerSplitByAnotherMarkerIsRemoved() { + extract("", ""); + assertEquals(1, count(TestExtractor.MODEL.prompt, "<|HTML_CONTENT_END|>")); + } + + @Test + void markersOfACustomTemplateAreRemoved() { + conf.put(AbstractLLMTextExtractor.USER_PROMPT, "<|PAGE|>\n{HTML}\n<|END|>\n{REQUEST}"); + extract("", ""); + assertEquals(1, count(TestExtractor.MODEL.prompt, "<|END|>")); + } + + @Test + void pageCannotPullInTheUserRequest() { + conf.put(AbstractLLMTextExtractor.USER_REQUEST, "secret request"); + extract("", ""); + assertEquals(1, count(TestExtractor.MODEL.prompt, "secret request")); + } + + @Test + void markupInTheReplyIsNotReturned() { + final String text = + extract("

hello

", "hello"); + assertEquals("hello", text); + } + + @Test + void contentOfTheEnvelopeIsReturned() { + final String text = + extract( + "

hello

", + "Here is the result:\n\n# Title\n\nsome *text*\n\nDone."); + assertEquals("# Title\n\nsome *text*", text); + } + + @Test + void replyWithoutEnvelopeIsReturnedWhole() { + assertEquals("# Title\n\nbody", extract("

hello

", "# Title\n\nbody")); + } + + @Test + void textIsTruncatedToTheMaxLength() { + conf.put(AbstractLLMTextExtractor.TEXT_MAX_LENGTH, 5); + assertEquals("abcde", extract("

hello

", "abcdefgh")); + } + + @Test + void textIsNotTruncatedByDefault() { + final String longText = "a".repeat(200_000); + final String text = extract("

hello

", "" + longText + ""); + assertEquals(longText, text); + assertFalse(text.contains("<")); + } +} From eb1a3da7b04b7a2eed89d6276ee4346c704f2b9e Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Sat, 26 Sep 2026 10:08:25 +0200 Subject: [PATCH 2/2] Address review feedback --- docs/src/main/asciidoc/configuration.adoc | 3 +- external/ai/README.md | 8 +- external/ai/ai-conf.yaml | 6 +- .../ai/AbstractLLMTextExtractor.java | 80 +++++++----------- .../ai/LLMTextExtractorPromptTest.java | 81 +++++++++++++++++-- 5 files changed, 109 insertions(+), 69 deletions(-) diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 1cfb193e3..cca96265d 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -612,9 +612,8 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/ai[ai mod | textextractor.llm.url | - | LLM API endpoint URL (e.g., OpenAI or Ollama endpoint). | textextractor.llm.model | - | Model name to use (e.g., "gpt-4", "llama2"). | textextractor.system.prompt | - | System prompt for the LLM (optional). -| textextractor.llm.prompt | - | User prompt template. Use `\{HTML}` and `\{REQUEST}` as placeholders (optional). Marker tokens of the form `<\|NAME\|>` in the template are removed from the page HTML before it is substituted. +| textextractor.llm.prompt | - | User prompt template. Use `\{HTML}` and `\{REQUEST}` as placeholders (optional). `<\|` in the page HTML is written as `< \|` before it is substituted, so the page cannot contain a marker token of the form `<\|NAME\|>`. The text returned is the content of the `` envelope of the reply if there is one, without the markup outside fenced code blocks; `textextractor.skip.after` limits its length. | textextractor.llm.user_request | - | Extra user request passed to the prompt template (optional). -| textextractor.llm.text.maxlength | -1 | Maximum number of characters of extracted text, -1 for no limit. The text is taken from the `` envelope of the reply if there is one, and any markup is removed. | textextractor.llm.listener.clazz | - | Listener class for tracking LLM response metrics (optional). |=== diff --git a/external/ai/README.md b/external/ai/README.md index db65d456f..3872daea5 100644 --- a/external/ai/README.md +++ b/external/ai/README.md @@ -50,16 +50,13 @@ textextractor.llm.prompt: | # Optional: extra request passed into the user prompt textextractor.llm.user_request: "Only include body content relevant to articles." -# Optional: maximum number of characters of extracted text, -1 (default) for no limit -textextractor.llm.text.maxlength: 100000 - # Optional: listener class implementing LlmResponseListener to hook into success/failure of LLM response, i.e. for tracking usage metrics. textextractor.llm.listener.clazz: "" ``` -Marker tokens of the form `<|NAME|>` that appear in the prompt template, such as `<|HTML_CONTENT_END|>` in the default one, are removed from the page HTML before it is substituted, so that a page cannot open or close a section of the prompt. If your own template delimits its sections, use markers of that form. +`<|` in the page HTML is written as `< |` before it is substituted, so that a page cannot open or close a section of the prompt with a marker token such as `<|HTML_CONTENT_END|>` in the default template. If your own template delimits its sections, use markers of the form `<|NAME|>`. -The text returned is the content of the `…` envelope that the default prompt asks for, or the whole reply if it has none. Any HTML markup left in it is removed, so the result contains only text, as with the default `TextExtractor`. +The text returned is the content of the `…` envelope that the default prompt asks for, or the whole reply if it has none. Any HTML markup outside fenced code blocks is removed; the code blocks are kept as they are. `textextractor.skip.after` limits its length, as for the default `TextExtractor`. Note: You **must** set `textextractor.class` to use this extractor in a StormCrawler topology. @@ -68,7 +65,6 @@ The `LlmTextExtractor` does not support the following configuration options from - `textextractor.include.pattern` - `textextractor.exclude.tags` - `textextractor.no.text` -- `textextractor.skip.after` ## Additional Notes - **LLM Costs:** Calls to LLM APIs may incur costs - monitor usage if billing is a concern. In addition, certain providers might impose **rate limits**, which are not (yet) handled by our implementation as it is vendor specific behaviour. diff --git a/external/ai/ai-conf.yaml b/external/ai/ai-conf.yaml index 2daf611a7..044bede6c 100644 --- a/external/ai/ai-conf.yaml +++ b/external/ai/ai-conf.yaml @@ -27,14 +27,10 @@ textextractor.llm.model: "" # Allows to define a custom prompt. {HTML} is replaced with the page html, {REQUEST} is replaced with the content of textextractor.llm.user_request #textextractor.llm.prompt: "see llm-default-prompt.txt - can be a multi line string with placeholders" -# Marker tokens of the form <|NAME|> found in the prompt are removed from the page html before it is substituted. +# <| in the page html is written as < | before it is substituted, so the page cannot contain a marker token of the form <|NAME|>. # Allows to configure a special user request which the LLM should honour. #textextractor.llm.user_request: "-" -# Maximum number of characters of extracted text, -1 for no limit. The text is taken from the envelope -# of the reply if there is one, with any markup removed, before it is truncated. -#textextractor.llm.text.maxlength: -1 - # Allows to define the listener class to have the possibility to hook in usage metrics (i.e. for payment related metrics) #textextractor.llm.listener.clazz: "org.apache.stormcrawler.ai.listener.NoOpListener" diff --git a/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java b/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java index 24698f679..17152a442 100644 --- a/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java +++ b/external/ai/src/main/java/org/apache/stormcrawler/ai/AbstractLLMTextExtractor.java @@ -27,9 +27,7 @@ import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; -import java.util.LinkedHashSet; import java.util.Map; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.stormcrawler.ai.listener.LlmResponseListener; @@ -37,7 +35,6 @@ import org.apache.stormcrawler.parse.TextExtractor; import org.apache.stormcrawler.util.ConfUtils; import org.jsoup.nodes.Element; -import org.jsoup.nodes.TextNode; import org.jsoup.parser.Parser; /** @@ -57,10 +54,16 @@ public abstract class AbstractLLMTextExtractor implements TextExtractor { public static final String USER_PROMPT = "textextractor.llm.prompt"; public static final String USER_REQUEST = "textextractor.llm.user_request"; public static final String LISTENER_CLASS = "textextractor.llm.listener.clazz"; - public static final String TEXT_MAX_LENGTH = "textextractor.llm.text.maxlength"; - /** marker tokens such as {@code <|HTML_CONTENT_END|>} used to delimit sections of a prompt */ - private static final Pattern MARKER_PATTERN = Pattern.compile("<\\|[^|<>\\s]+\\|>"); + /** + * Fenced code blocks as in CommonMark: a line holding a fence of three or more backticks or + * tildes, indented by at most three spaces, up to a line holding only a fence of the same + * character and at least the same length, or to the end of the text. + */ + private static final Pattern FENCED_CODE_PATTERN = + Pattern.compile( + "^ {0,3}(?:(`{3,})[^`\\n]*|(~{3,})[^\\n]*)$.*?(?:^ {0,3}(?:\\1`*|\\2~*)[ \\t\\r]*$|\\z)", + Pattern.DOTALL | Pattern.MULTILINE | Pattern.UNIX_LINES); private static final String CONTENT_START = ""; private static final String CONTENT_END = ""; @@ -69,7 +72,6 @@ public abstract class AbstractLLMTextExtractor implements TextExtractor { private final SystemMessage systemMessage; private final String userMessage; private final String userRequest; - private final Set markers; private final int textMaxLength; private final LlmResponseListener listener; @@ -91,8 +93,7 @@ public AbstractLLMTextExtractor(Map stormConf) { ConfUtils.getString( stormConf, USER_PROMPT, readFromClasspath("llm-default-prompt.txt")); this.userRequest = ConfUtils.getString(stormConf, USER_REQUEST, ""); - this.markers = findMarkers(userMessage); - this.textMaxLength = ConfUtils.getInt(stormConf, TEXT_MAX_LENGTH, -1); + this.textMaxLength = ConfUtils.getInt(stormConf, TEXT_MAX_TEXT_PARAM_NAME, -1); final String clazz = ConfUtils.getString(stormConf, LISTENER_CLASS, NoOpListener.class.getName()); try { @@ -141,8 +142,8 @@ protected String readFromClasspath(String resource) { * Extracts text from a given JSoup {@link Element} by sending a prompt to the LLM model. * *

The reply is reduced to the content of its {@code } envelope when it has one, any - * markup left in it is removed and it is truncated to the length set with {@value - * #TEXT_MAX_LENGTH}, if any. + * markup outside fenced code blocks is removed and it is truncated to the length set with + * {@value #TEXT_MAX_TEXT_PARAM_NAME}, if any. * * @param element an {@link Element} representing a portion of HTML * @return the LLM-extracted plain text or an empty string on failure @@ -170,8 +171,9 @@ public String text(Object element) { /** * Replaces placeholders in the user message template with the actual HTML content and user - * request. Marker tokens of the template, such as {@code <|HTML_CONTENT_END|>}, are removed - * from the HTML first so that the page cannot close or open a section of the prompt. + * request. Every {@code <|} in the HTML is written as {@code < |} first, so the page cannot + * hold a marker token such as {@code <|HTML_CONTENT_END|>} and close or open a section of the + * prompt. * * @param userMessage the original user message template * @param html the HTML string to insert @@ -180,13 +182,13 @@ public String text(Object element) { protected String replacePlaceholders(String userMessage, String html) { // the request is substituted first so that a {REQUEST} in the page is left as it is userMessage = userMessage.replace("{REQUEST}", userRequest); - return userMessage.replace("{HTML}", removeMarkers(html)); + return userMessage.replace("{HTML}", html.replace("<|", "< |")); } /** * Returns the text of a reply: the content of its {@code } envelope if there is one, - * or the whole reply otherwise, without markup and truncated to the length set with {@value - * #TEXT_MAX_LENGTH}. + * or the whole reply otherwise, without markup outside fenced code blocks and truncated to the + * length set with {@value #TEXT_MAX_TEXT_PARAM_NAME}. * * @param reply the text returned by the model * @return the extracted text @@ -202,9 +204,9 @@ protected String cleanReply(String reply) { reply = end >= from ? reply.substring(from, end) : reply.substring(from); } String text = stripMarkup(reply).strip(); - if (textMaxLength >= 0 && text.length() > textMaxLength) { + if (textMaxLength > 0 && text.length() > textMaxLength) { int cut = textMaxLength; - if (cut > 0 && Character.isHighSurrogate(text.charAt(cut - 1))) { + if (Character.isHighSurrogate(text.charAt(cut - 1))) { cut--; } text = text.substring(0, cut); @@ -212,42 +214,20 @@ protected String cleanReply(String reply) { return text; } - /** keeps the text nodes of the input and their line breaks, dropping elements and comments */ + /** drops the markup outside fenced code blocks, which are kept as they are */ private static String stripMarkup(String text) { final StringBuilder sb = new StringBuilder(text.length()); - Parser.htmlParser() - .parseInput(text, "") - .body() - .traverse( - (node, depth) -> { - if (node instanceof TextNode t) { - sb.append(t.getWholeText()); - } - }); - return sb.toString(); - } - - private String removeMarkers(String html) { - if (markers.isEmpty()) { - return html; + final Matcher code = FENCED_CODE_PATTERN.matcher(text); + int last = 0; + while (code.find()) { + sb.append(textOf(text.substring(last, code.start()))).append(code.group()); + last = code.end(); } - // repeat, as removing one marker can join the pieces of another - String previous; - do { - previous = html; - for (String marker : markers) { - html = html.replace(marker, ""); - } - } while (!html.equals(previous)); - return html; + return sb.append(textOf(text.substring(last))).toString(); } - private static Set findMarkers(String template) { - final Set found = new LinkedHashSet<>(); - final Matcher m = MARKER_PATTERN.matcher(template); - while (m.find()) { - found.add(m.group()); - } - return found; + /** keeps the text of the input and its line breaks, dropping elements and comments */ + private static String textOf(String html) { + return Parser.parseBodyFragment(html, "").body().wholeText(); } } diff --git a/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java b/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java index 7de0e7fd7..f84d8f4f7 100644 --- a/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java +++ b/external/ai/src/test/java/org/apache/stormcrawler/ai/LLMTextExtractorPromptTest.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.storm.Config; +import org.apache.stormcrawler.parse.TextExtractor; import org.jsoup.parser.Parser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -93,22 +94,30 @@ void pageContentCannotCloseTheHtmlSection() { } @Test - void markerSplitByAnotherMarkerIsRemoved() { - extract("", ""); + void markerSplitByAnotherMarkerIsNeutralised() { + extract("

hello

", ""); assertEquals(1, count(TestExtractor.MODEL.prompt, "<|HTML_CONTENT_END|>")); } @Test - void markersOfACustomTemplateAreRemoved() { + void markersOfACustomTemplateAreNeutralised() { conf.put(AbstractLLMTextExtractor.USER_PROMPT, "<|PAGE|>\n{HTML}\n<|END|>\n{REQUEST}"); - extract("", ""); + extract("

hello

", ""); assertEquals(1, count(TestExtractor.MODEL.prompt, "<|END|>")); } + @Test + void anyMarkerInThePageIsNeutralised() { + // not a marker of the template, but a chat token some models act on + extract("

hello

", ""); + assertFalse(TestExtractor.MODEL.prompt.contains("<|im_start|>")); + assertTrue(TestExtractor.MODEL.prompt.contains("< |im_start|>")); + } + @Test void pageCannotPullInTheUserRequest() { conf.put(AbstractLLMTextExtractor.USER_REQUEST, "secret request"); - extract("", ""); + extract("

hello

", ""); assertEquals(1, count(TestExtractor.MODEL.prompt, "secret request")); } @@ -119,6 +128,66 @@ void markupInTheReplyIsNotReturned() { assertEquals("hello", text); } + @Test + void codeInTheReplyIsKept() { + final String text = + extract( + "

hello

", + "Use a list here:\n\n```java\nList l;\n```\n\n" + + "done"); + assertEquals("Use a list here:\n\n```java\nList l;\n```\n\ndone", text); + } + + @Test + void tildeFencesAreKept() { + final String reply = "~~~\nList l;\n~~~"; + assertEquals(reply, extract("

hello

", "" + reply + "")); + } + + @Test + void fenceWithAnInfoStringDoesNotCloseABlock() { + final String reply = "```text\n```java\n
keep
\n```"; + assertEquals(reply, extract("

hello

", "" + reply + "")); + } + + @Test + void fenceIndentedInAListItemIsKept() { + final String reply = "1. Step\n ```java\n List a;\n ```"; + assertEquals(reply, extract("

hello

", "" + reply + "")); + } + + @Test + void markupAfterWhatIsNotAFencedBlockIsRemoved() { + final String img = ""; + for (String reply : + new String[] { + // indented closing fence + "```\ncode\n ```\n" + img, + // a backtick in the info string makes it inline code + "```x``` and " + img, + // a closing fence holds only the fence character + "~~~\na\n~~~`\n~~~\n" + img, + // only \n ends a line + "Text" + Character.toString(0x2028) + "```\n" + img + }) { + assertFalse( + extract("

hello

", "" + reply + "").contains("hello

", "" + reply + "")); + } + + @Test + void strayFenceDoesNotKeepMarkup() { + final String text = extract("

hello

", "text ``` bold"); + assertFalse(text.contains("")); + assertTrue(text.contains("bold")); + } + @Test void contentOfTheEnvelopeIsReturned() { final String text = @@ -135,7 +204,7 @@ void replyWithoutEnvelopeIsReturnedWhole() { @Test void textIsTruncatedToTheMaxLength() { - conf.put(AbstractLLMTextExtractor.TEXT_MAX_LENGTH, 5); + conf.put(TextExtractor.TEXT_MAX_TEXT_PARAM_NAME, 5); assertEquals("abcde", extract("

hello

", "abcdefgh")); }