diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java index bdfbd9fda39..6af199ddaf8 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchAnd.java @@ -7,10 +7,12 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; /** Search expression for AND operator. */ @Getter @@ -23,8 +25,8 @@ public class SearchAnd extends SearchExpression { private final SearchExpression right; @Override - public String toQueryString() { - return left.toQueryString() + " AND " + right.toQueryString(); + public String toQueryString(Function fieldTypeResolver) { + return left.toQueryString(fieldTypeResolver) + " AND " + right.toQueryString(fieldTypeResolver); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java index c429e6f66cc..a0006019b8f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchComparison.java @@ -9,10 +9,12 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.utils.QueryStringUtils; /** Search expression for field comparisons. */ @@ -46,9 +48,11 @@ public String getSymbol() { private final SearchLiteral value; @Override - public String toQueryString() { - String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString()); - String valueStr = value.toQueryString(); + public String toQueryString(Function fieldTypeResolver) { + String rawFieldName = field.getField().toString(); + String fieldName = QueryStringUtils.escapeFieldName(rawFieldName); + ExprType resolvedType = fieldTypeResolver.apply(rawFieldName); + String valueStr = value.toQueryString(resolvedType); switch (operator) { case NOT_EQUALS: return "( _exists_:" + fieldName + " AND NOT " + fieldName + ":" + valueStr + " )"; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java index b705909445f..a68163959c7 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchExpression.java @@ -5,17 +5,33 @@ package org.opensearch.sql.ast.expression; +import java.util.function.Function; import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.data.type.ExprType; /** Base class for search expressions that get converted to query_string syntax. */ public abstract class SearchExpression extends UnresolvedExpression { /** - * Convert this search expression to query_string syntax. + * Convert this search expression to query_string syntax without field-type awareness. * * @return the query string representation */ - public abstract String toQueryString(); + public String toQueryString() { + return toQueryString(f -> null); + } + + /** + * Convert this search expression to query_string syntax, using {@code fieldTypeResolver} to + * resolve the OpenSearch type of a field when the emission depends on whether the field is + * keyword vs. text. When the resolver returns {@code null}, emission falls back to the + * field-type-agnostic form (same as {@link #toQueryString()}). + * + * @param fieldTypeResolver maps a field name to its resolved {@link ExprType}, or null when + * unknown + * @return the query string representation + */ + public abstract String toQueryString(Function fieldTypeResolver); /** * Convert the search expression to anonymized string diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java index 09197202dc0..ebd8ad9b0ef 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchGroup.java @@ -7,10 +7,12 @@ import java.util.Collections; import java.util.List; +import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; /** Search expression for grouped expressions (parentheses). */ @Getter @@ -22,8 +24,8 @@ public class SearchGroup extends SearchExpression { private final SearchExpression expression; @Override - public String toQueryString() { - return "(" + expression.toQueryString() + ")"; + public String toQueryString(Function fieldTypeResolver) { + return "(" + expression.toQueryString(fieldTypeResolver) + ")"; } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java index 8291d130dff..3ce113a5e39 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchIn.java @@ -7,11 +7,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.utils.QueryStringUtils; /** Search expression for IN operator. */ @@ -25,10 +27,12 @@ public class SearchIn extends SearchExpression { private final List values; @Override - public String toQueryString() { + public String toQueryString(Function fieldTypeResolver) { + String rawFieldName = field.getField().toString(); String fieldName = QueryStringUtils.escapeFieldName(field.getField().toString()); + ExprType resolvedType = fieldTypeResolver.apply(rawFieldName); String valueList = - values.stream().map(SearchLiteral::toQueryString).collect(Collectors.joining(" OR ")); + values.stream().map(v -> v.toQueryString(resolvedType)).collect(Collectors.joining(" OR ")); return fieldName + ":( " + valueList + " )"; } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java index 460615afa64..6e184286d4f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchLiteral.java @@ -7,10 +7,12 @@ import java.util.Collections; import java.util.List; +import java.util.function.Function; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.utils.QueryStringUtils; /** Search expression for standalone literals. */ @@ -24,7 +26,23 @@ public class SearchLiteral extends SearchExpression { private final boolean isPhrase; @Override - public String toQueryString() { + public String toQueryString(Function fieldTypeResolver) { + // Unfielded literal: no enclosing field, so no index type. Take the field-agnostic branch. + return toQueryString((ExprType) null); + } + + /** + * Emits the query_string form for a literal on the RHS of {@link SearchComparison} or inside + * {@link SearchIn}. The decision tree is documented in {@code + * docs/dev/ppl-search-command-contract-empirical.md} — briefly: whitespace + wildcard on a + * non-text index escapes the space so the parser keeps the value as one whole-value pattern; + * everything else falls through to phrase (with whitespace) or unquoted-escaped (without). + * + * @param indexType the enclosing field's OpenSearch index-mapping type (text/keyword/...) — used + * only to distinguish text-like from everything else; null means unknown, treated as + * text-like so we don't regress the phrase form. + */ + public String toQueryString(ExprType indexType) { if (literal instanceof Literal) { Literal lit = (Literal) literal; Object val = lit.getValue(); @@ -38,23 +56,52 @@ public String toQueryString() { if (val instanceof String) { String str = (String) val; - // Phrase search - preserve quotes + // [D] whitespace + wildcard on a non-text index: single term with space escaped, so the + // query_string parser keeps the value as one whole-value pattern (a raw space would + // split it into two clauses and drop the field binding on the right half). + if (isPhrase && !isTextLike(indexType) && hasUnescapedWildcard(str)) { + return QueryStringUtils.escapeLuceneSpecialCharacters(str).replace(" ", "\\ "); + } + + // [B]/[C] quoted phrase. if (isPhrase) { - // Escape special chars inside the phrase str = QueryStringUtils.escapeLuceneSpecialCharacters(str); return "\"" + str + "\""; } - // Regular string - escape special characters + // [A] unquoted; escape Lucene specials, wildcards preserved. return QueryStringUtils.escapeLuceneSpecialCharacters(str); } } - // Default: escape the text representation String text = literal.toString(); return QueryStringUtils.escapeLuceneSpecialCharacters(text); } + private static boolean isTextLike(ExprType type) { + if (type == null) { + // Unknown type → treat as text-like so we take the phrase branch and avoid a text + // regression when the resolver fails to identify the field. + return true; + } + String legacyName = type.getOriginalExprType().legacyTypeName(); + return "TEXT".equalsIgnoreCase(legacyName) || "MATCH_ONLY_TEXT".equalsIgnoreCase(legacyName); + } + + private static boolean hasUnescapedWildcard(String s) { + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '\\' && i + 1 < s.length()) { + i++; + continue; + } + if (c == '*' || c == '?') { + return true; + } + } + return false; + } + @Override public String toAnonymizedString() { return "***"; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java index b9ea7b416b4..ed20a59cf48 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchNot.java @@ -7,10 +7,12 @@ import java.util.Collections; import java.util.List; +import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; /** Search expression for NOT operator. */ @Getter @@ -22,8 +24,8 @@ public class SearchNot extends SearchExpression { private final SearchExpression expression; @Override - public String toQueryString() { - return "NOT(" + expression.toQueryString() + ")"; + public String toQueryString(Function fieldTypeResolver) { + return "NOT(" + expression.toQueryString(fieldTypeResolver) + ")"; } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java b/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java index 1a9e95e89a2..258cd09d3eb 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SearchOr.java @@ -7,10 +7,12 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.opensearch.sql.data.type.ExprType; /** Search expression for OR operator. */ @Getter @@ -23,8 +25,8 @@ public class SearchOr extends SearchExpression { private final SearchExpression right; @Override - public String toQueryString() { - return left.toQueryString() + " OR " + right.toQueryString(); + public String toQueryString(Function fieldTypeResolver) { + return left.toQueryString(fieldTypeResolver) + " OR " + right.toQueryString(fieldTypeResolver); } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 9995895bfa1..2afb0eb35fb 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -173,6 +173,7 @@ import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; import org.opensearch.sql.ast.tree.Xyseries; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; import org.opensearch.sql.calcite.plan.AliasFieldsWrappable; import org.opensearch.sql.calcite.plan.HighlightPushDown; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -192,6 +193,7 @@ import org.opensearch.sql.common.patterns.PatternUtils; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; @@ -297,11 +299,33 @@ private RelBuilder scan(RelOptTable tableSchema, CalcitePlanContext context) { public RelNode visitSearch(Search node, CalcitePlanContext context) { // Visit the Relation child to get the scan node.getChild().get(0).accept(this, context); + // Resolve query_string from the structured expression when available so we can consult the + // OpenSearch table's field-type map for per-field text/keyword awareness (e.g. escape + // space + wildcard on keyword vs. quoted phrase on text). Falls back to the pre-computed + // string for callers that never populated the structured expression. + String queryString; + if (node.getOriginalExpression() != null) { + // TODO: index-mapping type (text/keyword) is storage metadata, not a data type — the right + // home is a field/scan annotation on RelDataType, but that needs a Calcite rule-pipeline + // audit (rules rebuild row types and can drop custom fields). For now, unwrap the table + // and read the ExprType map directly. + java.util.Map typesByName = new java.util.HashMap<>(); + RelNode scan = context.relBuilder.peek(); + RelOptTable relOptTable = scan.getTable(); + if (relOptTable != null) { + AbstractOpenSearchTable osTable = relOptTable.unwrap(AbstractOpenSearchTable.class); + if (osTable != null) { + typesByName.putAll(osTable.getFieldTypes()); + } + } + queryString = node.getOriginalExpression().toQueryString(typesByName::get); + } else { + queryString = node.getQueryString(); + } // Create query_string function Function queryStringFunc = AstDSL.function( - "query_string", - AstDSL.unresolvedArg("query", AstDSL.stringLiteral(node.getQueryString()))); + "query_string", AstDSL.unresolvedArg("query", AstDSL.stringLiteral(queryString))); RexNode queryStringRex = rexVisitor.analyze(queryStringFunc, context); context.relBuilder.filter(queryStringRex); diff --git a/core/src/test/java/org/opensearch/sql/ast/expression/SearchLiteralTest.java b/core/src/test/java/org/opensearch/sql/ast/expression/SearchLiteralTest.java new file mode 100644 index 00000000000..6efdb5b8b58 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/ast/expression/SearchLiteralTest.java @@ -0,0 +1,203 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.expression; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.type.ExprType; + +/** + * Field-type-aware emission for {@link SearchLiteral} and {@link SearchComparison}. + * + *

Space + unescaped wildcard on a keyword field must not be phrase-quoted (the phrase form + * silently strips wildcard semantics inside quotes on keyword). On text (or when the type is + * unknown), the legacy phrase form is preserved to avoid regressing today's behavior — see the + * repro matrix in issue #5682. + */ +class SearchLiteralTest { + + /** Stub {@link ExprType} that reports a given legacyTypeName — enough for isTextLike(). */ + private static ExprType typeOf(String legacyName) { + return new ExprType() { + @Override + public String typeName() { + return legacyName; + } + + @Override + public String legacyTypeName() { + return legacyName; + } + }; + } + + private static final ExprType KEYWORD = typeOf("KEYWORD"); + private static final ExprType TEXT = typeOf("TEXT"); + private static final ExprType MATCH_ONLY_TEXT = typeOf("MATCH_ONLY_TEXT"); + + private static SearchLiteral phrase(String value) { + return new SearchLiteral(new Literal(value, DataType.STRING), true); + } + + private static SearchLiteral bare(String value) { + return new SearchLiteral(new Literal(value, DataType.STRING), false); + } + + // ------------------------------------------------------------------------- + // Phrase without wildcard: unchanged on both field types (regression guard). + // ------------------------------------------------------------------------- + + @Test + void phrase_no_wildcard_keyword_stays_quoted() { + assertEquals("\"foo bar\"", phrase("foo bar").toQueryString(KEYWORD)); + } + + @Test + void phrase_no_wildcard_text_stays_quoted() { + assertEquals("\"foo bar\"", phrase("foo bar").toQueryString(TEXT)); + } + + // ------------------------------------------------------------------------- + // Phrase with unescaped wildcard on keyword: emit escaped-space wildcard term. + // These are the D-family rows fixed by issue #5682. + // ------------------------------------------------------------------------- + + @Test + void phrase_trailing_wildcard_keyword_emits_escaped_space_prefix() { + // P6-k: name="foo bar*" → PrefixQuery on whole keyword term + assertEquals("foo\\ bar*", phrase("foo bar*").toQueryString(KEYWORD)); + } + + @Test + void phrase_leading_wildcard_keyword_emits_escaped_space_wildcard() { + // L3-k: name="*foo bar" + assertEquals("*foo\\ bar", phrase("*foo bar").toQueryString(KEYWORD)); + } + + @Test + void phrase_interior_wildcard_keyword_emits_escaped_space_wildcard() { + // I3-k: name="foo *baz" + assertEquals("foo\\ *baz", phrase("foo *baz").toQueryString(KEYWORD)); + } + + @Test + void phrase_question_wildcard_keyword_emits_escaped_space_wildcard() { + // Q5-k: name="foo b?r" + assertEquals("foo\\ b?r", phrase("foo b?r").toQueryString(KEYWORD)); + } + + @Test + void phrase_with_special_chars_and_wildcard_keyword_escapes_all() { + // D3-k repro: name="POST /test-logs/_search*" → PrefixQuery, special chars still escaped. + assertEquals( + "POST\\ \\/test\\-logs\\/_search*", + phrase("POST /test-logs/_search*").toQueryString(KEYWORD)); + } + + // ------------------------------------------------------------------------- + // Phrase with unescaped wildcard on text / match_only_text / unknown: keep + // legacy phrase form (no regression on text; wildcard silently ignored by + // Lucene inside phrase, same as today). + // ------------------------------------------------------------------------- + + @Test + void phrase_wildcard_text_keeps_phrase_form() { + assertEquals("\"foo bar*\"", phrase("foo bar*").toQueryString(TEXT)); + } + + @Test + void phrase_wildcard_match_only_text_keeps_phrase_form() { + assertEquals("\"foo bar*\"", phrase("foo bar*").toQueryString(MATCH_ONLY_TEXT)); + } + + @Test + void phrase_wildcard_unknown_type_keeps_phrase_form() { + // Unknown → treat as text-like so we never regress an unresolvable field. + assertEquals("\"foo bar*\"", phrase("foo bar*").toQueryString((ExprType) null)); + } + + // ------------------------------------------------------------------------- + // Phrase with escaped wildcard: user asked for a LITERAL '*'/'?' — keep the + // phrase form even on keyword. The isPhrase flag was set for a reason. + // ------------------------------------------------------------------------- + + @Test + void phrase_escaped_wildcard_keyword_keeps_phrase_form() { + // Input string literally contains `foo \*` (backslash + '*'). hasUnescapedWildcard() sees the + // backslash and skips '*', so we treat this as a phrase with a literal '*' — no emission + // change. QueryStringUtils.escapeLuceneSpecialCharacters keeps '\\' and '*' untouched. + assertEquals("\"foo \\*\"", phrase("foo \\*").toQueryString(KEYWORD)); + } + + // ------------------------------------------------------------------------- + // Non-phrase (no space): field type does not matter — legacy code path. + // ------------------------------------------------------------------------- + + @Test + void bare_wildcard_keyword_emits_unquoted() { + // P1: name="foo*" — never a phrase, works today. + assertEquals("foo*", bare("foo*").toQueryString(KEYWORD)); + } + + @Test + void bare_wildcard_text_emits_unquoted() { + assertEquals("foo*", bare("foo*").toQueryString(TEXT)); + } + + // ------------------------------------------------------------------------- + // Backwards-compat: arg-less toQueryString() must match the null-type branch + // (legacy phrase form) so callers that never wire the resolver see no change. + // ------------------------------------------------------------------------- + + @Test + void argless_toQueryString_matches_null_type_branch() { + SearchLiteral lit = phrase("foo bar*"); + assertEquals(lit.toQueryString((ExprType) null), lit.toQueryString()); + } + + // ------------------------------------------------------------------------- + // End-to-end via SearchComparison + resolver — the actual planner call path. + // ------------------------------------------------------------------------- + + @Test + void comparison_resolves_field_type_and_emits_wildcard_on_keyword() { + // name="foo bar*" on a keyword field → name:foo\ bar* + SearchComparison cmp = + new SearchComparison( + new Field(new QualifiedName("name"), List.of()), + SearchComparison.Operator.EQUALS, + phrase("foo bar*")); + Function resolver = Map.of("name", KEYWORD)::get; + assertEquals("name:foo\\ bar*", cmp.toQueryString(resolver)); + } + + @Test + void comparison_resolves_field_type_and_keeps_phrase_on_text() { + SearchComparison cmp = + new SearchComparison( + new Field(new QualifiedName("name"), List.of()), + SearchComparison.Operator.EQUALS, + phrase("foo bar*")); + Function resolver = Map.of("name", TEXT)::get; + assertEquals("name:\"foo bar*\"", cmp.toQueryString(resolver)); + } + + @Test + void comparison_unknown_field_falls_back_to_phrase_form() { + SearchComparison cmp = + new SearchComparison( + new Field(new QualifiedName("unmapped"), List.of()), + SearchComparison.Operator.EQUALS, + phrase("foo bar*")); + // Resolver returns null for unknown fields. + Function resolver = f -> null; + assertEquals("unmapped:\"foo bar*\"", cmp.toQueryString(resolver)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java index e1743b5fc26..0d227280f72 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSearchCommandIT.java @@ -5,12 +5,382 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.SearchCommandIT; public class CalciteSearchCommandIT extends SearchCommandIT { + + private static final String IDX_KEYWORD = "test_5682_keyword"; + private static final String IDX_TEXT = "test_5682_text"; + @Override public void init() throws Exception { super.init(); enableCalcite(); + setupSpecIndices(); + } + + private void setupSpecIndices() throws IOException { + createSpecIndex(IDX_KEYWORD, "keyword"); + createSpecIndex(IDX_TEXT, "text"); + } + + private void createSpecIndex(String indexName, String fieldType) throws IOException { + try { + client().performRequest(new Request("DELETE", "/" + indexName)); + } catch (ResponseException ignore) { + // ok + } + + Request createIndex = new Request("PUT", "/" + indexName); + createIndex.setJsonEntity( + "{\n" + + " \"settings\": {\"number_of_shards\": 1, \"number_of_replicas\": 0},\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"name\": {\"type\": \"" + + fieldType + + "\"}\n" + + " }\n" + + " }\n" + + "}"); + client().performRequest(createIndex); + + Request bulk = new Request("POST", "/" + indexName + "/_bulk?refresh=true"); + bulk.setJsonEntity( + "{\"index\":{}}\n{\"name\":\"foo\"}\n" + + "{\"index\":{}}\n{\"name\":\"foobar\"}\n" + + "{\"index\":{}}\n{\"name\":\"food\"}\n" + + "{\"index\":{}}\n{\"name\":\"FOO\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo bar\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo barbaz\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo-bar\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo_bar\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo.bar\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo/bar\"}\n" + + "{\"index\":{}}\n{\"name\":\"foo@bar\"}\n"); + client().performRequest(bulk); + } + + private JSONObject search(String indexName, String predicate) throws IOException { + // Escape embedded quotes for JSON body wrapping done by executeQuery helper. + String query = + "search source=" + indexName + " " + predicate.replace("\"", "\\\"") + " | fields name"; + return executeQuery(query); + } + + // ============================================================================= + // Group 1 — no special chars, no wildcards + // ============================================================================= + + @Test + public void testGroup1_1_keyword_foo() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=foo"), 1); + } + + @Test + public void testGroup1_1_text_foo() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=foo"), 7); + } + + @Test + public void testGroup1_2_keyword_quoted_foo() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo\""), 1); + } + + @Test + public void testGroup1_2_text_quoted_foo() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo\""), 7); + } + + // ============================================================================= + // Group 2 — special chars, no wildcards + // ============================================================================= + + @Test + public void testGroup2_1_keyword_underscore() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo_bar\""), 1); + } + + @Test + public void testGroup2_1_text_underscore() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo_bar\""), 1); + } + + @Test + public void testGroup2_2_keyword_dot() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo.bar\""), 1); + } + + @Test + public void testGroup2_2_text_dot() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo.bar\""), 1); + } + + @Test + public void testGroup2_3_keyword_hyphen() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo-bar\""), 1); + } + + @Test + public void testGroup2_3_text_hyphen() throws IOException { + // Emitted: name:foo\-bar (unquoted, - escaped so parser treats as literal). + // Analyzer tokenizes to [foo, bar]; boolean OR matches 7 docs whose analyzed tokens + // contain foo or bar. + verifyNumOfRows(search(IDX_TEXT, "name=\"foo-bar\""), 7); + } + + @Test + public void testGroup2_4_keyword_slash() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo/bar\""), 1); + } + + @Test + public void testGroup2_4_text_slash() throws IOException { + // Emitted: name:foo\/bar (unquoted, / escaped). Analyzer tokenizes to [foo, bar]; + // boolean OR matches 7 docs. + verifyNumOfRows(search(IDX_TEXT, "name=\"foo/bar\""), 7); + } + + @Test + public void testGroup2_5_keyword_at() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo@bar\""), 1); + } + + @Test + public void testGroup2_5_text_at() throws IOException { + // Emitted: name:foo@bar (unquoted; @ is not a Lucene special). Analyzer tokenizes + // to [foo, bar]; boolean OR matches 7 docs. + verifyNumOfRows(search(IDX_TEXT, "name=\"foo@bar\""), 7); + } + + @Test + public void testGroup2_6_keyword_space() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo bar\""), 1); + } + + @Test + public void testGroup2_6_text_space() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo bar\""), 4); + } + + // ============================================================================= + // Group 3 — trailing wildcard (postfix) + // ============================================================================= + + @Test + public void testGroup3_1_keyword_unquoted_foostar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=foo*"), 10); + } + + @Test + public void testGroup3_1_text_unquoted_foostar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=foo*"), 11); + } + + @Test + public void testGroup3_2_keyword_quoted_foostar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo*\""), 10); + } + + @Test + public void testGroup3_2_text_quoted_foostar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo*\""), 11); + } + + @Test + public void testGroup3_3_keyword_foo_underscore_star() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo_*\""), 1); + } + + @Test + public void testGroup3_3_text_foo_underscore_star() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo_*\""), 1); + } + + @Test + public void testGroup3_4_keyword_foo_dot_star() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo.*\""), 1); + } + + @Test + public void testGroup3_4_text_foo_dot_star() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo.*\""), 1); + } + + @Test + public void testGroup3_5_keyword_foo_hyphen_star() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo-*\""), 1); + } + + @Test + public void testGroup3_5_text_foo_hyphen_star() throws IOException { + // Emitted: name:foo\-* (unquoted, - escaped). This is a WildcardQuery over the analyzed + // token dictionary — text tokens don't contain '-'. 0 hits (accepted analyzer limit). + verifyNumOfRows(search(IDX_TEXT, "name=\"foo-*\""), 0); + } + + @Test + public void testGroup3_6_keyword_foo_slash_star() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo/*\""), 1); + } + + @Test + public void testGroup3_6_text_foo_slash_star() throws IOException { + // Emitted: name:foo\/* (unquoted, / escaped). WildcardQuery over analyzed tokens; + // text tokens don't contain '/'. 0 hits (accepted analyzer limit). + verifyNumOfRows(search(IDX_TEXT, "name=\"foo/*\""), 0); + } + + @Test + public void testGroup3_7_keyword_foo_space_barstar() throws IOException { + // Reported bug row (#5682): keyword whole-value pattern "foo bar*" should match + // "foo bar" and "foo barbaz". + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo bar*\""), 2); + } + + @Test + public void testGroup3_7_text_foo_space_barstar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo bar*\""), 4); + } + + // ============================================================================= + // Group 4 — leading wildcard (prefix) + // ============================================================================= + + @Test + public void testGroup4_1_keyword_starfoo() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"*foo\""), 1); + } + + @Test + public void testGroup4_1_text_starfoo() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"*foo\""), 7); + } + + @Test + public void testGroup4_2_keyword_starbar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"*bar\""), 7); + } + + @Test + public void testGroup4_2_text_starbar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"*bar\""), 7); + } + + @Test + public void testGroup4_3_keyword_starfoo_bar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"*foo bar\""), 1); + } + + @Test + public void testGroup4_3_text_starfoo_bar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"*foo bar\""), 4); + } + + // ============================================================================= + // Group 5 — interior wildcard (* in-between) + // ============================================================================= + + @Test + public void testGroup5_1_keyword_fstar_r() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"f*r\""), 7); + } + + @Test + public void testGroup5_1_text_fstar_r() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"f*r\""), 3); + } + + @Test + public void testGroup5_2_keyword_foostar_bar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo*bar\""), 7); + } + + @Test + public void testGroup5_2_text_foostar_bar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo*bar\""), 3); + } + + @Test + public void testGroup5_3_keyword_foo_space_starbaz() throws IOException { + // Keyword whole-value wildcard: "foo *baz" matches "foo barbaz". + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo *baz\""), 1); + } + + @Test + public void testGroup5_3_text_foo_space_starbaz() throws IOException { + // Corpus-dependent 0 (no adjacent tokens foo→baz analyzed in this fixture). + verifyNumOfRows(search(IDX_TEXT, "name=\"foo *baz\""), 0); + } + + @Test + public void testGroup5_4_keyword_starfoo_barstar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"*foo bar*\""), 2); + } + + @Test + public void testGroup5_4_text_starfoo_barstar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"*foo bar*\""), 4); + } + + // ============================================================================= + // Group 6 — ? wildcard (exactly one character) + // ============================================================================= + + @Test + public void testGroup6_1_keyword_foo_qmark() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo?\""), 1); + } + + @Test + public void testGroup6_1_text_foo_qmark() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo?\""), 1); + } + + @Test + public void testGroup6_2_keyword_qmark_oo() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"?oo\""), 1); + } + + @Test + public void testGroup6_2_text_qmark_oo() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"?oo\""), 7); + } + + @Test + public void testGroup6_3_keyword_f_qmark_o() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"f?o\""), 1); + } + + @Test + public void testGroup6_3_text_f_qmark_o() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"f?o\""), 7); + } + + @Test + public void testGroup6_4_keyword_foo_qmark_bar() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo?bar\""), 6); + } + + @Test + public void testGroup6_4_text_foo_qmark_bar() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo?bar\""), 2); + } + + @Test + public void testGroup6_5_keyword_foo_space_bqmarkr() throws IOException { + verifyNumOfRows(search(IDX_KEYWORD, "name=\"foo b?r\""), 1); + } + + @Test + public void testGroup6_5_text_foo_space_bqmarkr() throws IOException { + verifyNumOfRows(search(IDX_TEXT, "name=\"foo b?r\""), 0); } }