diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/PredicateCombinator.java b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/PredicateCombinator.java new file mode 100644 index 00000000000..d80e597bb15 --- /dev/null +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/PredicateCombinator.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum PredicateCombinator { + OR("any (OR)"), + + AND("all (AND)"); + + private final String name; + + private PredicateCombinator(String name) { + this.name = name; + } + + // use the name string instead of enum string in JSON + @JsonValue + public String getName() { + return this.name; + } + + // Handle custom deserialization for enum + @JsonCreator + public static PredicateCombinator fromString(String value) { + for (PredicateCombinator combinator : PredicateCombinator.values()) { + if (combinator.name.equalsIgnoreCase(value)) { + return combinator; + } + } + throw new IllegalArgumentException("Unknown predicate combinator: " + value); + } +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala index f1654f08574..81205279486 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDesc.scala @@ -19,7 +19,8 @@ package org.apache.texera.amber.operator.filter -import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} +import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription, JsonSetter, Nulls} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -33,8 +34,16 @@ import scala.util.Try class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator { + // Optional so that Filters saved before this property existed still pass the + // frontend's schema validation. A missing or null value reads as OR. + @JsonProperty(required = false, defaultValue = "any (OR)") + @JsonSetter(nulls = Nulls.SKIP) + @JsonSchemaTitle("Combine Predicates With") + @JsonPropertyDescription("how to combine multiple predicates") + var predicateCombinator: PredicateCombinator = PredicateCombinator.OR + @JsonProperty(value = "predicates", required = true) - @JsonPropertyDescription("multiple predicates in OR") + @JsonPropertyDescription("multiple predicates to be combined") var predicates: List[FilterPredicate] = List.empty override def getPhysicalOp( @@ -58,7 +67,7 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator override def operatorInfo: OperatorInfo = { OperatorInfo( "Filter", - "Performs a filter operation using OR between multiple predicates", + "Performs a filter operation using AND or OR between multiple predicates", OperatorGroupConstants.CLEANING_GROUP, List(InputPort()), List(OutputPort()), @@ -69,9 +78,9 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator override def generateStandaloneCode(): String = generateStandaloneCode(Map.empty) override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = { - // No predicate keeps no row: the executor's filter is `predicates.exists`, - // which answers false on an empty list. Passing the frame through would be - // the opposite answer. + // No predicate keeps no row, under OR and AND alike: the executor answers + // false on an empty list in both modes (`exists`, and `nonEmpty && forall`). + // Passing the frame through would be the opposite answer. if (predicates.isEmpty) return "out1df = in1df.iloc[0:0].copy()" val schema = inputSchemas.values.headOption val conditions = predicates.map { p => @@ -94,7 +103,11 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator s"$keep(in1df[$colLit], $opLit, $valueLit)" } } - s"out1df = in1df[${conditions.mkString(" | ")}].reset_index(drop=True)" + val joiner = predicateCombinator match { + case PredicateCombinator.AND => " & " + case PredicateCombinator.OR => " | " + } + s"out1df = in1df[${conditions.mkString(joiner)}].reset_index(drop=True)" } override def standaloneHelpers(): Seq[String] = diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala index 30f1d955865..ee6c338dcca 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExec.scala @@ -25,5 +25,10 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper class SpecializedFilterOpExec(descString: String) extends FilterOpExec { private val desc: SpecializedFilterOpDesc = objectMapper.readValue(descString, classOf[SpecializedFilterOpDesc]) - setFilterFunc((tuple: Tuple) => desc.predicates.exists(_.evaluate(tuple))) + setFilterFunc(desc.predicateCombinator match { + case PredicateCombinator.AND => + (tuple: Tuple) => desc.predicates.nonEmpty && desc.predicates.forall(_.evaluate(tuple)) + case PredicateCombinator.OR => + (tuple: Tuple) => desc.predicates.exists(_.evaluate(tuple)) + }) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/PredicateCombinatorSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/PredicateCombinatorSpec.scala new file mode 100644 index 00000000000..c4d56a3156c --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/PredicateCombinatorSpec.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.filter + +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class PredicateCombinatorSpec extends AnyFlatSpec with Matchers { + + "PredicateCombinator" should "map each constant to its wire name via getName" in { + PredicateCombinator.OR.getName shouldBe "any (OR)" + PredicateCombinator.AND.getName shouldBe "all (AND)" + PredicateCombinator.values() should have length 2 + } + + "PredicateCombinator.fromString" should "resolve names case-insensitively" in { + PredicateCombinator.fromString("any (OR)") shouldBe PredicateCombinator.OR + PredicateCombinator.fromString("ALL (AND)") shouldBe PredicateCombinator.AND + } + + it should "reject an unknown name" in { + intercept[IllegalArgumentException](PredicateCombinator.fromString("xor")) + } + + "PredicateCombinator" should "round-trip through Jackson using its name" in { + objectMapper.writeValueAsString(PredicateCombinator.AND) shouldBe "\"all (AND)\"" + objectMapper.readValue( + "\"any (OR)\"", + classOf[PredicateCombinator] + ) shouldBe PredicateCombinator.OR + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala index c7e43c28f4f..228b6940ca9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpDescSpec.scala @@ -19,13 +19,14 @@ package org.apache.texera.amber.operator.filter +import com.fasterxml.jackson.databind.exc.ValueInstantiationException import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorMetadataGenerator} import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.Tag @@ -39,6 +40,7 @@ import java.time.{LocalDateTime, ZoneOffset} import java.util.TimeZone import java.util.concurrent.TimeUnit import scala.io.Source +import scala.jdk.CollectionConverters._ import scala.util.Try class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { @@ -52,6 +54,7 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { "advertise the name, Cleaning group, and reconfiguration support" in { val info = (new SpecializedFilterOpDesc).operatorInfo info.userFriendlyName shouldBe "Filter" + info.operatorDescription should include("AND or OR") info.operatorGroupName shouldBe OperatorGroupConstants.CLEANING_GROUP info.inputPorts should have length 1 info.outputPorts should have length 1 @@ -62,6 +65,10 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { (new SpecializedFilterOpDesc).predicates shouldBe empty } + "SpecializedFilterOpDesc.predicateCombinator" should "default to OR" in { + (new SpecializedFilterOpDesc).predicateCombinator shouldBe PredicateCombinator.OR + } + "SpecializedFilterOpDesc.getPhysicalOp" should "wire SpecializedFilterOpExec and carry port identities" in { val op = new SpecializedFilterOpDesc @@ -87,6 +94,71 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { restored.asInstanceOf[SpecializedFilterOpDesc].predicates shouldBe empty } + "SpecializedFilterOpDesc" should + "deserialize to OR when predicateCombinator is absent" in { + val restored = + objectMapper.readValue( + """{"operatorType":"Filter","predicates":[]}""", + classOf[LogicalOp] + ) + restored + .asInstanceOf[SpecializedFilterOpDesc] + .predicateCombinator shouldBe PredicateCombinator.OR + } + + it should "round-trip an explicit AND combinator through the polymorphic base" in { + val op = new SpecializedFilterOpDesc + op.predicateCombinator = PredicateCombinator.AND + val restored = + objectMapper.readValue(objectMapper.writeValueAsString(op), classOf[LogicalOp]) + restored + .asInstanceOf[SpecializedFilterOpDesc] + .predicateCombinator shouldBe PredicateCombinator.AND + } + + it should "reject an unknown predicateCombinator and name the field" in { + val error = intercept[ValueInstantiationException] { + objectMapper.readValue( + """{"operatorType":"Filter","predicates":[],"predicateCombinator":"xor"}""", + classOf[LogicalOp] + ) + } + error.getMessage should include("predicateCombinator") + error.getMessage should include("xor") + } + + // Filters saved before predicateCombinator existed lack the key, and the + // frontend validates against this schema without filling defaults, so a + // required combinator would mark every one of them invalid. + it should "leave predicateCombinator optional and list it before predicates in the schema" in { + val schema = + OperatorMetadataGenerator.generateOperatorJsonSchema(classOf[SpecializedFilterOpDesc]) + val required = schema.get("required").elements().asScala.map(_.asText).toList + required should contain("predicates") + required should not contain "predicateCombinator" + val order = schema.get("properties").fieldNames().asScala.toList + order.indexOf("predicateCombinator") should be < order.indexOf("predicates") + schema.at("/properties/predicateCombinator/default").asText shouldBe "any (OR)" + } + + // A null combinator keeps the OR default rather than reaching the executor as + // null, where neither branch of its match would apply. + it should "build an executor that behaves as OR from a null predicateCombinator" in { + val tuple = Tuple + .builder(Schema().add("a", AttributeType.INTEGER).add("b", AttributeType.INTEGER)) + .add(new Attribute("a", AttributeType.INTEGER), 1) + .add(new Attribute("b", AttributeType.INTEGER), 2) + .build() + val exec = new SpecializedFilterOpExec( + """{"operatorType":"Filter","predicateCombinator":null,"predicates":[""" + + """{"attribute":"a","condition":"=","value":"1"},""" + + """{"attribute":"b","condition":"=","value":"99"}]}""" + ) + exec.open() + exec.processTuple(tuple, 0) should not be empty + exec.close() + } + "SpecializedFilterOpDesc.generateStandaloneCode" should "emit one condition per predicate" in { val d = new SpecializedFilterOpDesc d.predicates = List(new FilterPredicate("age", ComparisonType.GREATER_THAN, "18")) @@ -95,6 +167,28 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { code should include("out1df") } + it should "join two predicates with | under OR and & under AND" in { + val d = new SpecializedFilterOpDesc + d.predicates = List( + new FilterPredicate("age", ComparisonType.GREATER_THAN, "30"), + new FilterPredicate("country", ComparisonType.EQUAL_TO, "US") + ) + val inputSchemas = Map( + PortIdentity() -> Schema() + .add("age", AttributeType.INTEGER) + .add("country", AttributeType.STRING) + ) + val age = """_texera_filter_keep_double(in1df["age"], ">", "30")""" + val country = """_texera_filter_keep_string(in1df["country"], "=", "US")""" + + d.generateStandaloneCode(inputSchemas) shouldBe + s"out1df = in1df[$age | $country].reset_index(drop=True)" + + d.predicateCombinator = PredicateCombinator.AND + d.generateStandaloneCode(inputSchemas) shouldBe + s"out1df = in1df[$age & $country].reset_index(drop=True)" + } + // FilterPredicate reads `Name` from a column `name`, since Schema ignores case. // pandas would raise a KeyError on `Name`, so the script reads the column as the // table spells it. @@ -111,11 +205,14 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers { """out1df = in1df[_texera_filter_keep_string(in1df["name"], "=", "a") | (in1df["name"].isna())].reset_index(drop=True)""" } - // The executor filters on `predicates.exists`, which answers false on an empty - // list, so no predicate keeps no row. The columns survive; the rows do not. - it should "keep no row when there is no predicate" in { - (new SpecializedFilterOpDesc).generateStandaloneCode() shouldBe - "out1df = in1df.iloc[0:0].copy()" + // The executor answers false on an empty list under OR (`exists`) and AND + // (`nonEmpty && forall`) alike, so no predicate keeps no row. The columns + // survive; the rows do not. + it should "keep no row when there is no predicate, under either combinator" in { + val d = new SpecializedFilterOpDesc + d.generateStandaloneCode() shouldBe "out1df = in1df.iloc[0:0].copy()" + d.predicateCombinator = PredicateCombinator.AND + d.generateStandaloneCode() shouldBe "out1df = in1df.iloc[0:0].copy()" } // FilterPredicate reads `KEY` from a column `Key`, since Schema ignores case, diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExecSpec.scala index 95758fabc90..d309dd04113 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/filter/SpecializedFilterOpExecSpec.scala @@ -61,6 +61,10 @@ class SpecializedFilterOpExecSpec extends AnyFlatSpec with BeforeAndAfter { .add(new Attribute("long", AttributeType.LONG), Long.MaxValue) .build() + before { + opDesc.predicateCombinator = PredicateCombinator.OR + } + it should "open and close" in { opDesc.predicates = List() val opExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) @@ -135,4 +139,76 @@ class SpecializedFilterOpExecSpec extends AnyFlatSpec with BeforeAndAfter { opExec.close() }) } + + it should "keep a tuple under OR when only one predicate matches" in { + opDesc.predicates = List( + new FilterPredicate("int", ComparisonType.EQUAL_TO, "0"), + new FilterPredicate("string", ComparisonType.EQUAL_TO, "world") + ) + val opExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + assert(opExec.processTuple(nonNullTuple, inputPort).nonEmpty) + opExec.close() + } + + it should "drop a tuple under AND when only one predicate matches" in { + opDesc.predicates = List( + new FilterPredicate("int", ComparisonType.EQUAL_TO, "0"), + new FilterPredicate("string", ComparisonType.EQUAL_TO, "world") + ) + opDesc.predicateCombinator = PredicateCombinator.AND + val opExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + assert(opExec.processTuple(nonNullTuple, inputPort).isEmpty) + opExec.close() + } + + it should "drop a tuple under AND when the first predicate fails" in { + opDesc.predicates = List( + new FilterPredicate("string", ComparisonType.EQUAL_TO, "world"), + new FilterPredicate("int", ComparisonType.EQUAL_TO, "0") + ) + opDesc.predicateCombinator = PredicateCombinator.AND + val opExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + assert(opExec.processTuple(nonNullTuple, inputPort).isEmpty) + opExec.close() + } + + it should "keep a tuple under AND when every predicate matches" in { + opDesc.predicates = List( + new FilterPredicate("int", ComparisonType.EQUAL_TO, "0"), + new FilterPredicate("string", ComparisonType.EQUAL_TO, "hello") + ) + opDesc.predicateCombinator = PredicateCombinator.AND + val opExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + assert(opExec.processTuple(nonNullTuple, inputPort).nonEmpty) + opExec.close() + } + + it should "do nothing under AND when predicates is an empty list" in { + opDesc.predicates = List() + opDesc.predicateCombinator = PredicateCombinator.AND + val opExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + opExec.open() + assert(opExec.processTuple(allNullTuple, inputPort).isEmpty) + assert(opExec.processTuple(nonNullTuple, inputPort).isEmpty) + opExec.close() + } + + it should "agree with OR on a single predicate under AND" in { + opDesc.predicates = List(new FilterPredicate("string", ComparisonType.EQUAL_TO, "hello")) + opDesc.predicateCombinator = PredicateCombinator.AND + val andExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + andExec.open() + assert(andExec.processTuple(nonNullTuple, inputPort).nonEmpty) + andExec.close() + + opDesc.predicateCombinator = PredicateCombinator.OR + val orExec = new SpecializedFilterOpExec(objectMapper.writeValueAsString(opDesc)) + orExec.open() + assert(orExec.processTuple(nonNullTuple, inputPort).nonEmpty) + orExec.close() + } } diff --git a/docs/reference/operators/data-cleaning/_index.md b/docs/reference/operators/data-cleaning/_index.md index 9355774923a..169bb12d786 100644 --- a/docs/reference/operators/data-cleaning/_index.md +++ b/docs/reference/operators/data-cleaning/_index.md @@ -39,7 +39,7 @@ tags: [data-cleaning] | Operator | Description | |----------|-------------| | [Distinct](distinct/) | Remove duplicate tuples | -| [Filter](filter/) | Performs a filter operation using OR between multiple predicates | +| [Filter](filter/) | Performs a filter operation using AND or OR between multiple predicates | | [Limit](limit/) | Limit the number of output rows | | [Projection](projection/) | Keeps or drops the column | | [Type Casting](type-casting/) | Cast between types | diff --git a/docs/reference/operators/data-cleaning/filter.md b/docs/reference/operators/data-cleaning/filter.md index 54af25f3780..f1925124d9d 100644 --- a/docs/reference/operators/data-cleaning/filter.md +++ b/docs/reference/operators/data-cleaning/filter.md @@ -19,7 +19,7 @@ --- title: "Filter" -description: "Performs a filter operation using OR between multiple predicates" +description: "Performs a filter operation using AND or OR between multiple predicates" category: "Data Cleaning" operator_type: "Filter" tags: [data-cleaning] @@ -31,11 +31,14 @@ tags: [data-cleaning] | Property | Requirement | Type | Default | Description | |----------|-------------|------|---------|-------------| -| Predicates | ✓ | List | - | Multiple predicates in OR | +| Combine Predicates With | | any (OR), all (AND) | any (OR) | How to combine multiple predicates | +| Predicates | ✓ | List | - | Multiple predicates to be combined | | ↳ Attribute | ✓ | String | - | | | ↳ Condition | ✓ | =, >, >=, <, <=, !=, is null,
is not null | - | | | ↳ Value | | String | - | | +A Filter with no predicates keeps no rows in either mode, including "all (AND)". + ### Output Ports | Port | Mode |