Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.texera.amber.operator.filter;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;

public enum PredicateCombinator {
OR("any (OR)"),

AND("all (AND)");

private final String name;

private PredicateCombinator(String name) {
this.name = name;
}

// use the name string instead of enum string in JSON
@JsonValue
public String getName() {
return this.name;
}

// Handle custom deserialization for enum
Comment thread
gupta-sahil01 marked this conversation as resolved.
Resolved
@JsonCreator
public static PredicateCombinator fromString(String value) {
for (PredicateCombinator combinator : PredicateCombinator.values()) {
if (combinator.name.equalsIgnoreCase(value)) {
return combinator;
}
}
throw new IllegalArgumentException("Unknown predicate combinator: " + value);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@

package org.apache.texera.amber.operator.filter

import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription, JsonSetter, Nulls}
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle
import org.apache.texera.amber.core.executor.OpExecWithClassName
import org.apache.texera.amber.core.tuple.{AttributeType, Schema}
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
Expand All @@ -33,8 +34,16 @@ import scala.util.Try

class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator {

// Optional so that Filters saved before this property existed still pass the
// frontend's schema validation. A missing or null value reads as OR.
@JsonProperty(required = false, defaultValue = "any (OR)")
@JsonSetter(nulls = Nulls.SKIP)
@JsonSchemaTitle("Combine Predicates With")
@JsonPropertyDescription("how to combine multiple predicates")
var predicateCombinator: PredicateCombinator = PredicateCombinator.OR

@JsonProperty(value = "predicates", required = true)
@JsonPropertyDescription("multiple predicates in OR")
@JsonPropertyDescription("multiple predicates to be combined")
var predicates: List[FilterPredicate] = List.empty

override def getPhysicalOp(
Expand All @@ -58,7 +67,7 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator
override def operatorInfo: OperatorInfo = {
OperatorInfo(
"Filter",
"Performs a filter operation using OR between multiple predicates",
"Performs a filter operation using AND or OR between multiple predicates",
OperatorGroupConstants.CLEANING_GROUP,
List(InputPort()),
List(OutputPort()),
Expand All @@ -69,9 +78,9 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator
override def generateStandaloneCode(): String = generateStandaloneCode(Map.empty)

override def generateStandaloneCode(inputSchemas: Map[PortIdentity, Schema]): String = {
// No predicate keeps no row: the executor's filter is `predicates.exists`,
// which answers false on an empty list. Passing the frame through would be
// the opposite answer.
// No predicate keeps no row, under OR and AND alike: the executor answers
// false on an empty list in both modes (`exists`, and `nonEmpty && forall`).
// Passing the frame through would be the opposite answer.
if (predicates.isEmpty) return "out1df = in1df.iloc[0:0].copy()"
val schema = inputSchemas.values.headOption
val conditions = predicates.map { p =>
Expand All @@ -94,7 +103,11 @@ class SpecializedFilterOpDesc extends FilterOpDesc with StandaloneCodeGenerator
s"$keep(in1df[$colLit], $opLit, $valueLit)"
}
}
s"out1df = in1df[${conditions.mkString(" | ")}].reset_index(drop=True)"
val joiner = predicateCombinator match {
case PredicateCombinator.AND => " & "
case PredicateCombinator.OR => " | "
}
s"out1df = in1df[${conditions.mkString(joiner)}].reset_index(drop=True)"
}

override def standaloneHelpers(): Seq[String] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,10 @@ import org.apache.texera.amber.util.JSONUtils.objectMapper
class SpecializedFilterOpExec(descString: String) extends FilterOpExec {
private val desc: SpecializedFilterOpDesc =
objectMapper.readValue(descString, classOf[SpecializedFilterOpDesc])
setFilterFunc((tuple: Tuple) => desc.predicates.exists(_.evaluate(tuple)))
setFilterFunc(desc.predicateCombinator match {
case PredicateCombinator.AND =>
(tuple: Tuple) => desc.predicates.nonEmpty && desc.predicates.forall(_.evaluate(tuple))
case PredicateCombinator.OR =>
(tuple: Tuple) => desc.predicates.exists(_.evaluate(tuple))
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.texera.amber.operator.filter

import org.apache.texera.amber.util.JSONUtils.objectMapper
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class PredicateCombinatorSpec extends AnyFlatSpec with Matchers {

"PredicateCombinator" should "map each constant to its wire name via getName" in {
PredicateCombinator.OR.getName shouldBe "any (OR)"
PredicateCombinator.AND.getName shouldBe "all (AND)"
PredicateCombinator.values() should have length 2
}

"PredicateCombinator.fromString" should "resolve names case-insensitively" in {
PredicateCombinator.fromString("any (OR)") shouldBe PredicateCombinator.OR
PredicateCombinator.fromString("ALL (AND)") shouldBe PredicateCombinator.AND
}

it should "reject an unknown name" in {
intercept[IllegalArgumentException](PredicateCombinator.fromString("xor"))
}

"PredicateCombinator" should "round-trip through Jackson using its name" in {
objectMapper.writeValueAsString(PredicateCombinator.AND) shouldBe "\"all (AND)\""
objectMapper.readValue(
"\"any (OR)\"",
classOf[PredicateCombinator]
) shouldBe PredicateCombinator.OR
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@

package org.apache.texera.amber.operator.filter

import com.fasterxml.jackson.databind.exc.ValueInstantiationException
import com.typesafe.config.ConfigFactory
import org.apache.texera.amber.core.executor.OpExecWithClassName
import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple}
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.core.workflow.PortIdentity
import org.apache.texera.amber.operator.LogicalOp
import org.apache.texera.amber.operator.metadata.OperatorGroupConstants
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorMetadataGenerator}
import org.apache.texera.amber.operator.tags.IntegrationTest
import org.apache.texera.amber.util.JSONUtils.objectMapper
import org.scalatest.Tag
Expand All @@ -39,6 +40,7 @@ import java.time.{LocalDateTime, ZoneOffset}
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import scala.io.Source
import scala.jdk.CollectionConverters._
import scala.util.Try

class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers {
Expand All @@ -52,6 +54,7 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers {
"advertise the name, Cleaning group, and reconfiguration support" in {
val info = (new SpecializedFilterOpDesc).operatorInfo
info.userFriendlyName shouldBe "Filter"
info.operatorDescription should include("AND or OR")
info.operatorGroupName shouldBe OperatorGroupConstants.CLEANING_GROUP
info.inputPorts should have length 1
info.outputPorts should have length 1
Expand All @@ -62,6 +65,10 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers {
(new SpecializedFilterOpDesc).predicates shouldBe empty
}

"SpecializedFilterOpDesc.predicateCombinator" should "default to OR" in {
(new SpecializedFilterOpDesc).predicateCombinator shouldBe PredicateCombinator.OR
}

"SpecializedFilterOpDesc.getPhysicalOp" should
"wire SpecializedFilterOpExec and carry port identities" in {
val op = new SpecializedFilterOpDesc
Expand All @@ -87,6 +94,71 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers {
restored.asInstanceOf[SpecializedFilterOpDesc].predicates shouldBe empty
}

"SpecializedFilterOpDesc" should
Comment thread
gupta-sahil01 marked this conversation as resolved.
Resolved
"deserialize to OR when predicateCombinator is absent" in {
val restored =
objectMapper.readValue(
"""{"operatorType":"Filter","predicates":[]}""",
classOf[LogicalOp]
)
restored
.asInstanceOf[SpecializedFilterOpDesc]
.predicateCombinator shouldBe PredicateCombinator.OR
}

it should "round-trip an explicit AND combinator through the polymorphic base" in {
val op = new SpecializedFilterOpDesc
op.predicateCombinator = PredicateCombinator.AND
val restored =
objectMapper.readValue(objectMapper.writeValueAsString(op), classOf[LogicalOp])
restored
.asInstanceOf[SpecializedFilterOpDesc]
.predicateCombinator shouldBe PredicateCombinator.AND
}

it should "reject an unknown predicateCombinator and name the field" in {
val error = intercept[ValueInstantiationException] {
objectMapper.readValue(
"""{"operatorType":"Filter","predicates":[],"predicateCombinator":"xor"}""",
classOf[LogicalOp]
)
}
error.getMessage should include("predicateCombinator")
error.getMessage should include("xor")
}

// Filters saved before predicateCombinator existed lack the key, and the
// frontend validates against this schema without filling defaults, so a
// required combinator would mark every one of them invalid.
it should "leave predicateCombinator optional and list it before predicates in the schema" in {
val schema =
OperatorMetadataGenerator.generateOperatorJsonSchema(classOf[SpecializedFilterOpDesc])
val required = schema.get("required").elements().asScala.map(_.asText).toList
required should contain("predicates")
required should not contain "predicateCombinator"
val order = schema.get("properties").fieldNames().asScala.toList
order.indexOf("predicateCombinator") should be < order.indexOf("predicates")
schema.at("/properties/predicateCombinator/default").asText shouldBe "any (OR)"
}

// A null combinator keeps the OR default rather than reaching the executor as
// null, where neither branch of its match would apply.
it should "build an executor that behaves as OR from a null predicateCombinator" in {
val tuple = Tuple
.builder(Schema().add("a", AttributeType.INTEGER).add("b", AttributeType.INTEGER))
.add(new Attribute("a", AttributeType.INTEGER), 1)
.add(new Attribute("b", AttributeType.INTEGER), 2)
.build()
val exec = new SpecializedFilterOpExec(
"""{"operatorType":"Filter","predicateCombinator":null,"predicates":[""" +
"""{"attribute":"a","condition":"=","value":"1"},""" +
"""{"attribute":"b","condition":"=","value":"99"}]}"""
)
exec.open()
exec.processTuple(tuple, 0) should not be empty
exec.close()
}

"SpecializedFilterOpDesc.generateStandaloneCode" should "emit one condition per predicate" in {
val d = new SpecializedFilterOpDesc
d.predicates = List(new FilterPredicate("age", ComparisonType.GREATER_THAN, "18"))
Expand All @@ -95,6 +167,28 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers {
code should include("out1df")
}

it should "join two predicates with | under OR and & under AND" in {
val d = new SpecializedFilterOpDesc
d.predicates = List(
new FilterPredicate("age", ComparisonType.GREATER_THAN, "30"),
new FilterPredicate("country", ComparisonType.EQUAL_TO, "US")
)
val inputSchemas = Map(
PortIdentity() -> Schema()
.add("age", AttributeType.INTEGER)
.add("country", AttributeType.STRING)
)
val age = """_texera_filter_keep_double(in1df["age"], ">", "30")"""
val country = """_texera_filter_keep_string(in1df["country"], "=", "US")"""

d.generateStandaloneCode(inputSchemas) shouldBe
s"out1df = in1df[$age | $country].reset_index(drop=True)"

d.predicateCombinator = PredicateCombinator.AND
d.generateStandaloneCode(inputSchemas) shouldBe
s"out1df = in1df[$age & $country].reset_index(drop=True)"
}

// FilterPredicate reads `Name` from a column `name`, since Schema ignores case.
// pandas would raise a KeyError on `Name`, so the script reads the column as the
// table spells it.
Expand All @@ -111,11 +205,14 @@ class SpecializedFilterOpDescSpec extends AnyFlatSpec with Matchers {
"""out1df = in1df[_texera_filter_keep_string(in1df["name"], "=", "a") | (in1df["name"].isna())].reset_index(drop=True)"""
}

// The executor filters on `predicates.exists`, which answers false on an empty
// list, so no predicate keeps no row. The columns survive; the rows do not.
it should "keep no row when there is no predicate" in {
(new SpecializedFilterOpDesc).generateStandaloneCode() shouldBe
"out1df = in1df.iloc[0:0].copy()"
// The executor answers false on an empty list under OR (`exists`) and AND
// (`nonEmpty && forall`) alike, so no predicate keeps no row. The columns
// survive; the rows do not.
it should "keep no row when there is no predicate, under either combinator" in {
val d = new SpecializedFilterOpDesc
d.generateStandaloneCode() shouldBe "out1df = in1df.iloc[0:0].copy()"
d.predicateCombinator = PredicateCombinator.AND
d.generateStandaloneCode() shouldBe "out1df = in1df.iloc[0:0].copy()"
}

// FilterPredicate reads `KEY` from a column `Key`, since Schema ignores case,
Expand Down
Loading
Loading