Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,14 @@

package org.apache.texera.amber.operator

import com.typesafe.config.ConfigFactory
import org.apache.texera.amber.util.PythonTestSupport.configuredPython
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.util.concurrent.TimeUnit
import scala.io.Source
import scala.util.Try

/** The transcribed java.util.Random, checked against what Java would answer.
*
Expand All @@ -38,7 +37,7 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers {

/** Run `body` after the helper class, and return what it printed. */
private def runWithHelper(body: String): (Int, String) = {
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
val python = configuredPython().getOrElse(cancel("No runnable python executable"))
val script = Files.createTempFile("sampling-helpers-", ".py")
script.toFile.deleteOnExit()
Files.write(
Expand Down Expand Up @@ -96,22 +95,4 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers {
out.trim shouldBe "True"
}
}

private def resolvePython(): Option[String] = {
def fromConfig: Option[String] =
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption
.orElse(Try(ConfigFactory.load()).toOption)
.flatMap(c => Try(c.getConfig("python").getString("path")).toOption)
.map(_.trim)
.filter(_.nonEmpty)

def runnable(exe: String): Boolean =
Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption
.exists { p =>
if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}

(fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,26 @@

package org.apache.texera.amber.operator.distinct

import com.typesafe.config.ConfigFactory
import org.apache.texera.amber.util.PythonTestSupport.{
NeedsPythonPackages,
canImport,
configuredPython
}
import org.apache.texera.amber.core.executor.OpExecWithClassName
import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple}
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.core.workflow.{HashPartition, SinglePartition, UnknownPartition}
import org.apache.texera.amber.operator.metadata.OperatorGroupConstants
import org.apache.texera.amber.operator.tags.IntegrationTest
import org.scalatest.Tag
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.util.concurrent.TimeUnit
import scala.io.Source
import scala.util.Try

class DistinctOpDescSpec extends AnyFlatSpec with Matchers {

private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName)

private val workflowId = WorkflowIdentity(1L)
private val executionId = ExecutionIdentity(1L)

Expand Down Expand Up @@ -125,8 +124,8 @@ class DistinctOpDescSpec extends AnyFlatSpec with Matchers {
// rows are compared in order, since the first of each is the one kept.
"DistinctOpDesc.generateStandaloneCode" should
"keep the rows the engine keeps for NaN and signed zero" taggedAs NeedsPythonPackages in {
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
val python = configuredPython().getOrElse(cancel("No runnable python executable"))
if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas")

val values: Seq[java.lang.Double] =
Seq(1.0, Double.NaN, 0.0, -0.0, null, 1.0, Double.NaN, null, -0.0)
Expand Down Expand Up @@ -170,30 +169,4 @@ class DistinctOpDescSpec extends AnyFlatSpec with Matchers {
out.linesIterator.toList shouldBe engine
}
}

private def resolvePython(): Option[String] = {
def fromConfig: Option[String] =
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption
.orElse(Try(ConfigFactory.load()).toOption)
.flatMap(c => Try(c.getConfig("python").getString("path")).toOption)
.map(_.trim)
.filter(_.nonEmpty)

def runnable(exe: String): Boolean =
Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption
.exists { p =>
if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}

(fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable)
}

private def canImportPandas(python: String): Boolean =
Try(
new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start()
).toOption.exists { p =>
if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@

package org.apache.texera.amber.operator.hashJoin

import com.typesafe.config.ConfigFactory
import org.apache.texera.amber.util.PythonTestSupport.{
NeedsPythonPackages,
canImport,
configuredPython
}
import org.apache.texera.amber.core.tuple.{
Attribute,
AttributeType,
Expand All @@ -30,22 +34,17 @@ import org.apache.texera.amber.core.tuple.{
import org.apache.texera.amber.core.workflow.PortIdentity
import org.apache.texera.amber.operator.LogicalOp
import org.apache.texera.amber.operator.metadata.OperatorGroupConstants
import org.apache.texera.amber.operator.tags.IntegrationTest
import org.apache.texera.amber.util.JSONUtils.objectMapper
import org.scalatest.Tag
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.util.concurrent.TimeUnit
import scala.io.Source
import scala.util.Try

class HashJoinOpDescSpec extends AnyFlatSpec with Matchers {

private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName)

private def leftRight(): (Schema, Schema) =
(
Schema()
Expand Down Expand Up @@ -96,8 +95,8 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers {
// engine, so only the missing one matches a missing right key. The two are
// only distinct in a nullable dtype, which is what an Arrow file is read into.
"HashJoinOpDesc.generateStandaloneCode" should "match a missing key but not a NaN one" taggedAs NeedsPythonPackages in {
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
val python = configuredPython().getOrElse(cancel("No runnable python executable"))
if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas")

val d = new HashJoinOpDesc[Integer]
d.buildAttributeName = "k"
Expand Down Expand Up @@ -135,8 +134,8 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers {
// null. Each join type keeps or drops the unmatched NaN rows by its own rule.
for (joinType <- JoinType.values) {
it should s"pair NaN, signed-zero and null keys as the engine does in a $joinType join" taggedAs NeedsPythonPackages in {
val python = resolvePython().getOrElse(cancel("No runnable python executable"))
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
val python = configuredPython().getOrElse(cancel("No runnable python executable"))
if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas")

val d = new HashJoinOpDesc[java.lang.Double]
d.buildAttributeName = "k"
Expand Down Expand Up @@ -246,33 +245,4 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers {
r.probeAttributeName shouldBe "rk"
r.joinType shouldBe JoinType.LEFT_OUTER
}

// Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path
// (UDF_PYTHON_PATH), then python3 / python / py.
private def resolvePython(): Option[String] = {
def fromConfig: Option[String] =
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption
.orElse(Try(ConfigFactory.load()).toOption)
.flatMap(c => Try(c.getConfig("python").getString("path")).toOption)
.map(_.trim)
.filter(_.nonEmpty)

def runnable(exe: String): Boolean =
Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption
.exists { p =>
if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}

(fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable)
}

private def canImportPandas(python: String): Boolean =
Try(
new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start()
).toOption
.exists { p =>
if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@
package org.apache.texera.amber.operator.huggingFace.codegen

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.typesafe.config.ConfigFactory
import org.apache.texera.amber.util.PythonTestSupport.configuredPython
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
import java.util.concurrent.TimeUnit
import scala.jdk.CollectionConverters._
import scala.util.Try

/**
* Executes the *generated* `_read_binary_value` against column values at and
Expand All @@ -50,23 +49,6 @@ class HuggingFaceBinaryValueBehaviorSpec extends AnyFlatSpec with Matchers {
override def parsePython(ctx: CodegenContext): String = " return ''"
}

/** Same resolution order as PythonCodeRawInvalidTextSpec: udf.conf, then PATH. */
private lazy val pythonExe: Option[String] = {
def fromConfig: Option[String] =
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption
.orElse(Try(ConfigFactory.load()).toOption)
.flatMap(c => Try(c.getConfig("python").getString("path")).toOption)
.map(_.trim)
.filter(_.nonEmpty)
def isRunnable(exe: String): Boolean =
Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption
.exists { p =>
if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}
(fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable)
}

private lazy val probe: Path = {
val script = scala.io.Source.fromInputStream(
getClass.getResourceAsStream("/python/hf_binary_value_probe.py"),
Expand All @@ -83,7 +65,8 @@ class HuggingFaceBinaryValueBehaviorSpec extends AnyFlatSpec with Matchers {

/** Runs the probe once over the rendered operator; one entry per case. */
private lazy val outcomes: JsonNode = {
val exe = pythonExe.getOrElse(cancel("no python interpreter available to run the probe"))
val exe =
configuredPython().getOrElse(cancel("no python interpreter available to run the probe"))
val ctx = CodegenContext(
hfApiToken = "token",
modelId = "google/vit-base-patch16-224",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,26 @@

package org.apache.texera.amber.operator.ifStatement

import com.typesafe.config.ConfigFactory
import org.apache.texera.amber.util.PythonTestSupport.{
NeedsPythonPackages,
canImport,
configuredPython
}
import org.apache.texera.amber.core.executor.OpExecWithClassName
import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema}
import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity}
import org.apache.texera.amber.core.workflow.PortIdentity
import org.apache.texera.amber.operator.metadata.OperatorGroupConstants
import org.apache.texera.amber.operator.tags.IntegrationTest
import org.scalatest.Tag
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.util.concurrent.TimeUnit
import scala.io.Source
import scala.util.Try

class IfOpDescSpec extends AnyFlatSpec with Matchers {

private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName)

private val workflowId = WorkflowIdentity(1L)
private val executionId = ExecutionIdentity(1L)

Expand Down Expand Up @@ -98,8 +97,8 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers {
// The engine picks the route from a State message, which the verification
// harness has no channel for, so False is only reachable here.
it should "send the rows to True by default and to False when the switch is off" taggedAs NeedsPythonPackages in {
val python = resolvePythonExecutable().getOrElse(cancel("No runnable python executable"))
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
val python = configuredPython().getOrElse(cancel("No runnable python executable"))
if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas")

val op = new IfOpDesc
op.conditionName = "ready"
Expand Down Expand Up @@ -139,30 +138,4 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers {
lines.count(_.contains("_texera_if_ready = False before this step")) shouldBe 1
}
}

private def resolvePythonExecutable(): Option[String] = {
def fromConfig: Option[String] =
Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption
.orElse(Try(ConfigFactory.load()).toOption)
.flatMap(c => Try(c.getConfig("python").getString("path")).toOption)
.map(_.trim)
.filter(_.nonEmpty)

def isRunnable(exe: String): Boolean =
Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption
.exists { p =>
if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}

(fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable)
}

private def canImportPandas(python: String): Boolean =
Try(
new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start()
).toOption.exists { p =>
if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false }
else p.exitValue() == 0
}
}
Loading
Loading