diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala index 8bf4f8f6f22..536b188e2ea 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/SamplingHelpersSpec.scala @@ -19,7 +19,7 @@ package org.apache.texera.amber.operator -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.configuredPython import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -27,7 +27,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try /** The transcribed java.util.Random, checked against what Java would answer. * @@ -38,7 +37,7 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers { /** Run `body` after the helper class, and return what it printed. */ private def runWithHelper(body: String): (Int, String) = { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) + val python = configuredPython().getOrElse(cancel("No runnable python executable")) val script = Files.createTempFile("sampling-helpers-", ".py") script.toFile.deleteOnExit() Files.write( @@ -96,22 +95,4 @@ class SamplingHelpersSpec extends AnyFlatSpec with Matchers { out.trim shouldBe "True" } } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala index bd222cb61f7..bc289a4b96f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/distinct/DistinctOpDescSpec.scala @@ -19,14 +19,16 @@ package org.apache.texera.amber.operator.distinct -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, SinglePartition, UnknownPartition} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,12 +36,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class DistinctOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -125,8 +124,8 @@ class DistinctOpDescSpec extends AnyFlatSpec with Matchers { // rows are compared in order, since the first of each is the one kept. "DistinctOpDesc.generateStandaloneCode" should "keep the rows the engine keeps for NaN and signed zero" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val values: Seq[java.lang.Double] = Seq(1.0, Double.NaN, 0.0, -0.0, null, 1.0, Double.NaN, null, -0.0) @@ -170,30 +169,4 @@ class DistinctOpDescSpec extends AnyFlatSpec with Matchers { out.linesIterator.toList shouldBe engine } } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala index ae1ceae57db..2c73dd7960c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/hashJoin/HashJoinOpDescSpec.scala @@ -19,7 +19,11 @@ package org.apache.texera.amber.operator.hashJoin -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.tuple.{ Attribute, AttributeType, @@ -30,9 +34,7 @@ import org.apache.texera.amber.core.tuple.{ import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -40,12 +42,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private def leftRight(): (Schema, Schema) = ( Schema() @@ -96,8 +95,8 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { // engine, so only the missing one matches a missing right key. The two are // only distinct in a nullable dtype, which is what an Arrow file is read into. "HashJoinOpDesc.generateStandaloneCode" should "match a missing key but not a NaN one" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val d = new HashJoinOpDesc[Integer] d.buildAttributeName = "k" @@ -135,8 +134,8 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { // null. Each join type keeps or drops the unmatched NaN rows by its own rule. for (joinType <- JoinType.values) { it should s"pair NaN, signed-zero and null keys as the engine does in a $joinType join" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val d = new HashJoinOpDesc[java.lang.Double] d.buildAttributeName = "k" @@ -246,33 +245,4 @@ class HashJoinOpDescSpec extends AnyFlatSpec with Matchers { r.probeAttributeName shouldBe "rk" r.joinType shouldBe JoinType.LEFT_OUTER } - - // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path - // (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceBinaryValueBehaviorSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceBinaryValueBehaviorSpec.scala index 5c51f5ffbe9..b7e2e19e983 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceBinaryValueBehaviorSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/HuggingFaceBinaryValueBehaviorSpec.scala @@ -20,7 +20,7 @@ package org.apache.texera.amber.operator.huggingFace.codegen import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.configuredPython import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -28,7 +28,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path} import java.util.concurrent.TimeUnit import scala.jdk.CollectionConverters._ -import scala.util.Try /** * Executes the *generated* `_read_binary_value` against column values at and @@ -50,23 +49,6 @@ class HuggingFaceBinaryValueBehaviorSpec extends AnyFlatSpec with Matchers { override def parsePython(ctx: CodegenContext): String = " return ''" } - /** Same resolution order as PythonCodeRawInvalidTextSpec: udf.conf, then PATH. */ - private lazy val pythonExe: Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - def isRunnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - private lazy val probe: Path = { val script = scala.io.Source.fromInputStream( getClass.getResourceAsStream("/python/hf_binary_value_probe.py"), @@ -83,7 +65,8 @@ class HuggingFaceBinaryValueBehaviorSpec extends AnyFlatSpec with Matchers { /** Runs the probe once over the rendered operator; one entry per case. */ private lazy val outcomes: JsonNode = { - val exe = pythonExe.getOrElse(cancel("no python interpreter available to run the probe")) + val exe = + configuredPython().getOrElse(cancel("no python interpreter available to run the probe")) val ctx = CodegenContext( hfApiToken = "token", modelId = "google/vit-base-patch16-224", diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala index 03e879caf06..4ac62bb17d6 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/ifStatement/IfOpDescSpec.scala @@ -19,14 +19,16 @@ package org.apache.texera.amber.operator.ifStatement -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,12 +36,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class IfOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -98,8 +97,8 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { // The engine picks the route from a State message, which the verification // harness has no channel for, so False is only reachable here. it should "send the rows to True by default and to False when the switch is off" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new IfOpDesc op.conditionName = "ready" @@ -139,30 +138,4 @@ class IfOpDescSpec extends AnyFlatSpec with Matchers { lines.count(_.contains("_texera_if_ready = False before this step")) shouldBe 1 } } - - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def isRunnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala index aac64c1ba41..6e3e25579dd 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/intervalJoin/IntervalJoinOpDescSpec.scala @@ -19,16 +19,18 @@ package org.apache.texera.amber.operator.intervalJoin -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, PortIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -37,12 +39,9 @@ import java.nio.file.Files import java.sql.Timestamp import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -131,10 +130,10 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { * columns against the schema the operator promises for the same two inputs. */ it should "keep an input column that carries a temporary's name" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val left = Schema() .add(new Attribute("x", AttributeType.INTEGER)) @@ -188,10 +187,10 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { private val millisecondRight = "2024-01-01 12:00:00.0009" it should "compare timestamps by the millisecond as the engine does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val left = Schema().add(new Attribute("l", AttributeType.TIMESTAMP)) val right = Schema().add(new Attribute("r", AttributeType.TIMESTAMP)) @@ -253,10 +252,10 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { // on or before the engine's bound and one past it. Both sides are fed the // same wall clocks, and the script runs with TZ set to the same zone. it should "add a time interval on the machine's wall clock as the engine does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val zone = "America/Chicago" val cases = Seq( @@ -330,10 +329,10 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { // right row first and the left rows in ascending order, as the executor // assumes. it should "add the constant in the key's declared type as the engine does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val cases = Seq( (AttributeType.INTEGER, 10L, 0L, Seq(3L, 20L)), @@ -403,33 +402,4 @@ class IntervalJoinOpDescSpec extends AnyFlatSpec with Matchers { ij.includeLeftBound shouldBe false ij.includeRightBound shouldBe false } - - // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path - // (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala index f2efd67299a..eada18d4596 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala @@ -19,15 +19,17 @@ package org.apache.texera.amber.operator.machineLearning.Scorer +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import com.fasterxml.jackson.databind.node.ObjectNode -import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -35,12 +37,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.Base64 import java.util.concurrent.TimeUnit -import scala.util.Try class MachineLearningScorerOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - /** An EncodableString field renders as a runtime decode site in the emitted code. */ private val decodeSite = "self.decode_python_template" @@ -333,42 +332,6 @@ class MachineLearningScorerOpDescSpec extends AnyFlatSpec with Matchers { } } - // Python executable resolution, following FilledAreaPlotOpDescSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndSklearn(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, sklearn").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - // Driver executed by the runtime test below. It stubs only the pytexera import seam; // the generated module runs unmodified, against the real scikit-learn metrics. private val runtimeDriverScript: String = @@ -415,10 +378,10 @@ class MachineLearningScorerOpDescSpec extends AnyFlatSpec with Matchers { |""".stripMargin it should "score the rows it can and refuse a table that keeps none of them" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndSklearn(python)) { + if (!canImport(python, "pandas", "sklearn")) { cancel(s"'$python' cannot import pandas and sklearn; skipping runtime verification") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala index 4c7befcb7a0..ed8680b1af9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala @@ -19,28 +19,27 @@ package org.apache.texera.amber.operator.regex -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit -import scala.util.Try class RegexOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -114,8 +113,8 @@ class RegexOpDescSpec extends AnyFlatSpec with Matchers { // export, and the rows each keeps are named by their input position. "RegexOpDesc.generateStandaloneCode" should "keep the rows the engine keeps on non-ASCII text" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val cases = Seq( ("é", true, Seq("É", "é", "e")), @@ -182,32 +181,6 @@ class RegexOpDescSpec extends AnyFlatSpec with Matchers { } } - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - // The engine reads `Url` from a column `url`, since Schema ignores case. pandas // would raise a KeyError on `Url`, so the script reads the column as the table // spells it. diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala index 0e806034f64..6aa5cb18ac0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/reservoirsampling/ReservoirSamplingOpDescSpec.scala @@ -19,14 +19,16 @@ package org.apache.texera.amber.operator.reservoirsampling -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,12 +36,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -96,8 +95,8 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { // The script has to refuse it there too. Answering with an empty table would // report a result the run never produced. it should "fail on the first row when the reservoir holds nothing" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val d = new ReservoirSamplingOpDesc d.k = 0 @@ -134,30 +133,4 @@ class ReservoirSamplingOpDescSpec extends AnyFlatSpec with Matchers { physical.inputPorts.keySet shouldBe d.operatorInfo.inputPorts.map(_.id).toSet physical.outputPorts.keySet shouldBe d.operatorInfo.outputPorts.map(_.id).toSet } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpDescSpec.scala index 0437c1ef6b9..5bb20db24d8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/difference/DifferenceOpDescSpec.scala @@ -19,7 +19,11 @@ package org.apache.texera.amber.operator.set.difference -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} @@ -30,9 +34,7 @@ import org.apache.texera.amber.core.workflow.{ UnknownPartition } import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.testsupport.setop.SetRows -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -40,12 +42,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -159,8 +158,8 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { // Arrow file is read into. "DifferenceOpDesc.generateStandaloneCode" should "keep the NaN row that a null row does not cancel" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val driver = s"""import pandas as pd, numpy as np @@ -192,8 +191,8 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { // bits, so a left 0.0 survives a right -0.0. A finite value and a null on // both sides are the controls. it should "match rows by the engine's equality for NaN and signed zero" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val exec = new DifferenceOpExec exec.open() @@ -210,30 +209,4 @@ class DifferenceOpDescSpec extends AnyFlatSpec with Matchers { val desc = new DifferenceOpDesc (desc.standaloneHelpers() :+ desc.generateStandaloneCode()).mkString("\n") } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpDescSpec.scala index 4e854cda321..9bb74d4926a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/intersect/IntersectOpDescSpec.scala @@ -19,7 +19,11 @@ package org.apache.texera.amber.operator.set.intersect -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple} @@ -30,9 +34,7 @@ import org.apache.texera.amber.core.workflow.{ UnknownPartition } import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.testsupport.setop.SetRows -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -40,12 +42,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class IntersectOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -104,8 +103,8 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { // an Arrow file is read into. "IntersectOpDesc.generateStandaloneCode" should "keep the null row a null row matches and not the NaN row" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val driver = s"""import pandas as pd, numpy as np @@ -136,8 +135,8 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { // nothing, and hashes them by each double's bits, which tell 0.0 from -0.0. // A finite value and a null on both sides are the controls. it should "match rows by the engine's equality for NaN and signed zero" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val exec = new IntersectOpExec exec.open() @@ -153,8 +152,8 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { // The engine's Tuple equality includes the schema, so x:DOUBLE=1 on the left and // y:DOUBLE=1 on the right share no row, though their values are equal. it should "keep no row when the two inputs are declared differently" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val leftSchema = Schema().add("x", AttributeType.DOUBLE) val rightSchema = Schema().add("y", AttributeType.DOUBLE) @@ -194,30 +193,4 @@ class IntersectOpDescSpec extends AnyFlatSpec with Matchers { val desc = new IntersectOpDesc (desc.standaloneHelpers() :+ desc.generateStandaloneCode()).mkString("\n") } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpDescSpec.scala index 3bb65287428..9ca86c9d741 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/set/symmetricDifference/SymmetricDifferenceOpDescSpec.scala @@ -19,17 +19,19 @@ package org.apache.texera.amber.operator.set.symmetricDifference -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{HashPartition, PortIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.testsupport.setop.SetRows import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -37,12 +39,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -105,8 +104,8 @@ class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { // dtype, which is what an Arrow file is read into. "SymmetricDifferenceOpDesc.generateStandaloneCode" should "keep the NaN row that a null row does not cancel" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val driver = s"""import pandas as pd, numpy as np @@ -138,8 +137,8 @@ class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { // double's bits, so 0.0 and -0.0 are each in one. A finite value and a null // on both sides are the controls. it should "match rows by the engine's equality for NaN and signed zero" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val exec = new SymmetricDifferenceOpExec exec.open() @@ -156,30 +155,4 @@ class SymmetricDifferenceOpDescSpec extends AnyFlatSpec with Matchers { val desc = new SymmetricDifferenceOpDesc (desc.standaloneHelpers() :+ desc.generateStandaloneCode()).mkString("\n") } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala index 8458332d16e..b7678efc4f3 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sort/StableMergeSortOpDescSpec.scala @@ -19,15 +19,17 @@ package org.apache.texera.amber.operator.sort -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,12 +38,9 @@ import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.collection.mutable.ListBuffer import scala.io.Source -import scala.util.Try class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -86,8 +85,8 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { // into a nullable dtype can hold both, which is where this is run. "StableMergeSortOpDesc.generateStandaloneCode" should "put a null last both ways and a NaN where the key points" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") def blockFor(pref: SortPreference): String = { val unit = new SortCriteriaUnit @@ -141,8 +140,8 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { // keeps them in input order. Each row carries its input position, and the // engine runs the same rows both ways. it should "order signed zeros as the engine does both ways" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val values: Seq[java.lang.Double] = Seq(0.0, -0.0, Double.NaN, 1.0, null, -0.0) val schema = Schema().add("v", AttributeType.DOUBLE).add("i", AttributeType.INTEGER) @@ -201,8 +200,8 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { // The sort tiers are named after nothing the input can be carrying, so a // column that happens to answer to a helper's old name keeps its values. it should "keep an input column named after a sort helper" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val unit = new SortCriteriaUnit unit.attributeName = "x" @@ -241,8 +240,8 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { // U+1F600 is stored as the surrogate pair D83D DE00, so the engine puts it // before U+FF01, while Python's code-point order puts it after. it should "order strings the way the engine does above U+FFFF" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") def cp(c: Int): String = new String(Character.toChars(c)) val values = Seq(cp(0xff01), cp(0x1f600), "a", null, cp(0xffff)) @@ -310,30 +309,4 @@ class StableMergeSortOpDescSpec extends AnyFlatSpec with Matchers { lines(1) shouldBe engine(SortPreference.DESC) } } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala index 65d6a29e080..72ff7fd9672 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sortPartitions/SortPartitionsOpDescSpec.scala @@ -19,16 +19,18 @@ package org.apache.texera.amber.operator.sortPartitions -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.RangePartition import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,12 +38,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -95,8 +94,8 @@ class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { // file is read into, so that is where this is run. "SortPartitionsOpDesc.generateStandaloneCode" should "put a null first and a NaN after infinity" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") val driver = s"""import pandas as pd, numpy as np @@ -127,8 +126,8 @@ class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { // timestamp is compared by getTime, so two inside one millisecond are a tie // the stable sort leaves in input order. Each row carries its input position. it should "order signed zeros and same-millisecond timestamps as the engine does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "numpy")) cancel(s"'$python' cannot import pandas") def engineOrder(attrType: AttributeType, values: Seq[Any]): List[Int] = { val schema = Schema().add("v", attrType).add("i", AttributeType.INTEGER) @@ -183,30 +182,4 @@ class SortPartitionsOpDescSpec extends AnyFlatSpec with Matchers { out.trim.linesIterator.toList shouldBe engine } } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, numpy").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala index 71ae99280dd..62c8d482f22 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpDescSpec.scala @@ -19,7 +19,11 @@ package org.apache.texera.amber.operator.source.scan.arrow -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.arrow.memory.RootAllocator import org.apache.arrow.vector.{UInt8Vector, VectorSchemaRoot} import org.apache.arrow.vector.ipc.ArrowFileWriter @@ -27,13 +31,11 @@ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema = import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.source.scan.FileDecodingMethod import org.apache.texera.amber.util.ArrowUtils import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -44,12 +46,9 @@ import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source import scala.jdk.CollectionConverters._ -import scala.util.Try class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -176,8 +175,8 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { // type the same way. "ArrowSourceOpDesc.generateStandaloneCode" should "keep a missing value apart from a NaN, and an integer integral" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "pyarrow")) cancel(s"'$python' cannot import pandas") val schema = Schema( List(new Attribute("d", AttributeType.DOUBLE), new Attribute("i", AttributeType.INTEGER)) @@ -241,8 +240,8 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { // under the names "1" and "2", which is what the executor reads, and comes // back out of pandas labelled 1 and 2. it should "keep the names a pandas frame of numbered columns was written under" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas", "pyarrow")) cancel(s"'$python' cannot import pandas") val schema = Schema( @@ -336,30 +335,4 @@ class ArrowSourceOpDescSpec extends AnyFlatSpec with Matchers { /** What a script needs before the block: its imports, and the file it reads. */ private def scriptHeader(d: ArrowSourceOpDesc): String = (d.standaloneImports() :+ "import pandas as pd" :+ bindSourceFile(d)).mkString("\n") - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, pyarrow").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala index fdc0dd18da3..c58e4f74873 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala @@ -19,10 +19,14 @@ package org.apache.texera.amber.operator.source.scan.csv +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.TextNode import com.github.fge.jsonschema.main.JsonSchemaFactory -import com.typesafe.config.ConfigFactory import org.apache.texera.amber.core.executor.SourceOperatorExecutor import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.core.workflow.WorkflowContext.{ @@ -33,21 +37,17 @@ import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, Tes import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator import org.apache.texera.amber.operator.source.scan.ScanSourceOpDesc import org.apache.texera.amber.operator.source.scan.csvOld.CSVOldScanSourceOpDesc -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.{BeforeAndAfter, Tag} +import org.scalatest.BeforeAndAfter import org.scalatest.flatspec.AnyFlatSpec import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - var csvScanSourceOpDesc: CSVScanSourceOpDesc = _ var parallelCsvScanSourceOpDesc: ParallelCSVScanSourceOpDesc = _ before { @@ -659,8 +659,8 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // columns from and the script raised EmptyDataError where the engine emits no // rows. it should "read no rows with the schema's columns, as the engine does, when a headerless offset passes the end" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val tmpFile = Files.createTempFile("headerless-two-row-", ".csv") tmpFile.toFile.deleteOnExit() @@ -711,8 +711,8 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // as missing to every operator that reads its mask. "0x1p1024" is past the // largest double, which Java reads as an infinity. it should "read a DOUBLE column the way the engine's toDouble does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val tmpFile = Files.createTempFile("java-doubles-", ".csv") tmpFile.toFile.deleteOnExit() @@ -796,32 +796,4 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { withClue(s"python said:\n$out\nscript:\n$driver")(assert(process.exitValue() == 0)) out } - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala index 49b28540d99..f31e2358bbe 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDescSpec.scala @@ -19,16 +19,18 @@ package org.apache.texera.amber.operator.source.scan.csvOld -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.AttributeType import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.{LogicalOp, TestOperators} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.source.scan.FileDecodingMethod -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,12 +38,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -127,8 +126,8 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { // columns from and the script raised where the engine emits no rows. "CSVOldScanSourceOpDesc.generateStandaloneCode" should "read no rows with the schema's columns, as the engine does, when a headerless offset passes the end" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val path = writeCsv("1,a\n2,b\n") val d = describing(path) @@ -170,8 +169,8 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { // downstream reads it, so the "NaN" has to stay a value. "0x1p1024" is past // the largest double, which Java reads as an infinity. it should "read a DOUBLE column the way the engine's toDouble does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val d = describing(writeCsv("w\n1.5\nNaN\n2d\n0x1p3\n-Infinity\n0x1p1024\n-0x1p1024\n")) d.sourceSchema().getAttribute("w").getType shouldBe AttributeType.DOUBLE @@ -211,33 +210,6 @@ class CSVOldScanSourceOpDescSpec extends AnyFlatSpec with Matchers { } } - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - private def writeCsv(content: String): String = { val file = Files.createTempFile("csv-old-", ".csv") file.toFile.deleteOnExit() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala index 57debd2cd42..51c93579040 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala @@ -19,16 +19,18 @@ package org.apache.texera.amber.operator.source.scan.json -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.AttributeType import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator, TestOperators} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.source.scan.FileDecodingMethod -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,12 +38,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -149,8 +148,8 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { // rounds it to 1.0, so a filter on v > 1 would keep a different set of rows. "JSONLScanSourceOpDesc.generateStandaloneCode" should "read a double as exactly the value the file spells" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") // The block reads the file from the script's own directory, so run there. val workDir = Files.createTempDirectory("jsonl-standalone-") @@ -195,31 +194,4 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers { /** What a script needs before the block: its imports, and the file it reads. */ private def scriptHeader(d: JSONLScanSourceOpDesc): String = (d.standaloneImports() :+ "import pandas as pd" :+ bindSourceFile(d)).mkString("\n") - - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala index 48164d5c207..930772778e7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala @@ -19,7 +19,11 @@ package org.apache.texera.amber.operator.source.scan.text -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{ AttributeType, @@ -31,10 +35,8 @@ import org.apache.texera.amber.core.tuple.{ import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.TestOperators import org.apache.texera.amber.operator.source.scan.FileAttributeType -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import java.nio.charset.StandardCharsets @@ -45,7 +47,6 @@ import scala.util.Try class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) var textInputSourceOpDesc: TextInputSourceOpDesc = _ before { @@ -294,10 +295,10 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // integer that is true only at 1, and refuses anything else, which the export // has to refuse too rather than pass off as a row of false. "TextInputSourceOpDesc.generateStandaloneCode" should "refuse a boolean line the engine refuses" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") textInputSourceOpDesc.attributeType = FileAttributeType.BOOLEAN textInputSourceOpDesc.textInput = "true\nyes" @@ -314,10 +315,10 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // ended one at eight other characters, so text holding U+2028 was one row in // the engine and two in the export. it should "split the text into the lines the engine splits it into" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val separators = Seq( @@ -355,10 +356,10 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { // alone through the executor and through the export, and either both read // the same value or both refuse it. it should "read a number line the way the engine's parseField does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val linesByType = Seq( FileAttributeType.INTEGER -> Seq( @@ -535,35 +536,6 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter { (process.exitValue(), out) } - // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path - // (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - "TextInputSourceOpDesc.getPhysicalOp" should "wire the TextInputSourceOpExec class as a source op with one output port" in { val physical = diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/substringSearch/SubstringSearchOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/substringSearch/SubstringSearchOpDescSpec.scala index 5e3fa32c0c8..0579246db8f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/substringSearch/SubstringSearchOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/substringSearch/SubstringSearchOpDescSpec.scala @@ -19,28 +19,27 @@ package org.apache.texera.amber.operator.substringSearch -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tuple} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit -import scala.util.Try class SubstringSearchOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -99,8 +98,8 @@ class SubstringSearchOpDescSpec extends AnyFlatSpec with Matchers { // export, and the rows each keeps are named by their input position. "SubstringSearchOpDesc.generateStandaloneCode" should "keep the rows the engine keeps on non-ASCII text" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val cases = Seq( ("ss", false, Seq("ß", "SS", "Strasse")), @@ -155,32 +154,6 @@ class SubstringSearchOpDescSpec extends AnyFlatSpec with Matchers { } } - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - // The engine reads `Body` from a column `body`, since Schema ignores case. pandas // would raise a KeyError on `Body`, so the script reads the column as the table // spells it. diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDescSpec.scala index f6a539bb7ca..ab9acbf1ccb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/typecasting/TypeCastingOpDescSpec.scala @@ -19,15 +19,17 @@ package org.apache.texera.amber.operator.typecasting -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.executor.OpExecWithClassName import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, AttributeTypeUtils, Schema} import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -39,8 +41,6 @@ import scala.util.Try class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -139,10 +139,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { .getOrElse("error") it should "refuse a boolean the way AttributeTypeUtils does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List(castUnit("v", AttributeType.BOOLEAN)) @@ -216,10 +216,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { Seq(AttributeType.STRING, AttributeType.INTEGER, AttributeType.LONG, AttributeType.BOOLEAN) it should "cast a stored NaN the way AttributeTypeUtils does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") // One column per target, each holding the same doubles. val columns = doubleTargets.map(_.name().toLowerCase) @@ -284,10 +284,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { private val doubleTextCases = Seq("NaN", null, "1.25", "Infinity") it should "keep a NaN read from text apart from an empty cell" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val toDouble = new TypeCastingOpDesc toDouble.typeCastingUnits = List(castUnit("v", AttributeType.DOUBLE)) @@ -342,10 +342,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { Seq(Long.MaxValue, 9007199254740993L, null, -5L) it should "write a large LONG beside a null exactly" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List(castUnit("v", AttributeType.STRING)) @@ -438,10 +438,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { instants: Seq[java.time.ZonedDateTime], zone: Option[String] ): Seq[String] = { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List( @@ -564,10 +564,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { out.linesIterator.filter(_.startsWith("cell ")).map(_.drop("cell ".length)).toSeq it should "cast text to a timestamp the way AttributeTypeUtils does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List(castUnit("v", AttributeType.TIMESTAMP)) @@ -640,10 +640,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { * wall clocks, returning the engine's. */ private def readTextAsTheEngineDoes(texts: Seq[String], zone: String): Seq[String] = { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List(castUnit("v", AttributeType.TIMESTAMP)) @@ -692,10 +692,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { ) it should "keep the resolution a timestamp column arrived in" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List(castUnit("v", AttributeType.TIMESTAMP)) @@ -740,10 +740,10 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { // throws, and the script raises the engine's message rather than leaving the // cell empty. it should "refuse a cell it cannot read, as the engine does" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val op = new TypeCastingOpDesc op.typeCastingUnits = List(castUnit("v", AttributeType.TIMESTAMP)) @@ -774,33 +774,4 @@ class TypeCastingOpDescSpec extends AnyFlatSpec with Matchers { } engineAnswer("not a date", AttributeType.TIMESTAMP) shouldBe "error" } - - // Python resolution follows FilledAreaPlotOpDescSpec: udf.conf python.path - // (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePython(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption - .exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala index e6493d76473..3a894c6121c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/ImageViz/ImageVisualizerOpDescSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.operator.visualization.ImageViz +import org.apache.texera.amber.util.PythonTestSupport.configuredPython import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.scalatest.BeforeAndAfter @@ -29,7 +30,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { var opDesc: ImageVisualizerOpDesc = _ @@ -149,7 +149,7 @@ class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Mat // Both paths run the same two lines on a string, so the answer is one both can // be held to: b64encode raises, and the reason page is what the viewer sees. it should "answer a string column with the reason page in both paths" in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) + val python = configuredPython().getOrElse(cancel("No runnable python executable")) val body = standaloneFor(AttributeType.STRING).linesIterator .takeWhile(!_.startsWith("all_images_html")) @@ -174,20 +174,4 @@ class ImageVisualizerOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Mat // The executor's own reason page, which the run showed for the same value. opDesc.generatePythonCode() should include("Image is not available.") } - - private def resolvePython(): Option[String] = { - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( - "python3", - "python", - "py" - )).distinct - .find(runnable) - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala index 32785e58e1d..c32c1669e0f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/carpetPlot/CarpetPlotOpDescSpec.scala @@ -19,13 +19,15 @@ package org.apache.texera.amber.operator.visualization.carpetPlot -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,8 +38,6 @@ import scala.util.Try class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - "CarpetPlotOpDesc.operatorInfo" should "advertise the name and Scientific visualization group" in { val info = (new CarpetPlotOpDesc).operatorInfo @@ -79,10 +79,10 @@ class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { // astype as a ValueError, both of them ending the whole exported run. "CarpetPlotOpDesc.generateStandaloneCode" should "write the page the operator yields for input it cannot plot" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } @@ -200,40 +200,4 @@ class CarpetPlotOpDescSpec extends AnyFlatSpec with Matchers { | verdict = "OTHER_PAGE" | print("CASE %s %s" % (name, verdict)) |""".stripMargin - - // Python executable resolution, following FilledAreaPlotOpDescSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala index 221959265e4..5b826c1404d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/figureFactoryTable/FigureFactoryTableOpDescSpec.scala @@ -19,10 +19,12 @@ package org.apache.texera.amber.operator.visualization.figureFactoryTable -import com.typesafe.config.ConfigFactory -import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.scalatest.BeforeAndAfter -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,8 +36,6 @@ import scala.util.Try class FigureFactoryTableOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - var opDesc: FigureFactoryTableOpDesc = _ before { @@ -148,10 +148,10 @@ class FigureFactoryTableOpDescSpec extends AnyFlatSpec with BeforeAndAfter with |""".stripMargin it should "render its message when the table is empty or every row has a hole" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) cancel(s"'$python' cannot import pandas and plotly") + if (!canImport(python, "pandas", "plotly")) cancel(s"'$python' cannot import pandas and plotly") withColumns() val moduleFile = Files.createTempFile("figure_factory_table_op_", ".py") @@ -182,31 +182,4 @@ class FigureFactoryTableOpDescSpec extends AnyFlatSpec with BeforeAndAfter with () } } - - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def isRunnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala index da0b9830426..24111f8190f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/filledAreaPlot/FilledAreaPlotOpDescSpec.scala @@ -19,14 +19,16 @@ package org.apache.texera.amber.operator.visualization.filledAreaPlot -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorMetadataGenerator -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.BeforeAndAfter -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -38,8 +40,6 @@ import scala.util.Try class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - var opDesc: FilledAreaPlotOpDesc = _ before { @@ -210,42 +210,6 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc code should include("X or Y attribute does not exist") } - // Python executable resolution, following PythonCodeRawInvalidTextSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - // Driver executed by the runtime test below. It stubs only the pytexera import seam // (base class, decorator, and type aliases); the generated module itself runs unmodified, // including the real pandas groupby guard and plotly rendering. @@ -311,10 +275,10 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc |""".stripMargin it should "enforce the five-percent tolerance at runtime boundaries" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } @@ -408,10 +372,10 @@ class FilledAreaPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc // and 8 are past it. "FilledAreaPlotOpDesc.generateStandaloneCode" should "refuse the same tables the operator refuses" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala index 5d55f3e529d..ac606620226 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/gaugeChart/GaugeChartOpDescSpec.scala @@ -19,13 +19,15 @@ package org.apache.texera.amber.operator.visualization.gaugeChart -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,8 +38,6 @@ import scala.util.Try class GaugeChartOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - "GaugeChartOpDesc.operatorInfo" should "advertise the name and Financial visualization group" in { val info = (new GaugeChartOpDesc).operatorInfo @@ -142,10 +142,10 @@ class GaugeChartOpDescSpec extends AnyFlatSpec with Matchers { // pandas as a KeyError and ends the whole exported run, not just this chart. "GaugeChartOpDesc.generateStandaloneCode" should "answer what it cannot draw the way the operator answers it" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } @@ -225,40 +225,4 @@ class GaugeChartOpDescSpec extends AnyFlatSpec with Matchers { | verdict = "CHART" | print("CASE %s %s" % (name, verdict)) |""".stripMargin - - // Python executable resolution, following FilledAreaPlotOpDescSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala index 705684de677..3e5eb9f6fdf 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/histogram2d/Histogram2DOpDescSpec.scala @@ -19,13 +19,16 @@ package org.apache.texera.amber.operator.visualization.histogram2d +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImportWithin, + configuredPython +} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -33,12 +36,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent.TimeUnit import scala.io.Source -import scala.util.Try class Histogram2DOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private def configured(): Histogram2DOpDesc = { val d = new Histogram2DOpDesc d.xColumn = "x" @@ -115,8 +115,8 @@ class Histogram2DOpDescSpec extends AnyFlatSpec with Matchers { } it should "leave both files behind when it runs" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandasAndPlotly(python)) + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImportWithin(python, 120, "pandas", "plotly")) cancel(s"'$python' cannot import pandas and plotly") val dir = Files.createTempDirectory("histogram2d-outputs-") @@ -145,28 +145,4 @@ class Histogram2DOpDescSpec extends AnyFlatSpec with Matchers { Files.size(html) should be > 0L } } - - private def resolvePython(): Option[String] = { - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( - "python3", - "python", - "py" - )).distinct - .find(runnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(120, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala index aebe2e23ce7..c227e5d0968 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/polarChart/PolarChartOpDescSpec.scala @@ -19,13 +19,15 @@ package org.apache.texera.amber.operator.visualization.polarChart -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.core.tuple.{AttributeType, Schema} import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -36,8 +38,6 @@ import scala.util.Try class PolarChartOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - "PolarChartOpDesc.operatorInfo" should "advertise the name and Scientific visualization group" in { val info = (new PolarChartOpDesc).operatorInfo @@ -87,10 +87,10 @@ class PolarChartOpDescSpec extends AnyFlatSpec with Matchers { // np.issubdtype cannot interpret; the check must draw them, not raise. "PolarChartOpDesc.generateStandaloneCode" should "draw nullable numeric columns and refuse non-numeric ones" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } @@ -171,40 +171,4 @@ class PolarChartOpDescSpec extends AnyFlatSpec with Matchers { | verdict = "CHART" | print("CASE %s %s" % (name, verdict)) |""".stripMargin - - // Python executable resolution, following FilledAreaPlotOpDescSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala index c54df3f231e..2008011869a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/tablesChart/TablesPlotOpDescSpec.scala @@ -19,10 +19,12 @@ package org.apache.texera.amber.operator.visualization.tablesChart -import com.typesafe.config.ConfigFactory -import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.scalatest.BeforeAndAfter -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,8 +36,6 @@ import scala.util.Try class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - var opDesc: TablesPlotOpDesc = _ before { @@ -137,10 +137,10 @@ class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers |""".stripMargin it should "render its message when the table is empty or every row has a hole" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) cancel(s"'$python' cannot import pandas and plotly") + if (!canImport(python, "pandas", "plotly")) cancel(s"'$python' cannot import pandas and plotly") opDesc.includedColumns = List(column("col_one"), column("col_two")) val moduleFile = Files.createTempFile("tables_plot_op_", ".py") @@ -171,31 +171,4 @@ class TablesPlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers () } } - - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - - def isRunnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDescSpec.scala index 26289fe70e4..e23f885bed5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/treeplot/TreePlotOpDescSpec.scala @@ -19,10 +19,12 @@ package org.apache.texera.amber.operator.visualization.treeplot -import com.typesafe.config.ConfigFactory -import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.scalatest.BeforeAndAfter -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -30,12 +32,9 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.Base64 import java.util.concurrent.TimeUnit -import scala.util.Try class TreePlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - var opDesc: TreePlotOpDesc = _ before { @@ -75,42 +74,6 @@ class TreePlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { opDesc.generateStandaloneCode().toLowerCase should not include "igraph" } - // Python executable resolution, following FilledAreaPlotOpDescSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - // Driver executed by the runtime test below. It stubs only the pytexera import seam; // the generated module runs unmodified, and the layout is called directly so the // positions it computes are what is read, rather than a rendered picture. @@ -160,10 +123,10 @@ class TreePlotOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { * whole suite: the driver lays every case out in the same process. */ private lazy val layouts: Map[String, Map[String, (Double, Double)]] = { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala index f5aad7db0e8..924a01a16d9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/urlviz/UrlVizOpDescSpec.scala @@ -19,6 +19,11 @@ package org.apache.texera.amber.operator.visualization.urlviz +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import com.fasterxml.jackson.annotation.JsonProperty import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import org.apache.texera.amber.core.executor.OpExecWithClassName @@ -26,10 +31,8 @@ import org.apache.texera.amber.core.tuple.{Attribute, AttributeType, Schema, Tup import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.operator.metadata.OperatorGroupConstants import org.apache.texera.amber.operator.metadata.annotations.AutofillAttributeName -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import org.apache.texera.amber.util.JSONUtils.objectMapper -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -38,12 +41,9 @@ import java.nio.file.Files import java.util.concurrent.TimeUnit import javax.validation.constraints.NotNull import scala.io.Source -import scala.util.Try class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - private val workflowId = WorkflowIdentity(1L) private val executionId = ExecutionIdentity(1L) @@ -169,8 +169,8 @@ class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { } it should "write that cell the same way in the exported script" taggedAs NeedsPythonPackages in { - val python = resolvePython().getOrElse(cancel("No runnable python executable")) - if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas") + val python = configuredPython().getOrElse(cancel("No runnable python executable")) + if (!canImport(python, "pandas")) cancel(s"'$python' cannot import pandas") val desc = descFor("url") val attr = new Attribute("url", AttributeType.STRING) @@ -202,28 +202,4 @@ class UrlVizOpDescSpec extends AnyFlatSpec with Matchers { out.trim shouldBe fromExecutor } } - - private def resolvePython(): Option[String] = { - def runnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (!p.waitFor(5, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - - (sys.env.get("UDF_PYTHON_PATH").filter(_.nonEmpty).toList ++ List( - "python3", - "python", - "py" - )).distinct - .find(runnable) - } - - private def canImportPandas(python: String): Boolean = - Try( - new ProcessBuilder(python, "-c", "import pandas").redirectErrorStream(true).start() - ).toOption.exists { p => - if (!p.waitFor(60, TimeUnit.SECONDS)) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala index 9c89d9daf0a..5e2af755ba1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/visualization/waterfallChart/WaterfallChartOpDescSpec.scala @@ -19,10 +19,12 @@ package org.apache.texera.amber.operator.visualization.waterfallChart -import com.typesafe.config.ConfigFactory -import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.scalatest.BeforeAndAfter -import org.scalatest.Tag import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -34,8 +36,6 @@ import scala.util.Try class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matchers { - private val NeedsPythonPackages = Tag(classOf[IntegrationTest].getName) - var opDesc: WaterfallChartOpDesc = _ before { @@ -104,42 +104,6 @@ class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc configured().createPlotlyFigure().plain should include("""xaxis_type="category"""") } - // Python executable resolution, following FilledAreaPlotOpDescSpec: - // udf.conf python.path (UDF_PYTHON_PATH), then python3 / python / py. - private def resolvePythonExecutable(): Option[String] = { - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - (fromConfig.toList ++ List("python3", "python", "py")).distinct.find(isRunnable) - } - - private def canImportPandasAndPlotly(python: String): Boolean = { - val pTry = Try( - new ProcessBuilder(python, "-c", "import pandas, plotly").redirectErrorStream(true).start() - ) - pTry.toOption.exists { p => - val finished = p.waitFor(60, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - // Driver executed by the runtime test below. It stubs only the pytexera import seam and // swaps plotly's html renderer for its json one, so the trace the generated module hands // plotly can be read back bar by bar. @@ -192,10 +156,10 @@ class WaterfallChartOpDescSpec extends AnyFlatSpec with BeforeAndAfter with Matc |""".stripMargin it should "plot the last row and draw the total over the whole column at runtime" taggedAs NeedsPythonPackages in { - val python = resolvePythonExecutable().getOrElse( + val python = configuredPython().getOrElse( cancel("No runnable python executable (udf.conf python.path, python3, python, py)") ) - if (!canImportPandasAndPlotly(python)) { + if (!canImport(python, "pandas", "plotly")) { cancel(s"'$python' cannot import pandas and plotly; skipping runtime verification") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonCodeRawInvalidTextSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonCodeRawInvalidTextSpec.scala index 41b58c76f57..2c07acee04d 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonCodeRawInvalidTextSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonCodeRawInvalidTextSpec.scala @@ -19,21 +19,23 @@ package org.apache.texera.amber.util -import com.typesafe.config.ConfigFactory +import org.apache.texera.amber.util.PythonTestSupport.{ + NeedsPythonPackages, + canImport, + configuredPython +} import org.apache.texera.amber.operator.PythonOperatorDescriptor -import org.apache.texera.amber.operator.tags.IntegrationTest import org.apache.texera.amber.pybuilder.PythonReflectionTextUtils.truncateBlock import org.apache.texera.amber.pybuilder.PythonReflectionUtils import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.texera.amber.util.python.PythonWorkerPool -import org.scalatest.Tag import org.scalatest.funsuite.AnyFunSuite import java.nio.charset.StandardCharsets import java.nio.file.Files import java.util.concurrent import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.{Executors, TimeUnit} +import java.util.concurrent.Executors import scala.concurrent.duration.{DurationInt, FiniteDuration} import scala.concurrent.{Await, ExecutionContext, Future} import scala.util.Try @@ -208,44 +210,6 @@ final class PythonCodeRawInvalidTextSpec extends AnyFunSuite { } } - /** - * Loads the Python executable path from configuration, with fallbacks. - * - * Lookup strategy: - * 1) Try parsing udf.conf from resources and resolving it. - * 2) Fall back to ConfigFactory.load(). - * 3) Read python.path, trim, and ensure it's non-empty. - * 4) If missing or invalid, fall back to "python3", then "python", then "py" - * (validated by running --version). - */ - private def loadPythonExeFromUdfConf(): Option[String] = { - - def fromConfig: Option[String] = { - val configOpt = - Try(ConfigFactory.parseResources("udf.conf").resolve()).toOption - .orElse(Try(ConfigFactory.load()).toOption) - - configOpt - .flatMap(c => Try(c.getConfig("python").getString("path")).toOption) - .map(_.trim) - .filter(_.nonEmpty) - } - - def isRunnable(exe: String): Boolean = { - val pTry = Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()) - pTry.toOption.exists { p => - val finished = p.waitFor(5, TimeUnit.SECONDS) - if (!finished) { p.destroyForcibly(); false } - else p.exitValue() == 0 - } - } - - val candidates = - fromConfig.toList ++ List("python3", "python", "py") - - candidates.distinct.find(isRunnable) - } - test( "PythonOperatorDescriptor.generatePythonCode should not contain raw invalid JsonProperty Strings" ) { @@ -297,10 +261,10 @@ final class PythonCodeRawInvalidTextSpec extends AnyFunSuite { } test("PythonOperatorDescriptor.generatePythonCode should py_compile under isolated Python") { - val pythonExeOpt = loadPythonExeFromUdfConf() + val pythonExeOpt = configuredPython() if (pythonExeOpt.isEmpty) { fail( - "python.path not found in udf.conf (or application.conf). Configure python.path to enable this test." + "No runnable Python. Tried udf.conf python.path (UDF_PYTHON_PATH), python3, python and py." ) } val pythonExecutable = pythonExeOpt.get @@ -375,7 +339,7 @@ final class PythonCodeRawInvalidTextSpec extends AnyFunSuite { */ test( "the Python interpreter operator templates run in should import pandas and plotly", - Tag(classOf[IntegrationTest].getName) + NeedsPythonPackages ) { // Same env var and value the build reads to select this subset, in // common/workflow-operator/build.sbt; keep the two in step. Nothing enforces @@ -387,19 +351,9 @@ final class PythonCodeRawInvalidTextSpec extends AnyFunSuite { def unavailable(message: String): Nothing = if (provisioned) fail(message) else cancel(message) - val python = loadPythonExeFromUdfConf().getOrElse(unavailable("no runnable python")) - val imported = Try { - val process = new ProcessBuilder(python, "-c", "import pandas, plotly") - .redirectErrorStream(true) - .start() - // Killed on the way out: a probe that ran out of time is still running, and - // would otherwise leak into the rest of the run. - if (process.waitFor(60, TimeUnit.SECONDS)) process.exitValue() == 0 - else { process.destroyForcibly(); false } - } - if (!imported.getOrElse(false)) { + val python = configuredPython().getOrElse(unavailable("no runnable python")) + if (!canImport(python, "pandas", "plotly")) { unavailable(s"'$python' cannot import pandas and plotly") } } - } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonTestSupport.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonTestSupport.scala new file mode 100644 index 00000000000..f14c11f2164 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/PythonTestSupport.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.util + +import org.apache.texera.amber.operator.tags.IntegrationTest +import org.apache.texera.common.config.UdfConfig +import org.scalatest.Tag + +import java.util.concurrent.TimeUnit +import scala.util.Try + +private[amber] object PythonTestSupport { + val NeedsPythonPackages: Tag = Tag(classOf[IntegrationTest].getName) + private val PathCandidates = List("python3", "python", "py") + + /** The first interpreter that runs: `python.path` from udf.conf (set by `UDF_PYTHON_PATH`), then + * python3, python and py from the PATH. + */ + def configuredPython(): Option[String] = { + val configured = Some(UdfConfig.pythonPath.trim).filter(_.nonEmpty) + (configured.toList ++ PathCandidates).distinct.find(runnable) + } + + def pathPython(): Option[String] = PathCandidates.find(runnable) + + def canImport(python: String, modules: String*): Boolean = + canImportWithin(python, 60, modules: _*) + + def canImportWithin(python: String, timeoutSeconds: Int, modules: String*): Boolean = + succeeds(Seq(python, "-c", "import " + modules.mkString(", ")), timeoutSeconds) + + private def runnable(executable: String): Boolean = + succeeds(Seq(executable, "--version"), 5) + + // Nothing reads the child's output, and a pipe nobody reads blocks the child once it fills, so + // the output is discarded. + private def succeeds(command: Seq[String], timeoutSeconds: Int): Boolean = + Try( + new ProcessBuilder(command: _*) + .redirectErrorStream(true) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .start() + ).toOption.exists { process => + if (process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) process.exitValue() == 0 + else { + process.destroyForcibly() + false + } + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/python/PythonWorkerPoolSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/python/PythonWorkerPoolSpec.scala index 6187cf8e241..b7b6179a0c7 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/python/PythonWorkerPoolSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/util/python/PythonWorkerPoolSpec.scala @@ -19,11 +19,12 @@ package org.apache.texera.amber.util.python +import org.apache.texera.amber.util.PythonTestSupport.pathPython import com.fasterxml.jackson.databind.node.ObjectNode import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.funsuite.AnyFunSuite -import java.util.concurrent.{Executors, TimeUnit} +import java.util.concurrent.Executors import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutorService, Future} import scala.util.Try @@ -69,15 +70,7 @@ final class PythonWorkerPoolSpec extends AnyFunSuite { * this deliberately skips the configured `python.path` the suites that need * pandas resolve. A machine without one cancels rather than fails. */ - private lazy val python: String = { - def isRunnable(exe: String): Boolean = - Try(new ProcessBuilder(exe, "--version").redirectErrorStream(true).start()).toOption - .exists { p => - if (p.waitFor(5, TimeUnit.SECONDS)) p.exitValue() == 0 else { p.destroyForcibly(); false } - } - - List("python3", "python", "py").find(isRunnable).getOrElse(cancel("no runnable python")) - } + private lazy val python: String = pathPython().getOrElse(cancel("no runnable python")) /** Resolves [[python]] here so that the cancel lands on the test thread. Forced * inside a case, it would be forced inside a `Future`, and by the time it