From 90fb458b67a61b8425429e31d3e25474497642c0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Fri, 30 Oct 2026 20:35:41 -0700 Subject: [PATCH 1/7] feat: export Python scripts that import their helpers from texera-support The standalone helpers are packaged as `texera-support`, generated from the operators' helper blocks. An exported script imports the helpers it calls from it instead of defining them, and the export downloads as a zip with the module as `texera_support.py` next to `script.py`. Co-Authored-By: Claude Opus 5.5 --- .../operator/StandaloneCodeGenerator.scala | 11 +- .../amber/operator/StandaloneHelpers.scala | 46 ++ .../operator/StandaloneHelpersSpec.scala | 47 ++ .../workflow-to-python.service.ts | 2 + .../component/menu/menu.component.html | 37 +- .../component/menu/menu.component.scss | 10 +- .../component/menu/menu.component.spec.ts | 49 ++ .../component/menu/menu.component.ts | 20 + texera-support/.gitignore | 2 + texera-support/README.md | 37 ++ texera-support/pyproject.toml | 36 ++ texera-support/texera_support/__init__.py | 596 ++++++++++++++++++ .../WorkflowToPythonTranslator.scala | 43 +- .../resource/WorkflowToPythonResource.scala | 12 +- .../WorkflowToPythonTranslatorSpec.scala | 27 +- .../WorkflowToPythonResourceSpec.scala | 20 + 16 files changed, 963 insertions(+), 32 deletions(-) create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala create mode 100644 texera-support/.gitignore create mode 100644 texera-support/README.md create mode 100644 texera-support/pyproject.toml create mode 100644 texera-support/texera_support/__init__.py diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala index 7716436d38b..5d4a08e26e5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala @@ -100,14 +100,9 @@ trait StandaloneCodeGenerator { def producesDataFrame(): Boolean = true /** - * Definitions this operator's standalone code depends on, emitted once near - * the top of the script rather than inline. - * - * The translator concatenates operator bodies into a single module, so an - * operator needing a helper class has nowhere to put it that another operator - * would not duplicate. Helpers returned here are collected across the whole - * plan and deduplicated by their text, so two sampling operators in one - * workflow yield one copy of the generator they share. + * Definitions this operator's standalone code depends on. Each block must be + * one of [[StandaloneHelpers.packaged]]: the exported script imports the names + * it calls from the `texera-support` package rather than defining them. */ def standaloneHelpers(): Seq[String] = Seq.empty diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index 915224202fd..73c05b3056e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -19,6 +19,11 @@ package org.apache.texera.amber.operator +import org.apache.texera.amber.operator.aggregate.AggregateOpDesc +import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc +import org.apache.texera.amber.operator.source.scan.file.FileScanOpDesc +import org.apache.texera.amber.operator.source.scan.json.JSONLScanSourceOpDesc + /** Python definitions shared by several operators' standalone code, emitted * once per script via [[StandaloneCodeGenerator.standaloneHelpers]]. */ @@ -247,4 +252,45 @@ object StandaloneHelpers { | if pd.isna(x) | else ("true" if x else "false") if pd.api.types.is_bool(x) else str(x) | )""".stripMargin + + /** Every shared helper block, in the order the `texera-support` package holds them. + * A new block returned by some [[StandaloneCodeGenerator.standaloneHelpers]] belongs + * here too: an exported script imports its helpers, and the translator refuses a + * block this list lacks. + */ + def packaged: Seq[String] = + Seq( + AttributeCasts, + SamplingHelpers.JavaRandom, + AggregateOpDesc.Helpers, + SpecializedFilterOpDesc.FilterPredicates, + FileScanOpDesc.DecodeHelper, + JSONLScanSourceOpDesc.JsonFlatten, + JSONLScanSourceOpDesc.JsonText, + StandaloneCodeGenerator.PythonOperatorInput.definition + ) + + /** `texera_support/__init__.py`: the packaged blocks under the one import they share. + * The export also hands it out as `texera_support.py`, which a script finds when the + * two sit in one directory. + */ + def packageModule: String = + ("""# Licensed to the Apache Software Foundation (ASF) under one + |# or more contributor license agreements. See the NOTICE file + |# distributed with this work for additional information + |# regarding copyright ownership. The ASF licenses this file + |# to you under the Apache License, Version 2.0 (the + |# "License"); you may not use this file except in compliance + |# with the License. You may obtain a copy of the License at + |# + |# http://www.apache.org/licenses/LICENSE-2.0 + |# + |# Unless required by applicable law or agreed to in writing, + |# software distributed under the License is distributed on an + |# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + |# KIND, either express or implied. See the License for the + |# specific language governing permissions and limitations + |# under the License. + | + |import pandas as pd""".stripMargin +: packaged).mkString("", "\n\n\n", "\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala new file mode 100644 index 00000000000..736e5391498 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} + +/** The `texera-support` package against the helpers the operators declare. */ +class StandaloneHelpersSpec extends AnyFlatSpec with Matchers { + + private val module: Path = Path + .of(sys.env.getOrElse("TEXERA_HOME", ".")) + .resolve("texera-support") + .resolve("texera_support") + .resolve("__init__.py") + + // With TEXERA_SUPPORT_WRITE set, the module is rewritten instead of checked. + "texera_support/__init__.py" should "hold exactly the helpers the operators declare" in { + if (sys.env.contains("TEXERA_SUPPORT_WRITE")) { + Files.write(module, StandaloneHelpers.packageModule.getBytes(StandardCharsets.UTF_8)) + } + withClue(s"$module is stale; rerun this spec with TEXERA_SUPPORT_WRITE=1 to rewrite it.\n") { + new String(Files.readAllBytes(module), StandardCharsets.UTF_8) shouldBe + StandaloneHelpers.packageModule + } + } +} diff --git a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts index 301515d9568..465206f6ad9 100644 --- a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts +++ b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts @@ -28,6 +28,8 @@ export const WORKFLOW_TO_PYTHON_ENDPOINT = "workflow-to-python"; export interface WorkflowToPythonResponse { type: "success" | "failure"; pythonCode?: string; + /** The `texera_support.py` the script imports from, when it imports any. */ + supportModule?: string; errorMessage?: string; } diff --git a/frontend/src/app/workspace/component/menu/menu.component.html b/frontend/src/app/workspace/component/menu/menu.component.html index c5bde9d41dc..aa4b02e4d05 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.html +++ b/frontend/src/app/workspace/component/menu/menu.component.html @@ -524,16 +524,33 @@
- + + + Download the zip and run script.py next to the texera_support.py it holds. + + + + + +
{{ pythonCodeForModal }}
diff --git a/frontend/src/app/workspace/component/menu/menu.component.scss b/frontend/src/app/workspace/component/menu/menu.component.scss index 8b2bec4d2b7..ce56ec62693 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.scss +++ b/frontend/src/app/workspace/component/menu/menu.component.scss @@ -219,7 +219,15 @@ texera-coeditor-user-icon { .workflow-python-modal-toolbar { display: flex; - justify-content: flex-end; + justify-content: space-between; + align-items: center; + gap: 8px; +} + +.workflow-python-actions { + display: flex; + flex: none; + gap: 8px; } .workflow-python-code { diff --git a/frontend/src/app/workspace/component/menu/menu.component.spec.ts b/frontend/src/app/workspace/component/menu/menu.component.spec.ts index abcb1c619f7..30bb0ad1343 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.spec.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.spec.ts @@ -49,6 +49,7 @@ import { loadPersistedHeatmapView, savePersistedHeatmapView } from "../../servic import { ComputingUnitState } from "../../../common/type/computing-unit-connection.interface"; import { mockPoint, mockScanPredicate } from "../../service/workflow-graph/model/mock-workflow-data"; import { FileSaverService } from "../../../dashboard/service/user/file/file-saver.service"; +import JSZip from "jszip"; import type { ModalOptions } from "ng-zorro-antd/modal"; import type { ComputingUnitSelectionComponent } from "../power-button/computing-unit-selection.component"; import { WorkflowContent } from "../../../common/type/workflow"; @@ -762,6 +763,18 @@ describe("MenuComponent", () => { expect(convert).toHaveBeenCalled(); expect(convert.mock.calls[0][0].operators).toHaveLength(1); }); + + it("keeps the module the script imports for the download", () => { + vi.spyOn(component["workflowToPythonService"], "convertToPython").mockReturnValue( + of({ type: "success", pythonCode: "print(1)", supportModule: "def f(): pass" } as any) + ); + workflowActionService.addOperator(mockScanPredicate, mockPoint); + validationStream$.next({ errors: {}, workflowEmpty: false }); + + component.onClickExportAsPython(); + + expect(component.pythonSupportModuleForModal).toBe("def f(): pass"); + }); }); describe("hasOperators", () => { @@ -1094,6 +1107,42 @@ describe("MenuComponent", () => { vi.unstubAllGlobals(); }); + describe("downloadPythonZip", () => { + const download = async () => { + const saveAs = vi.spyOn(TestBed.inject(FileSaverService), "saveAs").mockImplementation(() => {}); + await component.downloadPythonZip(); + const [blob, name] = saveAs.mock.calls[0] as [Blob, string]; + // jsdom's Blob has no arrayBuffer(), so it is read the way a browser without one would. + const bytes = await new Promise(resolve => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as ArrayBuffer); + reader.readAsArrayBuffer(blob); + }); + return { name, zip: await JSZip.loadAsync(bytes) }; + }; + + it("zips the script with the module it imports, under the workflow name", async () => { + component.currentWorkflowName = "my-workflow"; + component.pythonCodeForModal = "from texera_support import f"; + component.pythonSupportModuleForModal = "def f(): pass"; + + const { name, zip } = await download(); + + expect(name).toBe("my-workflow.zip"); + expect(await zip.file("script.py")?.async("string")).toBe("from texera_support import f"); + expect(await zip.file("texera_support.py")?.async("string")).toBe("def f(): pass"); + }); + + it("zips the script alone when it imports nothing", async () => { + component.pythonCodeForModal = "print(1)"; + component.pythonSupportModuleForModal = undefined; + + const { zip } = await download(); + + expect(Object.keys(zip.files)).toEqual(["script.py"]); + }); + }); + // The export flags are reset when a menu is destroyed -- right on leaving the workspace, wrong on // a hand-over between a workflow's two views, where the results are kept. A menu mounting on // retained results asks for them to be recomputed rather than offering a dead button. diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts index b81eb7a9532..c41e94ca274 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.ts @@ -77,6 +77,7 @@ import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; import { JupyterPanelService } from "../../service/jupyter-panel/jupyter-panel.service"; import { WorkflowCompilingService } from "../../service/compile-workflow/workflow-compiling.service"; import { CompilationState } from "../../types/workflow-compiling.interface"; +import JSZip from "jszip"; /** * MenuComponent is the top level menu bar that shows @@ -750,6 +751,7 @@ export class MenuComponent implements OnInit, OnDestroy { public isTranslatingToPython = false; public pythonCodeForModal = ""; + public pythonSupportModuleForModal?: string; public onClickExportAsPython(): void { // Exporting the valid part of the graph answers a workflow with a required @@ -773,6 +775,7 @@ export class MenuComponent implements OnInit, OnDestroy { this.isTranslatingToPython = false; if (response.type === "success") { this.pythonCodeForModal = response.pythonCode ?? ""; + this.pythonSupportModuleForModal = response.supportModule; this.modalService.create({ nzTitle: "Workflow as Python Script", nzContent: this.workflowPythonScriptModal ?? "", @@ -803,6 +806,23 @@ export class MenuComponent implements OnInit, OnDestroy { } } + /** + * The script as a zip, with the `texera_support.py` it imports beside it: Python finds a + * module in the script's own directory, so the unzipped folder runs with nothing installed. + */ + public async downloadPythonZip(): Promise { + if (!this.pythonCodeForModal) { + return; + } + const zip = new JSZip(); + zip.file("script.py", this.pythonCodeForModal); + if (this.pythonSupportModuleForModal) { + zip.file("texera_support.py", this.pythonSupportModuleForModal); + } + const blob = await zip.generateAsync({ type: "blob" }); + this.fileSaverService.saveAs(blob, `${this.currentWorkflowName || "workflow"}.zip`); + } + public onClickExportWorkflow(): void { // The same shape the dashboard download produces (see exportedWorkflow): the content plus the // landing view as a sibling key, so a file exported here uploads as a form-default workflow too. diff --git a/texera-support/.gitignore b/texera-support/.gitignore new file mode 100644 index 00000000000..b99f7770149 --- /dev/null +++ b/texera-support/.gitignore @@ -0,0 +1,2 @@ +build/ +*.egg-info/ diff --git a/texera-support/README.md b/texera-support/README.md new file mode 100644 index 00000000000..80c6f8c8f3f --- /dev/null +++ b/texera-support/README.md @@ -0,0 +1,37 @@ + + +# texera-support + +The helpers a Python script exported from a Texera workflow calls, so the +script gives the answers the Texera engine gives. + +An exported script imports the helpers it calls from this package. The export +downloads as a zip that holds this module as `texera_support.py` next to the +script, so the unzipped folder runs with nothing installed. Installing the +package, with `pip install ./texera-support`, makes it importable from anywhere. + +`texera_support/__init__.py` is generated from the helper definitions in +`common/workflow-operator`. Do not edit it by hand. After changing a helper, run + +``` +TEXERA_SUPPORT_WRITE=1 sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.StandaloneHelpersSpec" +``` + +Without the variable, the same spec fails when the file is stale. diff --git a/texera-support/pyproject.toml b/texera-support/pyproject.toml new file mode 100644 index 00000000000..fe9f3613e96 --- /dev/null +++ b/texera-support/pyproject.toml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "texera-support" +version = "0.1.0.dev0" +description = "The helpers a Python script exported from a Texera workflow calls." +readme = "README.md" +license = { text = "Apache-2.0" } +requires-python = ">=3.11" +dependencies = [ + "pandas>=2.2", + "python-dateutil", + "tzlocal", +] + +[tool.setuptools] +packages = ["texera_support"] diff --git a/texera-support/texera_support/__init__.py b/texera-support/texera_support/__init__.py new file mode 100644 index 00000000000..cc360c651e4 --- /dev/null +++ b/texera-support/texera_support/__init__.py @@ -0,0 +1,596 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pandas as pd + + +def _texera_cast_boolean(x): + if isinstance(x, str): + text = _texera_java_trim(x) + lowered = text.lower() + if lowered == "true": + return True + if lowered == "false": + return False + return _texera_parse_int(text, 32) == 1 + return x != 0 + +def _texera_parse_number(text): + import re + + match = re.match(r"\s*(-?)([0-9,]*)(?:\.([0-9]*))?", text) + sign = -1 if match.group(1) == "-" else 1 + digits = (match.group(2) or "").replace(",", "") + fraction = match.group(3) or "" + if not digits and not fraction: + raise ValueError("Unparseable number: " + repr(text)) + if fraction: + return sign * float((digits or "0") + "." + fraction), True + value = sign * int(digits) + if -(2 ** 63) <= value <= 2 ** 63 - 1: + return value, False + return float(value), True + +def _texera_saturate(value, low, high): + if value != value: + return 0 + if value <= low: + return low + if value >= high: + return high + return int(value) + +def _texera_cast_integral(x): + if isinstance(x, str): + x, is_double = _texera_parse_number(x) + if not is_double: + return x + if isinstance(x, bool): + return 1 if x else 0 + if isinstance(x, float): + return _texera_saturate(x, -(2 ** 63), 2 ** 63 - 1) + return int(x) + +def _texera_cast_int32(x, wrap): + if isinstance(x, str): + value, is_double = _texera_parse_number(x) + wrap = not is_double + elif isinstance(x, bool): + value = 1 if x else 0 + else: + value = x + if wrap: + return ((int(value) + 2147483648) % 4294967296) - 2147483648 + return _texera_saturate(value, -2147483648, 2147483647) + +def _texera_text_to_timestamp(s): + from dateutil.parser import parse as _parse_date + + zone = _texera_local_zone() + + def _one(x): + try: + parsed = _parse_date(str(x).strip()) + except (ValueError, OverflowError) as e: + raise ValueError(f"Failed to parse type str to Timestamp: {x}") from e + if parsed.tzinfo is not None: + parsed = _texera_local_wall_clock(parsed, zone) + return parsed.replace(microsecond=parsed.microsecond // 1000 * 1000) + + text = s.astype(str).str.strip() + plain = s.notna() & text.str.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}([ T][0-9]{2}:[0-9]{2}(:[0-9]{2}(\.[0-9]+)?)?)?" + ) + out = pd.Series(pd.NaT, index=s.index, dtype="datetime64[us]") + out[plain] = pd.to_datetime(text[plain], format="ISO8601", errors="coerce").dt.floor("ms") + rest = s.notna() & out.isna() + out[rest] = s[rest].map(_one).astype("datetime64[us]") + return out + +def _texera_cast_timestamps(s): + if not pd.api.types.is_datetime64_any_dtype(s): + return _texera_text_to_timestamp(s) + if getattr(s.dtype, "tz", None) is not None: + return _texera_local_wall_clocks(s) + return s + +def _texera_epoch_millis_to_timestamp(s): + return _texera_local_wall_clocks(pd.to_datetime(s, unit="ms", errors="coerce", utc=True)) + +def _texera_local_zone(): + import os + import zoneinfo + + names = [os.environ.get("TZ", "").lstrip(":")] + try: + import tzlocal + + names.insert(0, str(tzlocal.get_localzone())) + except Exception: + pass + link = os.path.realpath("/etc/localtime") + if "zoneinfo/" in link: + names.append(link.split("zoneinfo/", 1)[1]) + for name in names: + try: + return zoneinfo.ZoneInfo(name) + except (ValueError, KeyError): + continue + return None + +def _texera_standard_offset(zone): + import datetime + import time + + if zone is None: + return datetime.timedelta(seconds=-time.timezone) + now = datetime.datetime.now(zone) + return now.utcoffset() - now.dst() + +def _texera_local_wall_clock(moment, zone): + import datetime + + from dateutil.tz import tzlocal + + utc = moment.astimezone(datetime.timezone.utc).replace(tzinfo=None) + if utc < datetime.datetime(1900, 1, 1): + return utc + _texera_standard_offset(zone) + if zone is not None: + return moment.astimezone(zone).replace(tzinfo=None) + try: + return moment.astimezone(tzlocal()).replace(tzinfo=None) + except (OSError, OverflowError): + return utc + _texera_standard_offset(zone) + +def _texera_local_wall_clocks(s): + import datetime + + from dateutil.tz import tzlocal + + zone = _texera_local_zone() + utc = s.dt.tz_convert("UTC").dt.tz_localize(None) + if zone is not None: + wall = s.dt.tz_convert(zone).dt.tz_localize(None) + before = utc < pd.Timestamp(1900, 1, 1) + return wall.where(~before, utc + _texera_standard_offset(zone)) + try: + return s.dt.tz_convert(tzlocal()).dt.tz_localize(None) + except (OSError, OverflowError): + pass + + def _offset(t): + if pd.isna(t): + return 0 + moment = t.to_pydatetime(warn=False) + utc = moment.astimezone(datetime.timezone.utc).replace(tzinfo=None) + return int((_texera_local_wall_clock(moment, None) - utc).total_seconds()) + + seconds = pd.Series([_offset(t) for t in s], index=s.index) + return utc + pd.to_timedelta(seconds, unit="s").astype("timedelta64[s]") + +def _texera_wall_clock_instants(s): + from dateutil.tz import tzlocal + + walls = s.dt.floor("ms").astype("datetime64[ms]") + zone = _texera_local_zone() + standard = walls - _texera_standard_offset(zone) + before = standard < pd.Timestamp(1900, 1, 1) + + def localized(t): + return ( + t.dt.tz_localize(zone or tzlocal(), ambiguous=False, nonexistent=pd.Timedelta("1h")) + .dt.tz_convert("UTC") + .dt.tz_localize(None) + ) + + try: + return localized(walls).where(~before, standard) + except (OSError, OverflowError): + pass + + def one(i): + if before.iloc[i]: + return standard.iloc[i] + try: + return localized(walls.iloc[i : i + 1]).iloc[0] + except (OSError, OverflowError): + return standard.iloc[i] + + return pd.Series([one(i) for i in range(len(walls))], index=s.index, dtype="datetime64[ms]") + +def _texera_cast_double(x): + return _texera_java_double(x) if isinstance(x, str) else float(x) + +def _texera_cast_doubles(s): + holes = s.isna().to_numpy() + values = _texera_cast_cells(s, _texera_cast_double, 0.0).to_numpy(dtype="float64") + return pd.Series(pd.arrays.FloatingArray(values, holes), index=s.index, name=s.name) + +def _texera_java_text(x): + import datetime + import decimal + + if pd.api.types.is_bool(x): + return "true" if x else "false" + if isinstance(x, datetime.datetime): + nanos = x.microsecond * 1000 + getattr(x, "nanosecond", 0) + return x.strftime("%Y-%m-%d %H:%M:%S") + "." + (f"{nanos:09d}".rstrip("0") or "0") + if pd.api.types.is_integer(x): + return str(int(x)) + if pd.api.types.is_float(x): + value = float(x) + if value != value: + return "NaN" + if value in (float("inf"), float("-inf")): + return "Infinity" if value > 0 else "-Infinity" + if value == 0 or 1e-3 <= abs(value) < 1e7: + return repr(value) + sign, digits, exponent = decimal.Decimal(repr(value)).as_tuple() + power = len(digits) + exponent - 1 + kept = "".join(map(str, digits)).rstrip("0") or "0" + return ("-" if sign else "") + kept[0] + "." + (kept[1:] or "0") + "E" + str(power) + return str(x) + +def _texera_java_trim(text): + return text.strip("".join(map(chr, range(33)))) + +def _texera_parse_int(text, bits): + import re + + text = _texera_java_trim(text) + bound = 2 ** (bits - 1) + if not re.fullmatch(r"[+-]?\d+", text) or not -bound <= int(text) < bound: + raise ValueError("For input string: " + repr(text)) + return int(text) + +def _texera_java_double(text): + import re + + text = _texera_java_trim(text) + if not re.fullmatch( + r"[+-]?(NaN|Infinity|((\d+\.?\d*|\.\d+)([eE][+-]?\d+)?" + r"|0[xX]([0-9a-fA-F]+\.?|[0-9a-fA-F]*\.[0-9a-fA-F]+)[pP][+-]?\d+)[fFdD]?)", + text, + re.ASCII, + ): + raise ValueError("For input string: " + repr(text)) + if text[-1] in "dDfF": + text = text[:-1] + if "x" not in text.lower(): + return float(text) + try: + return float.fromhex(text) + except OverflowError: + return float("-inf") if text.startswith("-") else float("inf") + +def _texera_double_key(x): + import math + + if math.isnan(x): + return (1, 0.0, 0.0) + return (0, x, math.copysign(1.0, x)) + +def _texera_key_codes(*frames, signed=False): + import math + + codes = {} + + def key(v, missing): + if missing: + return None + if isinstance(v, float): + if math.isnan(v): + return object() + if signed: + return (v, math.copysign(1.0, v)) + return v + + def code(df): + out = [] + for values, missing in zip(df.itertuples(index=False, name=None), df.isna().to_numpy()): + row = tuple(key(v, m) for v, m in zip(values, missing)) + out.append(codes.setdefault(row, len(codes))) + return pd.Series(out, index=df.index, dtype="int64") + + return [code(df) for df in frames] + +def _texera_read_doubles(s): + import numpy as np + + if pd.api.types.is_numeric_dtype(s): + return s + holes = s.isna().to_numpy() + values = np.array( + [np.nan if hole else _texera_java_double(x) for x, hole in zip(s, holes)], + dtype="float64", + ) + if np.isnan(values[~holes]).any(): + values = pd.arrays.FloatingArray(values, holes) + return pd.Series(values, index=s.index, name=s.name) + +def _texera_cast_string(s): + return _texera_cast_cells(s, _texera_java_text, None) + +def _texera_cast_cells(s, cast, hole): + holes = s.isna().to_numpy() + return pd.Series( + [hole if empty else cast(x) for x, empty in zip(s, holes)], + index=s.index, + name=s.name, + dtype=object, + ) + + +class _TexeraJavaRandom: + _MASK = (1 << 48) - 1 + _MULTIPLIER = 0x5DEECE66D + _ADDEND = 0xB + + def __init__(self, seed): + self._seed = (seed ^ self._MULTIPLIER) & self._MASK + + def _next(self, bits): + self._seed = (self._seed * self._MULTIPLIER + self._ADDEND) & self._MASK + value = self._seed >> (48 - bits) + return value - (1 << 32) if value >= (1 << 31) else value + + def next_double(self): + return ((self._next(26) << 27) + self._next(27)) * (2.0 ** -53) + + def next_int(self, bound): + if bound <= 0: + raise ValueError("bound must be positive") + if bound & (-bound) == bound: + return (bound * self._next(31)) >> 31 + while True: + bits = self._next(31) + value = bits % bound + probe = bits - value + (bound - 1) + if ((probe + (1 << 31)) % (1 << 32)) - (1 << 31) >= 0: + return value + + +def _texera_agg_concat(series): + partial = "" + for v, missing in zip(series, series.isna()): + text = "" if missing else _texera_java_text(v) + partial = text if partial == "" else partial + "," + text + return partial + +def _texera_agg_int_sum(series): + total = int(series.sum()) + return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) + +def _texera_agg_ts_epoch_ms(series): + return _texera_wall_clock_instants(series.dropna()).astype("int64") + +def _texera_agg_ts_sum(series): + total = int(_texera_agg_ts_epoch_ms(series).astype(object).sum()) + total = ((total + (1 << 63)) % (1 << 64)) - (1 << 63) + import datetime + + utc = pd.Timestamp(total, unit="ms") + zone = _texera_local_zone() + try: + moment = utc.to_pydatetime().replace(tzinfo=datetime.timezone.utc) + shift = _texera_local_wall_clock(moment, zone) - moment.replace(tzinfo=None) + except (OSError, OverflowError): + shift = _texera_standard_offset(zone) + return utc + shift + +def _texera_agg_ts_mean(series): + kept = _texera_agg_ts_epoch_ms(series) + if len(kept) == 0: + return None + return float(kept.astype(object).sum()) / len(kept) + +def _texera_agg_ts_pick(series, better): + kept = series.dropna() + if len(kept) == 0: + return pd.NaT + millis = kept.dt.floor("ms") + pick = 0 + for i in range(1, len(kept)): + if better(millis.iloc[i], millis.iloc[pick]): + pick = i + return kept.iloc[pick] + +def _texera_agg_ts_min(series): + return _texera_agg_ts_pick(series, lambda a, b: a < b) + +def _texera_agg_ts_max(series): + return _texera_agg_ts_pick(series, lambda a, b: a > b) + +def _texera_agg_double_values(series): + return [float(v) for v in series.dropna()] + +def _texera_agg_double_sum(series): + partial = 0.0 + for v in _texera_agg_double_values(series): + partial += v + return partial + +def _texera_agg_double_mean(series): + kept = _texera_agg_double_values(series) + if not kept: + return pd.NA + return _texera_agg_double_sum(series) / len(kept) + +def _texera_agg_double_min(series): + start = 1.7976931348623157e308 + partial = start + for v in _texera_agg_double_values(series): + if _texera_double_key(v) < _texera_double_key(partial): + partial = v + return pd.NA if partial == start else partial + +def _texera_agg_double_max(series): + start = float("-inf") + partial = start + for v in _texera_agg_double_values(series): + if _texera_double_key(v) > _texera_double_key(partial): + partial = v + return pd.NA if partial == start else partial + +def _texera_agg_double_by_group(column, fold): + results = column.apply(fold) + mask = results.map(lambda v: v is pd.NA).to_numpy(bool) + values = results.where(~mask, 0.0).to_numpy(float) + return pd.Series(pd.arrays.FloatingArray(values, mask), index=results.index) + + +def _texera_filter_text_key(s): + return s.encode("utf-16-be", "surrogatepass") + +def _texera_filter_compared(left, right): + return (left > right) - (left < right) + +def _texera_filter_rows(column, condition, order): + passes = { + "=": lambda c: c == 0, + ">": lambda c: c > 0, + ">=": lambda c: c >= 0, + "<": lambda c: c < 0, + "<=": lambda c: c <= 0, + "!=": lambda c: c != 0, + }[condition] + return pd.Series( + [ + False if missing else passes(order(v)) + for v, missing in zip(column, column.isna()) + ], + index=column.index, + dtype=bool, + ) + +def _texera_filter_keep_double(column, condition, value): + target = _texera_double_key(_texera_java_double(value)) + + def order(v): + return _texera_filter_compared(_texera_double_key(float(v)), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_long(column, condition, value): + target = _texera_parse_int(value, 64) + + def order(v): + return _texera_filter_compared(int(v), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_boolean(column, condition, value): + target = _texera_filter_text_key(_texera_java_trim(value).lower()) + + def order(v): + return _texera_filter_compared(_texera_filter_text_key("true" if v else "false"), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_timestamp(column, condition, value): + zone = _texera_local_zone() + + def wall_clock(ts): + ts = pd.Timestamp(ts).floor("ms") + if ts.tzinfo is None: + return ts + return pd.Timestamp(_texera_local_wall_clock(ts.to_pydatetime(), zone)) + + target = wall_clock(_texera_java_trim(value)) + + def order(v): + return _texera_filter_compared(wall_clock(v), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_string(column, condition, value): + try: + value_number = _texera_double_key(_texera_java_double(value)) + except ValueError: + value_number = None + + def order(v): + text = str(v) + if value_number is not None: + try: + number = _texera_double_key(_texera_java_double(text)) + return _texera_filter_compared(number, value_number) + except ValueError: + pass + return _texera_filter_compared( + _texera_filter_text_key(text), _texera_filter_text_key(value) + ) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep(column, condition, value): + dtype = column.dtype + if pd.api.types.is_bool_dtype(dtype): + return _texera_filter_keep_boolean(column, condition, value) + if pd.api.types.is_integer_dtype(dtype) or pd.api.types.is_float_dtype(dtype): + return _texera_filter_keep_double(column, condition, value) + if pd.api.types.is_datetime64_any_dtype(dtype): + return _texera_filter_keep_timestamp(column, condition, value) + return _texera_filter_keep_string(column, condition, value) + + +def _texera_decode(raw, encoding): + if encoding == "utf-16": + if raw[:2] == b"\xfe\xff": + return raw[2:].decode("utf-16-be", "replace") + if raw[:2] == b"\xff\xfe": + return raw[2:].decode("utf-16-le", "replace") + return raw.decode("utf-16-be", "replace") + return raw.decode(encoding, "replace") + + +def _texera_json_flatten(record): + flat = {} + stack = [(record, "")] + while stack: + node, parent = stack.pop() + if isinstance(node, dict): + for key, child in node.items(): + path = parent + "." + key if parent else key + if isinstance(child, (dict, list)): + stack.append((child, path)) + else: + flat[path] = child + elif isinstance(node, list): + for index, child in enumerate(node): + stack.append((child, parent + str(index + 1))) + elif parent: + flat[parent] = node + return flat + + +def _texera_json_text(value): + if value is None: + return "null" + if isinstance(value, (bool, int, float)): + return _texera_java_text(value) + if isinstance(value, str): + return value + return None + + +def _texera_python_operator_input(df): + df = df.copy() + for _c in df.columns: + if isinstance(df[_c].dtype, (pd.Float64Dtype, pd.Float32Dtype)): + df[_c] = df[_c].astype("float64") + return df diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala index 7fa1be9c752..970bc592361 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -24,7 +24,7 @@ import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.common.compiler.model.LogicalPlan -import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import java.util.regex.Matcher @@ -101,18 +101,22 @@ class WorkflowToPythonTranslator extends LazyLogging { .foreach(script += _) script += "" - // Helper definitions the operator bodies below refer to. Collected across the - // whole plan and deduplicated by text, so a workflow holding two operators - // that share one helper still emits it once. Order follows the topological - // order, which keeps the script stable for a given plan. - val helpers = topoOrder + // The helpers the operator bodies below refer to come from the texera-support + // package, which the script imports. A block the package does not hold would + // leave the script calling a name it cannot find, so it stops the export here. + topoOrder .map(logicalPlan.getOperator) .collect { case gen: StandaloneCodeGenerator => gen.standaloneHelpers() } .flatten - .distinct - if (helpers.nonEmpty) { - helpers.foreach { helper => script += helper; script += "" } - } + .find(!StandaloneHelpers.packaged.contains(_)) + .foreach { block => + throw new IllegalStateException( + s"A helper is missing from StandaloneHelpers.packaged:\n${block.take(200)}" + ) + } + // The import goes in once the bodies are written, since which helpers the + // script calls is only known then. + val helpersAt = script.length for (opIdentity <- topoOrder) { val opId = opIdentity.id @@ -221,9 +225,28 @@ class WorkflowToPythonTranslator extends LazyLogging { } } + // Named one by one: a leading underscore keeps a name out of `import *`. + val imported = packagedNamesIn(script.drop(helpersAt).mkString("\n")) + if (imported.nonEmpty) + script.insert(helpersAt - 1, s"from texera_support import ${imported.mkString(", ")}") script.mkString("\n") } + // What the package defines at its top level, which is only functions and classes. + private val packagedNames: Set[String] = StandaloneHelpers.packaged + .flatMap(_.split("\n")) + .flatMap(line => """(?:def|class) (\w+)""".r.findPrefixMatchOf(line).map(_.group(1))) + .toSet + + // The packaged names the given code refers to, read from code only, sorted. + private def packagedNamesIn(code: String): Seq[String] = + splitOffLiterals(code) + .collect { case (true, segment) => segment } + .flatMap("""\b\w+""".r.findAllIn(_)) + .filter(packagedNames) + .distinct + .sorted + // The stem of the files one operator writes, taken from its name so a reader // can tell whose picture is whose. Numbered from one even when the plan holds // a single chart, so the name a script writes does not depend on what else diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala index 38381e51aea..7bb5eb5222f 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala @@ -29,6 +29,7 @@ import org.apache.texera.amber.core.virtualidentity.{OperatorIdentity, WorkflowI import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} import org.apache.texera.common.compiler.model.{LogicalPlan, LogicalPlanPojo} import org.apache.texera.common.compiler.{CompilationErrorHandling, WorkflowCompiler} +import org.apache.texera.amber.operator.StandaloneHelpers import org.apache.texera.amber.translator.WorkflowToPythonTranslator @JsonTypeInfo( @@ -44,7 +45,9 @@ import org.apache.texera.amber.translator.WorkflowToPythonTranslator ) sealed trait WorkflowToPythonResponse -case class WorkflowToPythonSuccess(pythonCode: String) extends WorkflowToPythonResponse +/** @param supportModule the `texera_support.py` the script imports from, when it imports any. */ +case class WorkflowToPythonSuccess(pythonCode: String, supportModule: Option[String] = None) + extends WorkflowToPythonResponse case class WorkflowToPythonFailure(errorMessage: String) extends WorkflowToPythonResponse @@ -70,7 +73,12 @@ class WorkflowToPythonResource extends LazyLogging { // parsed. And a downstream operator that renders a column as text needs the type the // column was DECLARED as, which the file it reads cannot carry. val pythonCode = translator.translate(logicalPlan, outputSchemasOf(logicalPlanPojo)) - WorkflowToPythonSuccess(pythonCode) + WorkflowToPythonSuccess( + pythonCode, + Option.when(pythonCode.contains("from texera_support import"))( + StandaloneHelpers.packageModule + ) + ) } catch { case e: Exception => logger.error("Failed to translate workflow to Python", e) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala index b1321a3bc68..8298691c0e4 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -22,7 +22,7 @@ package org.apache.texera.amber.translator import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import org.apache.texera.amber.core.workflow.{InputPort, OutputPort, PhysicalOp, PortIdentity} import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} -import org.apache.texera.amber.operator.{LogicalOp, StandaloneCodeGenerator} +import org.apache.texera.amber.operator.{LogicalOp, SamplingHelpers, StandaloneCodeGenerator} import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlan} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -204,6 +204,31 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { new WorkflowToPythonTranslator().translate(LogicalPlan(ops.toList, List.empty)) } + it should "import a helper with the other imports rather than define it" in { + val op = new StubOp("_rng = _TexeraJavaRandom(1)\nout1df = in1df.copy()") { + override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) + } + op.setOperatorId("op") + val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + script should startWith("import pandas as pd\nfrom texera_support import _TexeraJavaRandom\n\n") + script should not include "class _TexeraJavaRandom" + } + + it should "not import a helper a comment or a string only names" in { + val op = stub("op", "# _TexeraJavaRandom\nout1df = in1df.assign(n=\"_TexeraJavaRandom\")") + val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + script should not include "from texera_support" + } + + it should "refuse an operator whose helper the package does not hold" in { + val op = new StubOp("out1df = in1df.apply(_texera_missing)") { + override def standaloneHelpers(): Seq[String] = Seq("def _texera_missing(x):\n return x") + } + op.setOperatorId("op") + an[IllegalStateException] should be thrownBy + new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + } + /** The translator's own contract when it meets an operator it cannot render: * a comment rather than a silently wrong line. */ diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala index e9bc1c96fef..bb639ef8875 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala @@ -24,6 +24,7 @@ import io.dropwizard.testing.junit5.ResourceExtension import jakarta.ws.rs.client.Entity import jakarta.ws.rs.core.{MediaType, Response} import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.StandaloneHelpers import org.apache.texera.amber.operator.distinct.DistinctOpDesc import org.apache.texera.amber.operator.limit.LimitOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc @@ -170,6 +171,25 @@ class WorkflowToPythonResourceSpec extends AnyFlatSpec with BeforeAndAfterAll { assert(code.contains("head(5)")) } + private def exported(pojo: LogicalPlanPojo): WorkflowToPythonSuccess = + objectMapper + .readValue(postExport(pojo).readEntity(classOf[String]), classOf[WorkflowToPythonResponse]) + .asInstanceOf[WorkflowToPythonSuccess] + + it should "import the helpers from texera-support and hand that module over with the script" in { + val success = exported(chainOf(distinctOp("distinct"), limitOp("limit", 5))) + assert(success.pythonCode.contains("from texera_support import _texera_key_codes")) + assert(!success.pythonCode.contains("def _texera_key_codes")) + assert(success.supportModule.contains(StandaloneHelpers.packageModule)) + } + + it should "hand over no module with a script that imports nothing from it" in { + val limit = limitOp("limit", 5) + val success = exported(LogicalPlanPojo(List(limit), List.empty, List.empty, List.empty)) + assert(!success.pythonCode.contains("texera_support")) + assert(success.supportModule.isEmpty) + } + it should "return a failure body rather than HTTP 500 when the plan cannot be read" in { // A link naming an operator the plan does not carry: the DAG refuses the // edge, and the resource has to answer with a reason rather than a stack From e38642238f0432f0b62c65d6f3da7931a1dddcdf Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 31 Oct 2026 21:27:23 -0700 Subject: [PATCH 2/7] feat: keep only the module the zip download needs Co-Authored-By: Claude Opus 5.5 --- texera-support/.gitignore | 2 -- texera-support/README.md | 7 +++---- texera-support/pyproject.toml | 36 ----------------------------------- 3 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 texera-support/.gitignore delete mode 100644 texera-support/pyproject.toml diff --git a/texera-support/.gitignore b/texera-support/.gitignore deleted file mode 100644 index b99f7770149..00000000000 --- a/texera-support/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -build/ -*.egg-info/ diff --git a/texera-support/README.md b/texera-support/README.md index 80c6f8c8f3f..cc0a89a5174 100644 --- a/texera-support/README.md +++ b/texera-support/README.md @@ -22,10 +22,9 @@ The helpers a Python script exported from a Texera workflow calls, so the script gives the answers the Texera engine gives. -An exported script imports the helpers it calls from this package. The export -downloads as a zip that holds this module as `texera_support.py` next to the -script, so the unzipped folder runs with nothing installed. Installing the -package, with `pip install ./texera-support`, makes it importable from anywhere. +An exported script imports the helpers it calls from this module. The export +downloads as a zip that holds it as `texera_support.py` next to the script, so +the unzipped folder runs with nothing installed. `texera_support/__init__.py` is generated from the helper definitions in `common/workflow-operator`. Do not edit it by hand. After changing a helper, run diff --git a/texera-support/pyproject.toml b/texera-support/pyproject.toml deleted file mode 100644 index fe9f3613e96..00000000000 --- a/texera-support/pyproject.toml +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[build-system] -requires = ["setuptools>=68"] -build-backend = "setuptools.build_meta" - -[project] -name = "texera-support" -version = "0.1.0.dev0" -description = "The helpers a Python script exported from a Texera workflow calls." -readme = "README.md" -license = { text = "Apache-2.0" } -requires-python = ">=3.11" -dependencies = [ - "pandas>=2.2", - "python-dateutil", - "tzlocal", -] - -[tool.setuptools] -packages = ["texera_support"] From c3f94c304f2a174c863bba233c9d995a2149abf0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Sat, 31 Oct 2026 21:30:13 -0700 Subject: [PATCH 3/7] refactor: name the shared helpers after the module, not a package Co-Authored-By: Claude Opus 5.5 --- .../operator/StandaloneCodeGenerator.scala | 4 ++-- .../amber/operator/StandaloneHelpers.scala | 10 +++++----- .../operator/StandaloneHelpersSpec.scala | 6 +++--- .../WorkflowToPythonTranslator.scala | 20 +++++++++---------- .../resource/WorkflowToPythonResource.scala | 2 +- .../WorkflowToPythonTranslatorSpec.scala | 2 +- .../WorkflowToPythonResourceSpec.scala | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala index 5d4a08e26e5..691987323cc 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala @@ -101,8 +101,8 @@ trait StandaloneCodeGenerator { /** * Definitions this operator's standalone code depends on. Each block must be - * one of [[StandaloneHelpers.packaged]]: the exported script imports the names - * it calls from the `texera-support` package rather than defining them. + * one of [[StandaloneHelpers.supportHelpers]]: the exported script imports the + * names it calls from the `texera_support` module rather than defining them. */ def standaloneHelpers(): Seq[String] = Seq.empty diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index 73c05b3056e..4623b8782a5 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -253,12 +253,12 @@ object StandaloneHelpers { | else ("true" if x else "false") if pd.api.types.is_bool(x) else str(x) | )""".stripMargin - /** Every shared helper block, in the order the `texera-support` package holds them. + /** Every shared helper block, in the order the `texera_support` module holds them. * A new block returned by some [[StandaloneCodeGenerator.standaloneHelpers]] belongs * here too: an exported script imports its helpers, and the translator refuses a * block this list lacks. */ - def packaged: Seq[String] = + def supportHelpers: Seq[String] = Seq( AttributeCasts, SamplingHelpers.JavaRandom, @@ -270,11 +270,11 @@ object StandaloneHelpers { StandaloneCodeGenerator.PythonOperatorInput.definition ) - /** `texera_support/__init__.py`: the packaged blocks under the one import they share. + /** `texera_support/__init__.py`: the shared blocks under the one import they share. * The export also hands it out as `texera_support.py`, which a script finds when the * two sit in one directory. */ - def packageModule: String = + def supportModule: String = ("""# Licensed to the Apache Software Foundation (ASF) under one |# or more contributor license agreements. See the NOTICE file |# distributed with this work for additional information @@ -292,5 +292,5 @@ object StandaloneHelpers { |# specific language governing permissions and limitations |# under the License. | - |import pandas as pd""".stripMargin +: packaged).mkString("", "\n\n\n", "\n") + |import pandas as pd""".stripMargin +: supportHelpers).mkString("", "\n\n\n", "\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala index 736e5391498..0be7b589653 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala @@ -25,7 +25,7 @@ import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path} -/** The `texera-support` package against the helpers the operators declare. */ +/** The `texera_support` module against the helpers the operators declare. */ class StandaloneHelpersSpec extends AnyFlatSpec with Matchers { private val module: Path = Path @@ -37,11 +37,11 @@ class StandaloneHelpersSpec extends AnyFlatSpec with Matchers { // With TEXERA_SUPPORT_WRITE set, the module is rewritten instead of checked. "texera_support/__init__.py" should "hold exactly the helpers the operators declare" in { if (sys.env.contains("TEXERA_SUPPORT_WRITE")) { - Files.write(module, StandaloneHelpers.packageModule.getBytes(StandardCharsets.UTF_8)) + Files.write(module, StandaloneHelpers.supportModule.getBytes(StandardCharsets.UTF_8)) } withClue(s"$module is stale; rerun this spec with TEXERA_SUPPORT_WRITE=1 to rewrite it.\n") { new String(Files.readAllBytes(module), StandardCharsets.UTF_8) shouldBe - StandaloneHelpers.packageModule + StandaloneHelpers.supportModule } } } diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala index 970bc592361..5b0ba1805f0 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -101,17 +101,17 @@ class WorkflowToPythonTranslator extends LazyLogging { .foreach(script += _) script += "" - // The helpers the operator bodies below refer to come from the texera-support - // package, which the script imports. A block the package does not hold would + // The helpers the operator bodies below refer to come from the texera_support + // module, which the script imports. A block the module does not hold would // leave the script calling a name it cannot find, so it stops the export here. topoOrder .map(logicalPlan.getOperator) .collect { case gen: StandaloneCodeGenerator => gen.standaloneHelpers() } .flatten - .find(!StandaloneHelpers.packaged.contains(_)) + .find(!StandaloneHelpers.supportHelpers.contains(_)) .foreach { block => throw new IllegalStateException( - s"A helper is missing from StandaloneHelpers.packaged:\n${block.take(200)}" + s"A helper is missing from StandaloneHelpers.supportHelpers:\n${block.take(200)}" ) } // The import goes in once the bodies are written, since which helpers the @@ -226,24 +226,24 @@ class WorkflowToPythonTranslator extends LazyLogging { } // Named one by one: a leading underscore keeps a name out of `import *`. - val imported = packagedNamesIn(script.drop(helpersAt).mkString("\n")) + val imported = supportNamesIn(script.drop(helpersAt).mkString("\n")) if (imported.nonEmpty) script.insert(helpersAt - 1, s"from texera_support import ${imported.mkString(", ")}") script.mkString("\n") } - // What the package defines at its top level, which is only functions and classes. - private val packagedNames: Set[String] = StandaloneHelpers.packaged + // What the module defines at its top level, which is only functions and classes. + private val supportNames: Set[String] = StandaloneHelpers.supportHelpers .flatMap(_.split("\n")) .flatMap(line => """(?:def|class) (\w+)""".r.findPrefixMatchOf(line).map(_.group(1))) .toSet - // The packaged names the given code refers to, read from code only, sorted. - private def packagedNamesIn(code: String): Seq[String] = + // The module's names the given code refers to, read from code only, sorted. + private def supportNamesIn(code: String): Seq[String] = splitOffLiterals(code) .collect { case (true, segment) => segment } .flatMap("""\b\w+""".r.findAllIn(_)) - .filter(packagedNames) + .filter(supportNames) .distinct .sorted diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala index 7bb5eb5222f..052d76bc1be 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala @@ -76,7 +76,7 @@ class WorkflowToPythonResource extends LazyLogging { WorkflowToPythonSuccess( pythonCode, Option.when(pythonCode.contains("from texera_support import"))( - StandaloneHelpers.packageModule + StandaloneHelpers.supportModule ) ) } catch { diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala index 8298691c0e4..09a20d6a8db 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -220,7 +220,7 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { script should not include "from texera_support" } - it should "refuse an operator whose helper the package does not hold" in { + it should "refuse an operator whose helper the module does not hold" in { val op = new StubOp("out1df = in1df.apply(_texera_missing)") { override def standaloneHelpers(): Seq[String] = Seq("def _texera_missing(x):\n return x") } diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala index bb639ef8875..2aee6ddedcd 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala @@ -180,7 +180,7 @@ class WorkflowToPythonResourceSpec extends AnyFlatSpec with BeforeAndAfterAll { val success = exported(chainOf(distinctOp("distinct"), limitOp("limit", 5))) assert(success.pythonCode.contains("from texera_support import _texera_key_codes")) assert(!success.pythonCode.contains("def _texera_key_codes")) - assert(success.supportModule.contains(StandaloneHelpers.packageModule)) + assert(success.supportModule.contains(StandaloneHelpers.supportModule)) } it should "hand over no module with a script that imports nothing from it" in { From ca9b265cbd8113d6dd4a1e4bdc4300e903272ad9 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Tue, 10 Nov 2026 20:59:32 -0800 Subject: [PATCH 4/7] feat: name the exported helper module texera_helpers The discussion on where the helpers go asked for a clearer name than texera_support. The module, its directory, the regenerate variable, the response field and the frontend property now all say helpers. Co-Authored-By: Claude Opus 5.5 --- .../operator/StandaloneCodeGenerator.scala | 4 ++-- .../amber/operator/StandaloneHelpers.scala | 12 ++++++------ .../amber/operator/StandaloneHelpersSpec.scala | 18 +++++++++--------- .../workflow-to-python.service.ts | 4 ++-- .../component/menu/menu.component.html | 4 ++-- .../component/menu/menu.component.spec.ts | 14 +++++++------- .../workspace/component/menu/menu.component.ts | 10 +++++----- {texera-support => texera-helpers}/README.md | 8 ++++---- .../texera_helpers}/__init__.py | 0 .../WorkflowToPythonTranslator.scala | 16 ++++++++-------- .../resource/WorkflowToPythonResource.scala | 8 ++++---- .../WorkflowToPythonTranslatorSpec.scala | 6 +++--- .../WorkflowToPythonResourceSpec.scala | 10 +++++----- 13 files changed, 57 insertions(+), 57 deletions(-) rename {texera-support => texera-helpers}/README.md (85%) rename {texera-support/texera_support => texera-helpers/texera_helpers}/__init__.py (100%) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala index 5ca0d3d0818..0b8ee474653 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala @@ -115,8 +115,8 @@ trait StandaloneCodeGenerator { /** * Definitions this operator's standalone code depends on. Each block must be - * one of [[StandaloneHelpers.supportHelpers]]: the exported script imports the - * names it calls from the `texera_support` module rather than defining them. + * one of [[StandaloneHelpers.moduleHelpers]]: the exported script imports the + * names it calls from the `texera_helpers` module rather than defining them. */ def standaloneHelpers(): Seq[String] = Seq.empty diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index 2ace8158e7b..144b48a537d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -395,12 +395,12 @@ object StandaloneHelpers { | dtype=object, | )""".stripMargin - /** Every shared helper block, in the order the `texera_support` module holds them. + /** Every shared helper block, in the order the `texera_helpers` module holds them. * A new block returned by some [[StandaloneCodeGenerator.standaloneHelpers]] belongs * here too: an exported script imports its helpers, and the translator refuses a * block this list lacks. */ - def supportHelpers: Seq[String] = + def moduleHelpers: Seq[String] = Seq( AttributeCasts, SamplingHelpers.JavaRandom, @@ -412,11 +412,11 @@ object StandaloneHelpers { StandaloneCodeGenerator.PythonOperatorInput.definition ) - /** `texera_support/__init__.py`: the shared blocks under the one import they share. - * The export also hands it out as `texera_support.py`, which a script finds when the + /** `texera_helpers/__init__.py`: the shared blocks under the one import they share. + * The export also hands it out as `texera_helpers.py`, which a script finds when the * two sit in one directory. */ - def supportModule: String = + def helpersModule: String = ("""# Licensed to the Apache Software Foundation (ASF) under one |# or more contributor license agreements. See the NOTICE file |# distributed with this work for additional information @@ -434,5 +434,5 @@ object StandaloneHelpers { |# specific language governing permissions and limitations |# under the License. | - |import pandas as pd""".stripMargin +: supportHelpers).mkString("", "\n\n\n", "\n") + |import pandas as pd""".stripMargin +: moduleHelpers).mkString("", "\n\n\n", "\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala index 0be7b589653..6b0275f60c1 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala @@ -25,23 +25,23 @@ import org.scalatest.matchers.should.Matchers import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path} -/** The `texera_support` module against the helpers the operators declare. */ +/** The `texera_helpers` module against the helpers the operators declare. */ class StandaloneHelpersSpec extends AnyFlatSpec with Matchers { private val module: Path = Path .of(sys.env.getOrElse("TEXERA_HOME", ".")) - .resolve("texera-support") - .resolve("texera_support") + .resolve("texera-helpers") + .resolve("texera_helpers") .resolve("__init__.py") - // With TEXERA_SUPPORT_WRITE set, the module is rewritten instead of checked. - "texera_support/__init__.py" should "hold exactly the helpers the operators declare" in { - if (sys.env.contains("TEXERA_SUPPORT_WRITE")) { - Files.write(module, StandaloneHelpers.supportModule.getBytes(StandardCharsets.UTF_8)) + // With TEXERA_HELPERS_WRITE set, the module is rewritten instead of checked. + "texera_helpers/__init__.py" should "hold exactly the helpers the operators declare" in { + if (sys.env.contains("TEXERA_HELPERS_WRITE")) { + Files.write(module, StandaloneHelpers.helpersModule.getBytes(StandardCharsets.UTF_8)) } - withClue(s"$module is stale; rerun this spec with TEXERA_SUPPORT_WRITE=1 to rewrite it.\n") { + withClue(s"$module is stale; rerun this spec with TEXERA_HELPERS_WRITE=1 to rewrite it.\n") { new String(Files.readAllBytes(module), StandardCharsets.UTF_8) shouldBe - StandaloneHelpers.supportModule + StandaloneHelpers.helpersModule } } } diff --git a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts index 465206f6ad9..6772915b964 100644 --- a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts +++ b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts @@ -28,8 +28,8 @@ export const WORKFLOW_TO_PYTHON_ENDPOINT = "workflow-to-python"; export interface WorkflowToPythonResponse { type: "success" | "failure"; pythonCode?: string; - /** The `texera_support.py` the script imports from, when it imports any. */ - supportModule?: string; + /** The `texera_helpers.py` the script imports from, when it imports any. */ + helpersModule?: string; errorMessage?: string; } diff --git a/frontend/src/app/workspace/component/menu/menu.component.html b/frontend/src/app/workspace/component/menu/menu.component.html index af548a6acd2..824442b308a 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.html +++ b/frontend/src/app/workspace/component/menu/menu.component.html @@ -534,8 +534,8 @@
- - Download the zip and run script.py next to the texera_support.py it holds. + + Download the zip and run script.py next to the texera_helpers.py it holds. diff --git a/frontend/src/app/workspace/component/menu/menu.component.spec.ts b/frontend/src/app/workspace/component/menu/menu.component.spec.ts index cc2c6393c3d..b58648375f3 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.spec.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.spec.ts @@ -766,14 +766,14 @@ describe("MenuComponent", () => { it("keeps the module the script imports for the download", () => { vi.spyOn(component["workflowToPythonService"], "convertToPython").mockReturnValue( - of({ type: "success", pythonCode: "print(1)", supportModule: "def f(): pass" } as any) + of({ type: "success", pythonCode: "print(1)", helpersModule: "def f(): pass" } as any) ); workflowActionService.addOperator(mockScanPredicate, mockPoint); validationStream$.next({ errors: {}, workflowEmpty: false }); component.onClickExportAsPython(); - expect(component.pythonSupportModuleForModal).toBe("def f(): pass"); + expect(component.pythonHelpersModuleForModal).toBe("def f(): pass"); }); }); @@ -1123,19 +1123,19 @@ describe("MenuComponent", () => { it("zips the script with the module it imports, under the workflow name", async () => { component.currentWorkflowName = "my-workflow"; - component.pythonCodeForModal = "from texera_support import f"; - component.pythonSupportModuleForModal = "def f(): pass"; + component.pythonCodeForModal = "from texera_helpers import f"; + component.pythonHelpersModuleForModal = "def f(): pass"; const { name, zip } = await download(); expect(name).toBe("my-workflow.zip"); - expect(await zip.file("script.py")?.async("string")).toBe("from texera_support import f"); - expect(await zip.file("texera_support.py")?.async("string")).toBe("def f(): pass"); + expect(await zip.file("script.py")?.async("string")).toBe("from texera_helpers import f"); + expect(await zip.file("texera_helpers.py")?.async("string")).toBe("def f(): pass"); }); it("zips the script alone when it imports nothing", async () => { component.pythonCodeForModal = "print(1)"; - component.pythonSupportModuleForModal = undefined; + component.pythonHelpersModuleForModal = undefined; const { zip } = await download(); diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts index ce1f1182688..487f0c619ba 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.ts @@ -757,7 +757,7 @@ export class MenuComponent implements OnInit, OnDestroy { public isTranslatingToPython = false; public pythonCodeForModal = ""; - public pythonSupportModuleForModal?: string; + public pythonHelpersModuleForModal?: string; public onClickExportAsPython(): void { // Exporting the valid part of the graph answers a workflow with a required @@ -781,7 +781,7 @@ export class MenuComponent implements OnInit, OnDestroy { this.isTranslatingToPython = false; if (response.type === "success") { this.pythonCodeForModal = response.pythonCode ?? ""; - this.pythonSupportModuleForModal = response.supportModule; + this.pythonHelpersModuleForModal = response.helpersModule; this.modalService.create({ nzTitle: "Workflow as Python Script", nzContent: this.workflowPythonScriptModal ?? "", @@ -813,7 +813,7 @@ export class MenuComponent implements OnInit, OnDestroy { } /** - * The script as a zip, with the `texera_support.py` it imports beside it: Python finds a + * The script as a zip, with the `texera_helpers.py` it imports beside it: Python finds a * module in the script's own directory, so the unzipped folder runs with nothing installed. */ public async downloadPythonZip(): Promise { @@ -822,8 +822,8 @@ export class MenuComponent implements OnInit, OnDestroy { } const zip = new JSZip(); zip.file("script.py", this.pythonCodeForModal); - if (this.pythonSupportModuleForModal) { - zip.file("texera_support.py", this.pythonSupportModuleForModal); + if (this.pythonHelpersModuleForModal) { + zip.file("texera_helpers.py", this.pythonHelpersModuleForModal); } const blob = await zip.generateAsync({ type: "blob" }); this.fileSaverService.saveAs(blob, `${this.currentWorkflowName || "workflow"}.zip`); diff --git a/texera-support/README.md b/texera-helpers/README.md similarity index 85% rename from texera-support/README.md rename to texera-helpers/README.md index cc0a89a5174..564b793b695 100644 --- a/texera-support/README.md +++ b/texera-helpers/README.md @@ -17,20 +17,20 @@ under the License. --> -# texera-support +# texera-helpers The helpers a Python script exported from a Texera workflow calls, so the script gives the answers the Texera engine gives. An exported script imports the helpers it calls from this module. The export -downloads as a zip that holds it as `texera_support.py` next to the script, so +downloads as a zip that holds it as `texera_helpers.py` next to the script, so the unzipped folder runs with nothing installed. -`texera_support/__init__.py` is generated from the helper definitions in +`texera_helpers/__init__.py` is generated from the helper definitions in `common/workflow-operator`. Do not edit it by hand. After changing a helper, run ``` -TEXERA_SUPPORT_WRITE=1 sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.StandaloneHelpersSpec" +TEXERA_HELPERS_WRITE=1 sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.StandaloneHelpersSpec" ``` Without the variable, the same spec fails when the file is stale. diff --git a/texera-support/texera_support/__init__.py b/texera-helpers/texera_helpers/__init__.py similarity index 100% rename from texera-support/texera_support/__init__.py rename to texera-helpers/texera_helpers/__init__.py diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala index 7013d216eae..39e28f743f5 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -101,7 +101,7 @@ class WorkflowToPythonTranslator extends LazyLogging { .foreach(script += _) script += "" - // The helpers the operator bodies below refer to come from the texera_support + // The helpers the operator bodies below refer to come from the texera_helpers // module, which the script imports. A block the module does not hold would // leave the script calling a name it cannot find, so it stops the export here. topoOrder @@ -111,10 +111,10 @@ class WorkflowToPythonTranslator extends LazyLogging { gen.standaloneHelpers() ++ gen.standaloneInputAdapter().map(_.definition) } .flatten - .find(!StandaloneHelpers.supportHelpers.contains(_)) + .find(!StandaloneHelpers.moduleHelpers.contains(_)) .foreach { block => throw new IllegalStateException( - s"A helper is missing from StandaloneHelpers.supportHelpers:\n${block.take(200)}" + s"A helper is missing from StandaloneHelpers.moduleHelpers:\n${block.take(200)}" ) } // The import goes in once the bodies are written, since which helpers the @@ -242,24 +242,24 @@ class WorkflowToPythonTranslator extends LazyLogging { } // Named one by one: a leading underscore keeps a name out of `import *`. - val imported = supportNamesIn(script.drop(helpersAt).mkString("\n")) + val imported = helperNamesIn(script.drop(helpersAt).mkString("\n")) if (imported.nonEmpty) - script.insert(helpersAt - 1, s"from texera_support import ${imported.mkString(", ")}") + script.insert(helpersAt - 1, s"from texera_helpers import ${imported.mkString(", ")}") script.mkString("\n") } // What the module defines at its top level, which is only functions and classes. - private val supportNames: Set[String] = StandaloneHelpers.supportHelpers + private val helperNames: Set[String] = StandaloneHelpers.moduleHelpers .flatMap(_.split("\n")) .flatMap(line => """(?:def|class) (\w+)""".r.findPrefixMatchOf(line).map(_.group(1))) .toSet // The module's names the given code refers to, read from code only, sorted. - private def supportNamesIn(code: String): Seq[String] = + private def helperNamesIn(code: String): Seq[String] = splitOffLiterals(code) .collect { case (true, segment) => segment } .flatMap("""\b\w+""".r.findAllIn(_)) - .filter(supportNames) + .filter(helperNames) .distinct .sorted diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala index ed4ae5c4de4..1ad8b13f1e9 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala @@ -48,8 +48,8 @@ import org.apache.texera.auth.SessionUser ) sealed trait WorkflowToPythonResponse -/** @param supportModule the `texera_support.py` the script imports from, when it imports any. */ -case class WorkflowToPythonSuccess(pythonCode: String, supportModule: Option[String] = None) +/** @param helpersModule the `texera_helpers.py` the script imports from, when it imports any. */ +case class WorkflowToPythonSuccess(pythonCode: String, helpersModule: Option[String] = None) extends WorkflowToPythonResponse case class WorkflowToPythonFailure(errorMessage: String) extends WorkflowToPythonResponse @@ -80,8 +80,8 @@ class WorkflowToPythonResource extends LazyLogging { translator.translate(logicalPlan, outputSchemasOf(logicalPlanPojo, user.getUid)) WorkflowToPythonSuccess( pythonCode, - Option.when(pythonCode.contains("from texera_support import"))( - StandaloneHelpers.supportModule + Option.when(pythonCode.contains("from texera_helpers import"))( + StandaloneHelpers.helpersModule ) ) } catch { diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala index 110f695bf28..d17e4e265ac 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -186,7 +186,7 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { LogicalPlan(source :: pythons, pythons.map(link(source, _))) ) script.linesIterator.count( - _ == "from texera_support import _texera_python_operator_input" + _ == "from texera_helpers import _texera_python_operator_input" ) shouldBe 1 script should not include "def _texera_python_operator_input" script should not include "df1 = _texera_python_operator_input(" @@ -268,14 +268,14 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { } op.setOperatorId("op") val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) - script should startWith("import pandas as pd\nfrom texera_support import _TexeraJavaRandom\n\n") + script should startWith("import pandas as pd\nfrom texera_helpers import _TexeraJavaRandom\n\n") script should not include "class _TexeraJavaRandom" } it should "not import a helper a comment or a string only names" in { val op = stub("op", "# _TexeraJavaRandom\nout1df = in1df.assign(n=\"_TexeraJavaRandom\")") val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) - script should not include "from texera_support" + script should not include "from texera_helpers" } it should "refuse an operator whose helper the module does not hold" in { diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala index 30b2e1105da..dac44c11a37 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala @@ -210,21 +210,21 @@ class WorkflowToPythonResourceSpec .readValue(postExport(pojo).readEntity(classOf[String]), classOf[WorkflowToPythonResponse]) .asInstanceOf[WorkflowToPythonSuccess] - it should "import the helpers from texera-support and hand that module over with the script" in { + it should "import the helpers from texera-helpers and hand that module over with the script" in { val sample = new ReservoirSamplingOpDesc() sample.setOperatorId("sample") sample.k = 5 val success = exported(LogicalPlanPojo(List(sample), List.empty, List.empty, List.empty)) - assert(success.pythonCode.contains("from texera_support import _TexeraJavaRandom")) + assert(success.pythonCode.contains("from texera_helpers import _TexeraJavaRandom")) assert(!success.pythonCode.contains("class _TexeraJavaRandom")) - assert(success.supportModule.contains(StandaloneHelpers.supportModule)) + assert(success.helpersModule.contains(StandaloneHelpers.helpersModule)) } it should "hand over no module with a script that imports nothing from it" in { val limit = limitOp("limit", 5) val success = exported(LogicalPlanPojo(List(limit), List.empty, List.empty, List.empty)) - assert(!success.pythonCode.contains("texera_support")) - assert(success.supportModule.isEmpty) + assert(!success.pythonCode.contains("texera_helpers")) + assert(success.helpersModule.isEmpty) } it should "return a failure body rather than HTTP 500 when the plan cannot be read" in { From 52d945053554b3ee763de0383ca175b9cad11275 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Tue, 10 Nov 2026 21:00:15 -0800 Subject: [PATCH 5/7] feat(frontend): say the copied script needs texera_helpers.py A copied script that imports its helpers does not run on its own. The copy notice now says the module comes with the zip download. Co-Authored-By: Claude Opus 5.5 --- .../component/menu/menu.component.spec.ts | 16 ++++++++++++++++ .../workspace/component/menu/menu.component.ts | 7 ++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/workspace/component/menu/menu.component.spec.ts b/frontend/src/app/workspace/component/menu/menu.component.spec.ts index b58648375f3..7dfb0f41504 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.spec.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.spec.ts @@ -1107,6 +1107,22 @@ describe("MenuComponent", () => { vi.unstubAllGlobals(); }); + it("copyPythonCodeToClipboard says where the helper module is when the script imports one", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const successSpy = vi.spyOn(notificationService, "success").mockImplementation(() => {}); + component.pythonCodeForModal = "from texera_helpers import f"; + component.pythonHelpersModuleForModal = "def f(): pass"; + + await component.copyPythonCodeToClipboard(); + + expect(writeText).toHaveBeenCalledWith("from texera_helpers import f"); + expect(successSpy).toHaveBeenCalledWith( + "Python script copied to clipboard. It needs the texera_helpers.py in the zip download next to it." + ); + vi.unstubAllGlobals(); + }); + describe("downloadPythonZip", () => { const download = async () => { const saveAs = vi.spyOn(TestBed.inject(FileSaverService), "saveAs").mockImplementation(() => {}); diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts index 487f0c619ba..c21a556b466 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.ts @@ -806,7 +806,12 @@ export class MenuComponent implements OnInit, OnDestroy { try { await navigator.clipboard.writeText(this.pythonCodeForModal); - this.notificationService.success("Python script copied to clipboard"); + // The copied script imports its helpers, so it does not run without the module. + this.notificationService.success( + this.pythonHelpersModuleForModal + ? "Python script copied to clipboard. It needs the texera_helpers.py in the zip download next to it." + : "Python script copied to clipboard" + ); } catch (error) { this.notificationService.error("Failed to copy Python script"); } From 50af7822aadeff3ae3778935d7a6ef1b5390ab1d Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 11 Nov 2026 10:18:43 -0800 Subject: [PATCH 6/7] ci: rerun checks after a macOS frontend timeout Co-Authored-By: Claude Opus 5.5 From 39ff08866be6b4ba4e2e19f2129a2d956df72bd0 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Mon, 16 Nov 2026 23:35:32 -0800 Subject: [PATCH 7/7] fix(export): keep sources off the file names the zip holds The zip unzips script.py and texera_helpers.py beside the sources, so a source by either name was overwritten by the exported code. Both names now count as taken when sources are named. Co-Authored-By: Claude Opus 5.5 --- .../src/app/workspace/component/menu/menu.component.ts | 1 + .../amber/translator/WorkflowToPythonTranslator.scala | 10 +++++++++- .../translator/WorkflowToPythonTranslatorSpec.scala | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts index c21a556b466..cf347f39af7 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.ts @@ -820,6 +820,7 @@ export class MenuComponent implements OnInit, OnDestroy { /** * The script as a zip, with the `texera_helpers.py` it imports beside it: Python finds a * module in the script's own directory, so the unzipped folder runs with nothing installed. + * WorkflowToPythonTranslator keeps sources off both file names, since they unzip beside them. */ public async downloadPythonZip(): Promise { if (!this.pythonCodeForModal) { diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala index 39e28f743f5..95d7a96a2ed 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -38,6 +38,11 @@ class WorkflowToPythonTranslator extends LazyLogging { // (e.g. Split has port 0 and port 1, each with its own assigned dfN var). private type PortKey = (String, Int) // (opId, portIdx) + // The files the export's zip holds, which unzip into the directory the script + // reads its sources from. A source offering one of these names would be + // overwritten by it, so they count as taken before any source is named. + private val archiveFileNames = Set("script.py", "texera_helpers.py") + /** * @param outputSchemas what each operator's output ports carry, as * [[org.apache.texera.common.compiler.WorkflowCompiler]] @@ -82,7 +87,10 @@ class WorkflowToPythonTranslator extends LazyLogging { gen.standaloneSourcePath().foreach { path => sourceFileNames.getOrElseUpdate( path, - distinctName(gen.standaloneSourceName().getOrElse(""), sourceFileNames.values.toSet) + distinctName( + gen.standaloneSourceName().getOrElse(""), + sourceFileNames.values.toSet ++ archiveFileNames + ) ) } case _ => () diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala index d17e4e265ac..7124b2fe06b 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -240,6 +240,15 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { script should not include "data-2.csv" } + /** The zip unzips `script.py` and `texera_helpers.py` beside the sources, so a + * source by either name would be read back as the exported code. + */ + it should "keep a source off the file names the zip holds" in { + val script = translateSources("/alice/v1/texera_helpers.py", "/alice/v1/script.py") + script should include("""pd.read_csv("texera_helpers-2.py")""") + script should include("""pd.read_csv("script-2.py")""") + } + /** A file name is the user's text, so it reaches the script through the escaper * the operators use and through the replacement quoting on top of that: a bare * `$` in a name is a group reference to `replaceAll`.