diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala index 38eb5122573..0b8ee474653 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneCodeGenerator.scala @@ -114,14 +114,9 @@ trait StandaloneCodeGenerator { def producesDataFrame(): Boolean = true /** - * Definitions this operator's standalone code depends on, emitted once near - * the top of the script rather than inline. - * - * The translator concatenates operator bodies into a single module, so an - * operator needing a helper class has nowhere to put it that another operator - * would not duplicate. Helpers returned here are collected across the whole - * plan and deduplicated by their text, so two sampling operators in one - * workflow yield one copy of the generator they share. + * Definitions this operator's standalone code depends on. Each block must be + * one of [[StandaloneHelpers.moduleHelpers]]: the exported script imports the + * names it calls from the `texera_helpers` module rather than defining them. */ def standaloneHelpers(): Seq[String] = Seq.empty diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala index 81c7667d6a9..144b48a537d 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/StandaloneHelpers.scala @@ -19,6 +19,11 @@ package org.apache.texera.amber.operator +import org.apache.texera.amber.operator.aggregate.AggregateOpDesc +import org.apache.texera.amber.operator.filter.SpecializedFilterOpDesc +import org.apache.texera.amber.operator.source.scan.file.FileScanOpDesc +import org.apache.texera.amber.operator.source.scan.json.JSONLScanSourceOpDesc + /** Python definitions shared by several operators' standalone code, emitted * once per script via [[StandaloneCodeGenerator.standaloneHelpers]]. */ @@ -389,4 +394,45 @@ object StandaloneHelpers { | name=s.name, | dtype=object, | )""".stripMargin + + /** Every shared helper block, in the order the `texera_helpers` module holds them. + * A new block returned by some [[StandaloneCodeGenerator.standaloneHelpers]] belongs + * here too: an exported script imports its helpers, and the translator refuses a + * block this list lacks. + */ + def moduleHelpers: Seq[String] = + Seq( + AttributeCasts, + SamplingHelpers.JavaRandom, + AggregateOpDesc.Helpers, + SpecializedFilterOpDesc.FilterPredicates, + FileScanOpDesc.DecodeHelper, + JSONLScanSourceOpDesc.JsonFlatten, + JSONLScanSourceOpDesc.JsonText, + StandaloneCodeGenerator.PythonOperatorInput.definition + ) + + /** `texera_helpers/__init__.py`: the shared blocks under the one import they share. + * The export also hands it out as `texera_helpers.py`, which a script finds when the + * two sit in one directory. + */ + def helpersModule: String = + ("""# Licensed to the Apache Software Foundation (ASF) under one + |# or more contributor license agreements. See the NOTICE file + |# distributed with this work for additional information + |# regarding copyright ownership. The ASF licenses this file + |# to you under the Apache License, Version 2.0 (the + |# "License"); you may not use this file except in compliance + |# with the License. You may obtain a copy of the License at + |# + |# http://www.apache.org/licenses/LICENSE-2.0 + |# + |# Unless required by applicable law or agreed to in writing, + |# software distributed under the License is distributed on an + |# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + |# KIND, either express or implied. See the License for the + |# specific language governing permissions and limitations + |# under the License. + | + |import pandas as pd""".stripMargin +: moduleHelpers).mkString("", "\n\n\n", "\n") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala new file mode 100644 index 00000000000..6b0275f60c1 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/StandaloneHelpersSpec.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path} + +/** The `texera_helpers` module against the helpers the operators declare. */ +class StandaloneHelpersSpec extends AnyFlatSpec with Matchers { + + private val module: Path = Path + .of(sys.env.getOrElse("TEXERA_HOME", ".")) + .resolve("texera-helpers") + .resolve("texera_helpers") + .resolve("__init__.py") + + // With TEXERA_HELPERS_WRITE set, the module is rewritten instead of checked. + "texera_helpers/__init__.py" should "hold exactly the helpers the operators declare" in { + if (sys.env.contains("TEXERA_HELPERS_WRITE")) { + Files.write(module, StandaloneHelpers.helpersModule.getBytes(StandardCharsets.UTF_8)) + } + withClue(s"$module is stale; rerun this spec with TEXERA_HELPERS_WRITE=1 to rewrite it.\n") { + new String(Files.readAllBytes(module), StandardCharsets.UTF_8) shouldBe + StandaloneHelpers.helpersModule + } + } +} diff --git a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts index 301515d9568..6772915b964 100644 --- a/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts +++ b/frontend/src/app/dashboard/service/user/workflow-to-python/workflow-to-python.service.ts @@ -28,6 +28,8 @@ export const WORKFLOW_TO_PYTHON_ENDPOINT = "workflow-to-python"; export interface WorkflowToPythonResponse { type: "success" | "failure"; pythonCode?: string; + /** The `texera_helpers.py` the script imports from, when it imports any. */ + helpersModule?: string; errorMessage?: string; } diff --git a/frontend/src/app/workspace/component/menu/menu.component.html b/frontend/src/app/workspace/component/menu/menu.component.html index acef7bbb9e7..824442b308a 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.html +++ b/frontend/src/app/workspace/component/menu/menu.component.html @@ -533,16 +533,33 @@
- + + + Download the zip and run script.py next to the texera_helpers.py it holds. + + + + + +
{{ pythonCodeForModal }}
diff --git a/frontend/src/app/workspace/component/menu/menu.component.scss b/frontend/src/app/workspace/component/menu/menu.component.scss index 8b2bec4d2b7..ce56ec62693 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.scss +++ b/frontend/src/app/workspace/component/menu/menu.component.scss @@ -219,7 +219,15 @@ texera-coeditor-user-icon { .workflow-python-modal-toolbar { display: flex; - justify-content: flex-end; + justify-content: space-between; + align-items: center; + gap: 8px; +} + +.workflow-python-actions { + display: flex; + flex: none; + gap: 8px; } .workflow-python-code { diff --git a/frontend/src/app/workspace/component/menu/menu.component.spec.ts b/frontend/src/app/workspace/component/menu/menu.component.spec.ts index 2bf03c3e5ec..7dfb0f41504 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.spec.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.spec.ts @@ -49,6 +49,7 @@ import { loadPersistedHeatmapView, savePersistedHeatmapView } from "../../servic import { ComputingUnitState } from "../../../common/type/computing-unit-connection.interface"; import { mockPoint, mockScanPredicate } from "../../service/workflow-graph/model/mock-workflow-data"; import { FileSaverService } from "../../../dashboard/service/user/file/file-saver.service"; +import JSZip from "jszip"; import type { ModalOptions } from "ng-zorro-antd/modal"; import type { ComputingUnitSelectionComponent } from "../power-button/computing-unit-selection.component"; import { WorkflowContent } from "../../../common/type/workflow"; @@ -762,6 +763,18 @@ describe("MenuComponent", () => { expect(convert).toHaveBeenCalled(); expect(convert.mock.calls[0][0].operators).toHaveLength(1); }); + + it("keeps the module the script imports for the download", () => { + vi.spyOn(component["workflowToPythonService"], "convertToPython").mockReturnValue( + of({ type: "success", pythonCode: "print(1)", helpersModule: "def f(): pass" } as any) + ); + workflowActionService.addOperator(mockScanPredicate, mockPoint); + validationStream$.next({ errors: {}, workflowEmpty: false }); + + component.onClickExportAsPython(); + + expect(component.pythonHelpersModuleForModal).toBe("def f(): pass"); + }); }); describe("hasOperators", () => { @@ -1094,6 +1107,58 @@ describe("MenuComponent", () => { vi.unstubAllGlobals(); }); + it("copyPythonCodeToClipboard says where the helper module is when the script imports one", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const successSpy = vi.spyOn(notificationService, "success").mockImplementation(() => {}); + component.pythonCodeForModal = "from texera_helpers import f"; + component.pythonHelpersModuleForModal = "def f(): pass"; + + await component.copyPythonCodeToClipboard(); + + expect(writeText).toHaveBeenCalledWith("from texera_helpers import f"); + expect(successSpy).toHaveBeenCalledWith( + "Python script copied to clipboard. It needs the texera_helpers.py in the zip download next to it." + ); + vi.unstubAllGlobals(); + }); + + describe("downloadPythonZip", () => { + const download = async () => { + const saveAs = vi.spyOn(TestBed.inject(FileSaverService), "saveAs").mockImplementation(() => {}); + await component.downloadPythonZip(); + const [blob, name] = saveAs.mock.calls[0] as [Blob, string]; + // jsdom's Blob has no arrayBuffer(), so it is read the way a browser without one would. + const bytes = await new Promise(resolve => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as ArrayBuffer); + reader.readAsArrayBuffer(blob); + }); + return { name, zip: await JSZip.loadAsync(bytes) }; + }; + + it("zips the script with the module it imports, under the workflow name", async () => { + component.currentWorkflowName = "my-workflow"; + component.pythonCodeForModal = "from texera_helpers import f"; + component.pythonHelpersModuleForModal = "def f(): pass"; + + const { name, zip } = await download(); + + expect(name).toBe("my-workflow.zip"); + expect(await zip.file("script.py")?.async("string")).toBe("from texera_helpers import f"); + expect(await zip.file("texera_helpers.py")?.async("string")).toBe("def f(): pass"); + }); + + it("zips the script alone when it imports nothing", async () => { + component.pythonCodeForModal = "print(1)"; + component.pythonHelpersModuleForModal = undefined; + + const { zip } = await download(); + + expect(Object.keys(zip.files)).toEqual(["script.py"]); + }); + }); + // The export flags are reset when a menu is destroyed -- right on leaving the workspace, wrong on // a hand-over between a workflow's two views, where the results are kept. A menu mounting on // retained results asks for them to be recomputed rather than offering a dead button. diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts index f12520342e1..cf347f39af7 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.ts @@ -77,6 +77,7 @@ import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; import { JupyterPanelService } from "../../service/jupyter-panel/jupyter-panel.service"; import { WorkflowCompilingService } from "../../service/compile-workflow/workflow-compiling.service"; import { CompilationState } from "../../types/workflow-compiling.interface"; +import JSZip from "jszip"; /** * MenuComponent is the top level menu bar that shows @@ -756,6 +757,7 @@ export class MenuComponent implements OnInit, OnDestroy { public isTranslatingToPython = false; public pythonCodeForModal = ""; + public pythonHelpersModuleForModal?: string; public onClickExportAsPython(): void { // Exporting the valid part of the graph answers a workflow with a required @@ -779,6 +781,7 @@ export class MenuComponent implements OnInit, OnDestroy { this.isTranslatingToPython = false; if (response.type === "success") { this.pythonCodeForModal = response.pythonCode ?? ""; + this.pythonHelpersModuleForModal = response.helpersModule; this.modalService.create({ nzTitle: "Workflow as Python Script", nzContent: this.workflowPythonScriptModal ?? "", @@ -803,12 +806,35 @@ export class MenuComponent implements OnInit, OnDestroy { try { await navigator.clipboard.writeText(this.pythonCodeForModal); - this.notificationService.success("Python script copied to clipboard"); + // The copied script imports its helpers, so it does not run without the module. + this.notificationService.success( + this.pythonHelpersModuleForModal + ? "Python script copied to clipboard. It needs the texera_helpers.py in the zip download next to it." + : "Python script copied to clipboard" + ); } catch (error) { this.notificationService.error("Failed to copy Python script"); } } + /** + * The script as a zip, with the `texera_helpers.py` it imports beside it: Python finds a + * module in the script's own directory, so the unzipped folder runs with nothing installed. + * WorkflowToPythonTranslator keeps sources off both file names, since they unzip beside them. + */ + public async downloadPythonZip(): Promise { + if (!this.pythonCodeForModal) { + return; + } + const zip = new JSZip(); + zip.file("script.py", this.pythonCodeForModal); + if (this.pythonHelpersModuleForModal) { + zip.file("texera_helpers.py", this.pythonHelpersModuleForModal); + } + const blob = await zip.generateAsync({ type: "blob" }); + this.fileSaverService.saveAs(blob, `${this.currentWorkflowName || "workflow"}.zip`); + } + public onClickExportWorkflow(): void { // The same shape the dashboard download produces (see exportedWorkflow): the content plus the // landing view as a sibling key, so a file exported here uploads as a form-default workflow too. diff --git a/texera-helpers/README.md b/texera-helpers/README.md new file mode 100644 index 00000000000..564b793b695 --- /dev/null +++ b/texera-helpers/README.md @@ -0,0 +1,36 @@ + + +# texera-helpers + +The helpers a Python script exported from a Texera workflow calls, so the +script gives the answers the Texera engine gives. + +An exported script imports the helpers it calls from this module. The export +downloads as a zip that holds it as `texera_helpers.py` next to the script, so +the unzipped folder runs with nothing installed. + +`texera_helpers/__init__.py` is generated from the helper definitions in +`common/workflow-operator`. Do not edit it by hand. After changing a helper, run + +``` +TEXERA_HELPERS_WRITE=1 sbt "WorkflowOperator/testOnly org.apache.texera.amber.operator.StandaloneHelpersSpec" +``` + +Without the variable, the same spec fails when the file is stale. diff --git a/texera-helpers/texera_helpers/__init__.py b/texera-helpers/texera_helpers/__init__.py new file mode 100644 index 00000000000..72a70ca9f77 --- /dev/null +++ b/texera-helpers/texera_helpers/__init__.py @@ -0,0 +1,615 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pandas as pd + + +def _texera_cast_boolean(x): + if isinstance(x, str): + text = _texera_java_trim(x) + lowered = text.lower() + if lowered == "true": + return True + if lowered == "false": + return False + return _texera_parse_int(text, 32) == 1 + return x != 0 + +def _texera_parse_number(text): + import re + + match = re.match(r"\s*(-?)([0-9,]*)(?:\.([0-9]*))?", text) + sign = -1 if match.group(1) == "-" else 1 + digits = (match.group(2) or "").replace(",", "") + fraction = match.group(3) or "" + if not digits and not fraction: + raise ValueError("Unparseable number: " + repr(text)) + if fraction: + return sign * float((digits or "0") + "." + fraction), True + value = sign * int(digits) + if -(2 ** 63) <= value <= 2 ** 63 - 1: + return value, False + return float(value), True + +def _texera_saturate(value, low, high): + if value != value: + return 0 + if value <= low: + return low + if value >= high: + return high + return int(value) + +def _texera_cast_integral(x): + if isinstance(x, str): + x, is_double = _texera_parse_number(x) + if not is_double: + return x + if isinstance(x, bool): + return 1 if x else 0 + if isinstance(x, float): + return _texera_saturate(x, -(2 ** 63), 2 ** 63 - 1) + return int(x) + +def _texera_cast_int32(x, wrap): + if isinstance(x, str): + value, is_double = _texera_parse_number(x) + wrap = not is_double + elif isinstance(x, bool): + value = 1 if x else 0 + else: + value = x + if wrap: + return ((int(value) + 2147483648) % 4294967296) - 2147483648 + return _texera_saturate(value, -2147483648, 2147483647) + +def _texera_text_to_timestamp(s): + from dateutil.parser import parse as _parse_date + + zone = _texera_local_zone() + + def _one(x): + try: + parsed = _parse_date(str(x).strip()) + except (ValueError, OverflowError) as e: + raise ValueError(f"Failed to parse type str to Timestamp: {x}") from e + if parsed.tzinfo is not None: + parsed = _texera_local_wall_clock(parsed, zone) + return parsed.replace(microsecond=parsed.microsecond // 1000 * 1000) + + text = s.astype(str).str.strip() + plain = s.notna() & text.str.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}([ T][0-9]{2}:[0-9]{2}(:[0-9]{2}(\.[0-9]+)?)?)?" + ) + out = pd.Series(pd.NaT, index=s.index, dtype="datetime64[us]") + out[plain] = pd.to_datetime(text[plain], format="ISO8601", errors="coerce").dt.floor("ms") + rest = s.notna() & out.isna() + out[rest] = s[rest].map(_one).astype("datetime64[us]") + return _texera_skip_gaps(out, zone) + +def _texera_skip_gaps(walls, zone): + import datetime + + if zone is None: + return walls + inner = walls.between(datetime.datetime(1, 1, 2), datetime.datetime(9999, 12, 30)) + gap = pd.Series(False, index=walls.index) + gap[inner] = walls[inner].dt.tz_localize(zone, ambiguous=False, nonexistent="NaT").isna() + + def _later(t): + moment = t.to_pydatetime().replace(tzinfo=zone).astimezone(datetime.timezone.utc) + return moment.astimezone(zone).replace(tzinfo=None) + + walls[gap] = walls[gap].map(_later).astype("datetime64[us]") + return walls + +def _texera_cast_timestamps(s): + if not pd.api.types.is_datetime64_any_dtype(s): + return _texera_text_to_timestamp(s) + if getattr(s.dtype, "tz", None) is not None: + return _texera_local_wall_clocks(s) + return s + +def _texera_epoch_millis_to_timestamp(s): + return _texera_local_wall_clocks(pd.to_datetime(s, unit="ms", errors="coerce", utc=True)) + +def _texera_local_zone(): + import os + import zoneinfo + + names = [os.environ.get("TZ", "").lstrip(":")] + try: + import tzlocal + + names.insert(0, str(tzlocal.get_localzone())) + except Exception: + pass + link = os.path.realpath("/etc/localtime") + if "zoneinfo/" in link: + names.append(link.split("zoneinfo/", 1)[1]) + for name in names: + try: + return zoneinfo.ZoneInfo(name) + except (ValueError, KeyError): + continue + return None + +def _texera_standard_offset(zone): + import datetime + import time + + if zone is None: + return datetime.timedelta(seconds=-time.timezone) + now = datetime.datetime.now(zone) + return now.utcoffset() - now.dst() + +def _texera_local_wall_clock(moment, zone): + import datetime + + from dateutil.tz import tzlocal + + utc = moment.astimezone(datetime.timezone.utc).replace(tzinfo=None) + if utc < datetime.datetime(1900, 1, 1): + return utc + _texera_standard_offset(zone) + if zone is not None: + return moment.astimezone(zone).replace(tzinfo=None) + try: + return moment.astimezone(tzlocal()).replace(tzinfo=None) + except (OSError, OverflowError): + return utc + _texera_standard_offset(zone) + +def _texera_local_wall_clocks(s): + import datetime + + from dateutil.tz import tzlocal + + zone = _texera_local_zone() + utc = s.dt.tz_convert("UTC").dt.tz_localize(None) + if zone is not None: + wall = s.dt.tz_convert(zone).dt.tz_localize(None) + before = utc < pd.Timestamp(1900, 1, 1) + return wall.where(~before, utc + _texera_standard_offset(zone)) + try: + return s.dt.tz_convert(tzlocal()).dt.tz_localize(None) + except (OSError, OverflowError): + pass + + def _offset(t): + if pd.isna(t): + return 0 + moment = t.to_pydatetime(warn=False) + utc = moment.astimezone(datetime.timezone.utc).replace(tzinfo=None) + return int((_texera_local_wall_clock(moment, None) - utc).total_seconds()) + + seconds = pd.Series([_offset(t) for t in s], index=s.index) + return utc + pd.to_timedelta(seconds, unit="s").astype("timedelta64[s]") + +def _texera_wall_clock_instants(s): + from dateutil.tz import tzlocal + + walls = s.dt.floor("ms").astype("datetime64[ms]") + zone = _texera_local_zone() + standard = walls - _texera_standard_offset(zone) + before = standard < pd.Timestamp(1900, 1, 1) + + def localized(t): + return ( + t.dt.tz_localize(zone or tzlocal(), ambiguous=False, nonexistent=pd.Timedelta("1h")) + .dt.tz_convert("UTC") + .dt.tz_localize(None) + ) + + try: + return localized(walls).where(~before, standard) + except (OSError, OverflowError): + pass + + def one(i): + if before.iloc[i]: + return standard.iloc[i] + try: + return localized(walls.iloc[i : i + 1]).iloc[0] + except (OSError, OverflowError): + return standard.iloc[i] + + return pd.Series([one(i) for i in range(len(walls))], index=s.index, dtype="datetime64[ms]") + +def _texera_cast_double(x): + return _texera_java_double(x) if isinstance(x, str) else float(x) + +def _texera_cast_doubles(s): + holes = s.isna().to_numpy() + values = _texera_cast_cells(s, _texera_cast_double, 0.0).to_numpy(dtype="float64") + return pd.Series(pd.arrays.FloatingArray(values, holes), index=s.index, name=s.name) + +def _texera_java_text(x): + import datetime + import decimal + + if pd.api.types.is_bool(x): + return "true" if x else "false" + if isinstance(x, datetime.datetime): + nanos = x.microsecond * 1000 + getattr(x, "nanosecond", 0) + return x.strftime("%Y-%m-%d %H:%M:%S") + "." + (f"{nanos:09d}".rstrip("0") or "0") + if pd.api.types.is_integer(x): + return str(int(x)) + if pd.api.types.is_float(x): + value = float(x) + if value != value: + return "NaN" + if value in (float("inf"), float("-inf")): + return "Infinity" if value > 0 else "-Infinity" + if value == 0 or 1e-3 <= abs(value) < 1e7: + return repr(value) + sign, digits, exponent = decimal.Decimal(repr(value)).as_tuple() + power = len(digits) + exponent - 1 + kept = "".join(map(str, digits)).rstrip("0") or "0" + return ("-" if sign else "") + kept[0] + "." + (kept[1:] or "0") + "E" + str(power) + return str(x) + +def _texera_java_trim(text): + return text.strip("".join(map(chr, range(33)))) + +def _texera_parse_int(text, bits): + import re + + text = _texera_java_trim(text) + bound = 2 ** (bits - 1) + if not re.fullmatch(r"[+-]?\d+", text) or not -bound <= int(text) < bound: + raise ValueError("For input string: " + repr(text)) + return int(text) + +def _texera_java_double(text): + import re + + text = _texera_java_trim(text) + if not re.fullmatch( + r"[+-]?(NaN|Infinity|((\d+\.?\d*|\.\d+)([eE][+-]?\d+)?" + r"|0[xX]([0-9a-fA-F]+\.?|[0-9a-fA-F]*\.[0-9a-fA-F]+)[pP][+-]?\d+)[fFdD]?)", + text, + re.ASCII, + ): + raise ValueError("For input string: " + repr(text)) + if text[-1] in "dDfF": + text = text[:-1] + if "x" not in text.lower(): + return float(text) + try: + return float.fromhex(text) + except OverflowError: + return float("-inf") if text.startswith("-") else float("inf") + +def _texera_double_key(x): + import math + + if math.isnan(x): + return (1, 0.0, 0.0) + return (0, x, math.copysign(1.0, x)) + +def _texera_key_codes(*frames, signed=False): + import math + + codes = {} + + def key(v, missing): + if missing: + return None + if isinstance(v, float): + if math.isnan(v): + return object() + if signed: + return (v, math.copysign(1.0, v)) + return v + + def code(df): + out = [] + for values, missing in zip(df.itertuples(index=False, name=None), df.isna().to_numpy()): + row = tuple(key(v, m) for v, m in zip(values, missing)) + out.append(codes.setdefault(row, len(codes))) + return pd.Series(out, index=df.index, dtype="int64") + + return [code(df) for df in frames] + +def _texera_read_doubles(s): + import numpy as np + + if pd.api.types.is_numeric_dtype(s): + return s + holes = s.isna().to_numpy() + values = np.array( + [np.nan if hole else _texera_java_double(x) for x, hole in zip(s, holes)], + dtype="float64", + ) + if np.isnan(values[~holes]).any(): + values = pd.arrays.FloatingArray(values, holes) + return pd.Series(values, index=s.index, name=s.name) + +def _texera_cast_string(s): + return _texera_cast_cells(s, _texera_java_text, None) + +def _texera_cast_cells(s, cast, hole): + holes = s.isna().to_numpy() + return pd.Series( + [hole if empty else cast(x) for x, empty in zip(s, holes)], + index=s.index, + name=s.name, + dtype=object, + ) + + +class _TexeraJavaRandom: + _MASK = (1 << 48) - 1 + _MULTIPLIER = 0x5DEECE66D + _ADDEND = 0xB + + def __init__(self, seed): + self._seed = (seed ^ self._MULTIPLIER) & self._MASK + + def _next(self, bits): + self._seed = (self._seed * self._MULTIPLIER + self._ADDEND) & self._MASK + value = self._seed >> (48 - bits) + return value - (1 << 32) if value >= (1 << 31) else value + + def next_double(self): + return ((self._next(26) << 27) + self._next(27)) * (2.0 ** -53) + + def next_int(self, bound): + if bound <= 0: + raise ValueError("bound must be positive") + if bound & (-bound) == bound: + return (bound * self._next(31)) >> 31 + while True: + bits = self._next(31) + value = bits % bound + probe = bits - value + (bound - 1) + if ((probe + (1 << 31)) % (1 << 32)) - (1 << 31) >= 0: + return value + + +def _texera_agg_concat(series): + partial = "" + for v, missing in zip(series, series.isna()): + text = "" if missing else _texera_java_text(v) + partial = text if partial == "" else partial + "," + text + return partial + +def _texera_agg_int_sum(series): + total = int(series.sum()) + return ((total + (1 << 31)) % (1 << 32)) - (1 << 31) + +def _texera_agg_ts_epoch_ms(series): + return _texera_wall_clock_instants(series.dropna()).astype("int64") + +def _texera_agg_ts_sum(series): + total = int(_texera_agg_ts_epoch_ms(series).astype(object).sum()) + total = ((total + (1 << 63)) % (1 << 64)) - (1 << 63) + import datetime + + utc = pd.Timestamp(total, unit="ms") + zone = _texera_local_zone() + try: + moment = utc.to_pydatetime().replace(tzinfo=datetime.timezone.utc) + shift = _texera_local_wall_clock(moment, zone) - moment.replace(tzinfo=None) + except (OSError, OverflowError): + shift = _texera_standard_offset(zone) + return utc + shift + +def _texera_agg_ts_mean(series): + kept = _texera_agg_ts_epoch_ms(series) + if len(kept) == 0: + return None + return float(kept.astype(object).sum()) / len(kept) + +def _texera_agg_ts_pick(series, better): + kept = series.dropna() + if len(kept) == 0: + return pd.NaT + millis = kept.dt.floor("ms") + pick = 0 + for i in range(1, len(kept)): + if better(millis.iloc[i], millis.iloc[pick]): + pick = i + return kept.iloc[pick] + +def _texera_agg_ts_min(series): + return _texera_agg_ts_pick(series, lambda a, b: a < b) + +def _texera_agg_ts_max(series): + pick = _texera_agg_ts_pick(series, lambda a, b: a > b) + if pd.isna(pick) or _texera_agg_ts_epoch_ms(pd.Series([pick])).iloc[0] <= 0: + return pd.NaT + return pick + +def _texera_agg_double_values(series): + return [float(v) for v in series.dropna()] + +def _texera_agg_double_sum(series): + partial = 0.0 + for v in _texera_agg_double_values(series): + partial += v + return partial + +def _texera_agg_double_mean(series): + kept = _texera_agg_double_values(series) + if not kept: + return pd.NA + return _texera_agg_double_sum(series) / len(kept) + +def _texera_agg_double_min(series): + start = 1.7976931348623157e308 + partial = start + for v in _texera_agg_double_values(series): + if _texera_double_key(v) < _texera_double_key(partial): + partial = v + return pd.NA if partial == start else partial + +def _texera_agg_double_max(series): + start = float("-inf") + partial = start + for v in _texera_agg_double_values(series): + if _texera_double_key(v) > _texera_double_key(partial): + partial = v + return pd.NA if partial == start else partial + +def _texera_agg_double_by_group(column, fold): + results = column.apply(fold) + mask = results.map(lambda v: v is pd.NA).to_numpy(bool) + values = results.where(~mask, 0.0).to_numpy(float) + return pd.Series(pd.arrays.FloatingArray(values, mask), index=results.index) + + +def _texera_filter_text_key(s): + return s.encode("utf-16-be", "surrogatepass") + +def _texera_filter_compared(left, right): + return (left > right) - (left < right) + +def _texera_filter_rows(column, condition, order): + passes = { + "=": lambda c: c == 0, + ">": lambda c: c > 0, + ">=": lambda c: c >= 0, + "<": lambda c: c < 0, + "<=": lambda c: c <= 0, + "!=": lambda c: c != 0, + }[condition] + return pd.Series( + [ + False if missing else passes(order(v)) + for v, missing in zip(column, column.isna()) + ], + index=column.index, + dtype=bool, + ) + +def _texera_filter_keep_double(column, condition, value): + target = _texera_double_key(_texera_java_double(value)) + + def order(v): + return _texera_filter_compared(_texera_double_key(float(v)), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_long(column, condition, value): + target = _texera_parse_int(value, 64) + + def order(v): + return _texera_filter_compared(int(v), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_boolean(column, condition, value): + target = _texera_filter_text_key(_texera_java_trim(value).lower()) + + def order(v): + return _texera_filter_compared(_texera_filter_text_key("true" if v else "false"), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_timestamp(column, condition, value): + zone = _texera_local_zone() + + def wall_clock(ts): + ts = pd.Timestamp(ts).floor("ms") + if ts.tzinfo is None: + return ts + return pd.Timestamp(_texera_local_wall_clock(ts.to_pydatetime(), zone)) + + target = wall_clock(_texera_java_trim(value)) + + def order(v): + return _texera_filter_compared(wall_clock(v), target) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep_string(column, condition, value): + try: + value_number = _texera_double_key(_texera_java_double(value)) + except ValueError: + value_number = None + + def order(v): + text = str(v) + if value_number is not None: + try: + number = _texera_double_key(_texera_java_double(text)) + return _texera_filter_compared(number, value_number) + except ValueError: + pass + return _texera_filter_compared( + _texera_filter_text_key(text), _texera_filter_text_key(value) + ) + + return _texera_filter_rows(column, condition, order) + +def _texera_filter_keep(column, condition, value): + dtype = column.dtype + if pd.api.types.is_bool_dtype(dtype): + return _texera_filter_keep_boolean(column, condition, value) + if pd.api.types.is_integer_dtype(dtype) or pd.api.types.is_float_dtype(dtype): + return _texera_filter_keep_double(column, condition, value) + if pd.api.types.is_datetime64_any_dtype(dtype): + return _texera_filter_keep_timestamp(column, condition, value) + return _texera_filter_keep_string(column, condition, value) + + +def _texera_decode(raw, encoding): + if encoding == "utf-16": + if raw[:2] == b"\xfe\xff": + return raw[2:].decode("utf-16-be", "replace") + if raw[:2] == b"\xff\xfe": + return raw[2:].decode("utf-16-le", "replace") + return raw.decode("utf-16-be", "replace") + return raw.decode(encoding, "replace") + + +def _texera_json_flatten(record): + flat = {} + stack = [(record, "")] + while stack: + node, parent = stack.pop() + if isinstance(node, dict): + for key, child in node.items(): + path = parent + "." + key if parent else key + if isinstance(child, (dict, list)): + stack.append((child, path)) + else: + flat[path] = child + elif isinstance(node, list): + for index, child in enumerate(node): + stack.append((child, parent + str(index + 1))) + elif parent: + flat[parent] = node + return flat + + +def _texera_json_text(value): + if value is None: + return "null" + if isinstance(value, (bool, int, float)): + return _texera_java_text(value) + if isinstance(value, str): + return value + return None + + +def _texera_python_operator_input(df): + df = df.copy() + for _c in df.columns: + if isinstance(df[_c].dtype, (pd.Float64Dtype, pd.Float32Dtype)): + df[_c] = df[_c].astype("float64") + return df diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala index 7d08a2626e1..95d7a96a2ed 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslator.scala @@ -24,7 +24,7 @@ import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.core.virtualidentity.OperatorIdentity import org.apache.texera.amber.core.workflow.PortIdentity import org.apache.texera.common.compiler.model.LogicalPlan -import org.apache.texera.amber.operator.StandaloneCodeGenerator +import org.apache.texera.amber.operator.{StandaloneCodeGenerator, StandaloneHelpers} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral import java.util.regex.Matcher @@ -38,6 +38,11 @@ class WorkflowToPythonTranslator extends LazyLogging { // (e.g. Split has port 0 and port 1, each with its own assigned dfN var). private type PortKey = (String, Int) // (opId, portIdx) + // The files the export's zip holds, which unzip into the directory the script + // reads its sources from. A source offering one of these names would be + // overwritten by it, so they count as taken before any source is named. + private val archiveFileNames = Set("script.py", "texera_helpers.py") + /** * @param outputSchemas what each operator's output ports carry, as * [[org.apache.texera.common.compiler.WorkflowCompiler]] @@ -82,7 +87,10 @@ class WorkflowToPythonTranslator extends LazyLogging { gen.standaloneSourcePath().foreach { path => sourceFileNames.getOrElseUpdate( path, - distinctName(gen.standaloneSourceName().getOrElse(""), sourceFileNames.values.toSet) + distinctName( + gen.standaloneSourceName().getOrElse(""), + sourceFileNames.values.toSet ++ archiveFileNames + ) ) } case _ => () @@ -101,19 +109,24 @@ class WorkflowToPythonTranslator extends LazyLogging { .foreach(script += _) script += "" - // Helper definitions the operator bodies below refer to. Collected across the - // whole plan and deduplicated by text, so a workflow holding two operators - // that share one helper still emits it once. Order follows the topological - // order, which keeps the script stable for a given plan. They go in once the - // bodies are written, since which of them the script calls is only known then. - val helpers = topoOrder + // The helpers the operator bodies below refer to come from the texera_helpers + // module, which the script imports. A block the module does not hold would + // leave the script calling a name it cannot find, so it stops the export here. + topoOrder .map(logicalPlan.getOperator) .collect { case gen: StandaloneCodeGenerator => gen.standaloneHelpers() ++ gen.standaloneInputAdapter().map(_.definition) } .flatten - .distinct + .find(!StandaloneHelpers.moduleHelpers.contains(_)) + .foreach { block => + throw new IllegalStateException( + s"A helper is missing from StandaloneHelpers.moduleHelpers:\n${block.take(200)}" + ) + } + // The import goes in once the bodies are written, since which helpers the + // script calls is only known then. val helpersAt = script.length for (opIdentity <- topoOrder) { @@ -236,60 +249,27 @@ class WorkflowToPythonTranslator extends LazyLogging { } } - // One blank line between helpers, as each block spaces its own, and two - // around the helpers as a whole. - val called = calledHelpers(helpers, script.drop(helpersAt).mkString("\n")) - if (called.nonEmpty) script.insertAll(helpersAt, Seq("", called.mkString("\n\n"), "", "")) + // Named one by one: a leading underscore keeps a name out of `import *`. + val imported = helperNamesIn(script.drop(helpersAt).mkString("\n")) + if (imported.nonEmpty) + script.insert(helpersAt - 1, s"from texera_helpers import ${imported.mkString(", ")}") script.mkString("\n") } - // The helpers with only the definitions the script calls, directly or through - // another helper. A helper block serves several operators, so it can hold far - // more functions than one plan uses, and those would only be text for the - // reader to wade through. Only a top-level `def _texera_...` is ever dropped: - // anything else a block holds is kept, and what it calls counts as called. A - // name is read from code only, so a comment or a string naming a helper keeps - // nothing alive. A block left with nothing is dropped whole. - private def calledHelpers(helpers: Seq[String], body: String): Seq[String] = { - val blocks = helpers.map(topLevelUnits) - val defined = blocks.flatten.collect { case (Some(name), text) => name -> text }.toMap - val called = mutable.Set[String]() - def visit(text: String): Unit = { - val code = splitOffLiterals(text).collect { case (true, segment) => segment } - """\b_texera_\w+""".r.findAllIn(code.mkString("\n")).foreach { name => - if (defined.contains(name) && called.add(name)) visit(defined(name)) - } - } - visit(body) - blocks.flatten.collect { case (None, text) => text }.foreach(visit) - blocks - .map(_.collect { case (name, text) if name.forall(called) => text }.mkString) - .map(_.replaceAll("\\s+$", "")) - .filter(_.nonEmpty) - } - - // A helper block cut at its top-level statements, each with the comment and - // blank lines just above it and the indented lines below it, and named when it - // is a `def _texera_...`. - private def topLevelUnits(block: String): List[(Option[String], String)] = { - val units = ArrayBuffer[(Option[String], StringBuilder)]() - val lead = new StringBuilder - block.split("\n", -1).foreach { line => - val topLevel = line.nonEmpty && !line.head.isWhitespace - if (topLevel && !line.startsWith("#") && !line.startsWith("@")) { - val name = """def (_texera_\w+)\s*\(""".r.findPrefixMatchOf(line).map(_.group(1)) - units += ((name, new StringBuilder(lead.toString))) - lead.setLength(0) - units.last._2 ++= line + "\n" - } else if (topLevel || units.isEmpty || lead.nonEmpty) { - lead ++= line + "\n" - } else { - units.last._2 ++= line + "\n" - } - } - val cut = units.map { case (name, text) => (name, text.toString) }.toList - if (lead.isEmpty) cut else cut :+ ((None, lead.toString)) - } + // What the module defines at its top level, which is only functions and classes. + private val helperNames: Set[String] = StandaloneHelpers.moduleHelpers + .flatMap(_.split("\n")) + .flatMap(line => """(?:def|class) (\w+)""".r.findPrefixMatchOf(line).map(_.group(1))) + .toSet + + // The module's names the given code refers to, read from code only, sorted. + private def helperNamesIn(code: String): Seq[String] = + splitOffLiterals(code) + .collect { case (true, segment) => segment } + .flatMap("""\b\w+""".r.findAllIn(_)) + .filter(helperNames) + .distinct + .sorted // The stem of the files one operator writes, taken from its name so a reader // can tell whose picture is whose. Numbered from one even when the plan holds diff --git a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala index 633d1e4f594..1ad8b13f1e9 100644 --- a/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala +++ b/workflow-compiling-service/src/main/scala/org/apache/texera/service/resource/WorkflowToPythonResource.scala @@ -31,6 +31,7 @@ import org.apache.texera.amber.core.virtualidentity.{OperatorIdentity, WorkflowI import org.apache.texera.amber.core.workflow.{PortIdentity, WorkflowContext} import org.apache.texera.common.compiler.model.{LogicalPlan, LogicalPlanPojo} import org.apache.texera.common.compiler.{CompilationErrorHandling, WorkflowCompiler} +import org.apache.texera.amber.operator.StandaloneHelpers import org.apache.texera.amber.translator.WorkflowToPythonTranslator import org.apache.texera.auth.SessionUser @@ -47,7 +48,9 @@ import org.apache.texera.auth.SessionUser ) sealed trait WorkflowToPythonResponse -case class WorkflowToPythonSuccess(pythonCode: String) extends WorkflowToPythonResponse +/** @param helpersModule the `texera_helpers.py` the script imports from, when it imports any. */ +case class WorkflowToPythonSuccess(pythonCode: String, helpersModule: Option[String] = None) + extends WorkflowToPythonResponse case class WorkflowToPythonFailure(errorMessage: String) extends WorkflowToPythonResponse @@ -75,7 +78,12 @@ class WorkflowToPythonResource extends LazyLogging { // column was DECLARED as, which the file it reads cannot carry. val pythonCode = translator.translate(logicalPlan, outputSchemasOf(logicalPlanPojo, user.getUid)) - WorkflowToPythonSuccess(pythonCode) + WorkflowToPythonSuccess( + pythonCode, + Option.when(pythonCode.contains("from texera_helpers import"))( + StandaloneHelpers.helpersModule + ) + ) } catch { case e: Exception => logger.error("Failed to translate workflow to Python", e) diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala index 62cb703d812..7124b2fe06b 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/WorkflowToPythonTranslatorSpec.scala @@ -26,6 +26,7 @@ import org.apache.texera.amber.core.tuple.Schema import org.apache.texera.amber.operator.{ LogicalOp, PythonOperatorDescriptor, + SamplingHelpers, StandaloneCodeGenerator } import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlan} @@ -178,13 +179,16 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { script should not include "_texera_python_operator_input" } - it should "define the Python operator input helper once and leave the upstream frame alone" in { + it should "import the Python operator input helper once and leave the upstream frame alone" in { val source = upstream("source") val pythons = List("a", "b").map(id => pythonStub(id, "out1df = in1df.copy()")) val script = new WorkflowToPythonTranslator().translate( LogicalPlan(source :: pythons, pythons.map(link(source, _))) ) - script.linesIterator.count(_ == "def _texera_python_operator_input(df):") shouldBe 1 + script.linesIterator.count( + _ == "from texera_helpers import _texera_python_operator_input" + ) shouldBe 1 + script should not include "def _texera_python_operator_input" script should not include "df1 = _texera_python_operator_input(" } @@ -217,60 +221,6 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { script should include("""fig.write_html("stub_2.html")""") } - /** A stub that declares `helper` and runs `block`. */ - private def withHelper(id: String, helper: String, block: String): LogicalOp = { - val op = new StubOp(block) { - override def standaloneHelpers(): Seq[String] = Seq(helper) - } - op.setOperatorId(id) - op - } - - private val sharedHelper = - """import math - | - |def _texera_called(x): - | return _texera_reached(x) - | - |# Only ever named, never called. - |def _texera_named(x): - | return x - | - |def _texera_reached(x): - | return math.floor(x) - | - |def _texera_unused(x): - | return x""".stripMargin - - /** A helper block serves several operators, so one plan rarely calls all of - * it. What it does not call has no place in the script. - */ - it should "emit only the helper definitions the script calls" in { - val op = withHelper("op", sharedHelper, "out1df = in1df.apply(_texera_called)") - val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) - script should include("def _texera_called(x):") - script should include("def _texera_reached(x):") - script should not include "def _texera_unused(x):" - script should include("import math") - } - - it should "not keep a helper alive for a comment or a string naming it" in { - val op = withHelper( - "op", - sharedHelper, - "# _texera_named\nout1df = in1df.assign(note=\"_texera_named\")" - ) - val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) - script should not include "def _texera_named(x):" - script should not include "# Only ever named, never called." - } - - it should "emit nothing of a helper block whose definitions the script never calls" in { - val op = withHelper("op", "def _texera_unused(x):\n return x", "out1df = in1df.copy()") - val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) - script should not include "_texera_unused" - } - /** A source offers the last segment of its resolved path, which two sources * reading different files can spell the same. Handing both that one name left * the script reading one file twice and saying nothing about it. @@ -290,6 +240,15 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { script should not include "data-2.csv" } + /** The zip unzips `script.py` and `texera_helpers.py` beside the sources, so a + * source by either name would be read back as the exported code. + */ + it should "keep a source off the file names the zip holds" in { + val script = translateSources("/alice/v1/texera_helpers.py", "/alice/v1/script.py") + script should include("""pd.read_csv("texera_helpers-2.py")""") + script should include("""pd.read_csv("script-2.py")""") + } + /** A file name is the user's text, so it reaches the script through the escaper * the operators use and through the replacement quoting on top of that: a bare * `$` in a name is a group reference to `replaceAll`. @@ -312,6 +271,31 @@ class WorkflowToPythonTranslatorSpec extends AnyFlatSpec with Matchers { new WorkflowToPythonTranslator().translate(LogicalPlan(ops.toList, List.empty)) } + it should "import a helper with the other imports rather than define it" in { + val op = new StubOp("_rng = _TexeraJavaRandom(1)\nout1df = in1df.copy()") { + override def standaloneHelpers(): Seq[String] = Seq(SamplingHelpers.JavaRandom) + } + op.setOperatorId("op") + val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + script should startWith("import pandas as pd\nfrom texera_helpers import _TexeraJavaRandom\n\n") + script should not include "class _TexeraJavaRandom" + } + + it should "not import a helper a comment or a string only names" in { + val op = stub("op", "# _TexeraJavaRandom\nout1df = in1df.assign(n=\"_TexeraJavaRandom\")") + val script = new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + script should not include "from texera_helpers" + } + + it should "refuse an operator whose helper the module does not hold" in { + val op = new StubOp("out1df = in1df.apply(_texera_missing)") { + override def standaloneHelpers(): Seq[String] = Seq("def _texera_missing(x):\n return x") + } + op.setOperatorId("op") + an[IllegalStateException] should be thrownBy + new WorkflowToPythonTranslator().translate(LogicalPlan(List(op), List.empty)) + } + /** The translator's own contract when it meets an operator it cannot render: * a comment rather than a silently wrong line. */ diff --git a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala index 5d3ade4f87e..dac44c11a37 100644 --- a/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala +++ b/workflow-compiling-service/src/test/scala/org/apache/texera/service/resource/WorkflowToPythonResourceSpec.scala @@ -24,8 +24,10 @@ import io.dropwizard.testing.junit5.ResourceExtension import jakarta.ws.rs.client.Entity import jakarta.ws.rs.core.{MediaType, Response} import org.apache.texera.amber.core.workflow.PortIdentity +import org.apache.texera.amber.operator.StandaloneHelpers import org.apache.texera.amber.operator.distinct.DistinctOpDesc import org.apache.texera.amber.operator.limit.LimitOpDesc +import org.apache.texera.amber.operator.reservoirsampling.ReservoirSamplingOpDesc import org.apache.texera.amber.operator.source.scan.csv.CSVScanSourceOpDesc import org.apache.texera.amber.util.JSONUtils.objectMapper import org.apache.texera.common.compiler.model.{LogicalLink, LogicalPlanPojo} @@ -203,6 +205,28 @@ class WorkflowToPythonResourceSpec assert(code.contains("head(5)")) } + private def exported(pojo: LogicalPlanPojo): WorkflowToPythonSuccess = + objectMapper + .readValue(postExport(pojo).readEntity(classOf[String]), classOf[WorkflowToPythonResponse]) + .asInstanceOf[WorkflowToPythonSuccess] + + it should "import the helpers from texera-helpers and hand that module over with the script" in { + val sample = new ReservoirSamplingOpDesc() + sample.setOperatorId("sample") + sample.k = 5 + val success = exported(LogicalPlanPojo(List(sample), List.empty, List.empty, List.empty)) + assert(success.pythonCode.contains("from texera_helpers import _TexeraJavaRandom")) + assert(!success.pythonCode.contains("class _TexeraJavaRandom")) + assert(success.helpersModule.contains(StandaloneHelpers.helpersModule)) + } + + it should "hand over no module with a script that imports nothing from it" in { + val limit = limitOp("limit", 5) + val success = exported(LogicalPlanPojo(List(limit), List.empty, List.empty, List.empty)) + assert(!success.pythonCode.contains("texera_helpers")) + assert(success.helpersModule.isEmpty) + } + it should "return a failure body rather than HTTP 500 when the plan cannot be read" in { // A link naming an operator the plan does not carry: the DAG refuses the // edge, and the resource has to answer with a reason rather than a stack