From 67f109ed773122dceb11269a8241d50b2782ab52 Mon Sep 17 00:00:00 2001 From: carloea2 Date: Wed, 7 Oct 2026 17:38:51 -0600 Subject: [PATCH 01/11] feat(frontend): add independently versioned workflow macros Create reusable nested macros from selected operators, pin their definition versions, expose ports and properties, and expand them before compilation, execution, and paused reconfiguration. Bundle dependencies for portable JSON exports and keep their graphs intact when cloning workflows. Closes #8913 Discussion: https://github.com/apache/texera/discussions/5248#discussioncomment-18788116 Validation: - node node_modules/@angular/cli/bin/ng.js test --watch=false --progress=false (6608 passed, 1 skipped) - sbt "WorkflowExecutionService/testOnly org.apache.texera.web.resource.dashboard.user.workflow.WorkflowMacroCloneSpec" (2 passed) - node .yarn/releases/yarn-4.14.1.cjs build - node .yarn/releases/yarn-4.14.1.cjs format:ci - sbt "scalafixAll --check" - sbt scalafmtCheckAll - Playwright browser checks with mocked API: selection collapse, exposed properties, nested inspection/back navigation, and independent editable copy. --- .../user/workflow/WorkflowResource.scala | 26 +- .../workflow/WorkflowMacroCloneSpec.scala | 56 +++ frontend/README.md | 12 + frontend/src/app/common/type/workflow.ts | 5 + .../app/common/util/workflow-macro.spec.ts | 268 +++++++++++ .../src/app/common/util/workflow-macro.ts | 454 ++++++++++++++++++ .../macro-inspector.component.spec.ts | 80 +++ .../macro-inspector.component.ts | 149 ++++++ .../macro-panel.component.spec.ts | 127 +++++ .../operator-menu/macro-panel.component.ts | 393 +++++++++++++++ .../operator-menu.component.html | 1 + .../operator-menu/operator-menu.component.ts | 2 + .../workflow-editor.component.ts | 9 +- .../workflow-form/workflow-form.component.ts | 1 + .../component/workspace.component.ts | 1 + .../workflow-compiling.service.ts | 78 ++- .../execute-workflow.service.spec.ts | 136 ++++++ .../execute-workflow.service.ts | 119 ++++- .../service/joint-ui/joint-ui.service.ts | 17 +- .../service/macro/macro.service.spec.ts | 169 +++++++ .../workspace/service/macro/macro.service.ts | 196 ++++++++ .../operator-menu.service.spec.ts | 50 ++ .../operator-menu/operator-menu.service.ts | 31 +- .../operator-metadata.service.ts | 27 +- .../stub-operator-metadata.service.ts | 31 +- .../validation/validation-workflow.service.ts | 4 +- .../workflow-graph/model/shared-model.ts | 7 +- .../model/workflow-action.service.spec.ts | 32 ++ .../model/workflow-action.service.ts | 103 +++- .../util/workflow-util.service.ts | 8 + .../workflow-result.service.spec.ts | 16 + .../workflow-result.service.ts | 19 +- .../workflow-status.service.ts | 8 + .../assets/operator_images/SubWorkflow.svg | 16 + 34 files changed, 2571 insertions(+), 80 deletions(-) create mode 100644 amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala create mode 100644 frontend/src/app/common/util/workflow-macro.spec.ts create mode 100644 frontend/src/app/common/util/workflow-macro.ts create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts create mode 100644 frontend/src/app/workspace/service/macro/macro.service.spec.ts create mode 100644 frontend/src/app/workspace/service/macro/macro.service.ts create mode 100644 frontend/src/assets/operator_images/SubWorkflow.svg diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala index 6fbd19bcc2c..d3901548aa4 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala @@ -20,6 +20,7 @@ package org.apache.texera.web.resource.dashboard.user.workflow import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.typesafe.scalalogging.LazyLogging import io.dropwizard.auth.Auth @@ -245,11 +246,19 @@ object WorkflowResource { */ def assignNewOperatorIds(workflowContent: String): String = { val objectMapper = new ObjectMapper().registerModule(DefaultScalaModule) - val operatorIdMap = objectMapper - .readValue(workflowContent, classOf[Map[String, List[Map[String, String]]]])("operators") + val canvas = objectMapper.readTree(workflowContent).asInstanceOf[ObjectNode] + // Pinned dependency graphs keep their definition IDs, even when a canvas ID is identical. + val dependencies = canvas.remove("macroDependencies") + val operators = Option(canvas.get("operators")) + .filter(_.isArray) + .getOrElse(throw new IllegalArgumentException("Missing operators array")) + val operatorIdMap = operators + .elements() + .asScala .map(operator => { - val oldOperatorId = operator("operatorID") - val operatorType = operator("operatorType") + require(operator.path("operatorID").isTextual && operator.path("operatorType").isTextual) + val oldOperatorId = operator.get("operatorID").asText() + val operatorType = operator.get("operatorType").asText() // operator id in frontend: operatorSchema.operatorType + "-operator-" + uuid(); // v4 = UUID.randomUUID().toString val newOperatorId = s"$operatorType-operator-${UUID.randomUUID()}" oldOperatorId -> newOperatorId @@ -257,10 +266,17 @@ object WorkflowResource { .toMap // replace all old operator ids with new operator ids - operatorIdMap.foldLeft(workflowContent) { + val rewritten = operatorIdMap.foldLeft(canvas.toString) { case (updatedContent, (oldId, newId)) => updatedContent.replace(oldId, newId) } + val result = objectMapper.readTree(rewritten).asInstanceOf[ObjectNode] + if (dependencies != null) + result.set[com.fasterxml.jackson.databind.JsonNode]("macroDependencies", dependencies) + Option(result.get("macroInterface")).filter(_.isObject).foreach { boundary => + boundary.asInstanceOf[ObjectNode].put("id", UUID.randomUUID().toString) + } + result.toString } def baseWorkflowSelect(): SelectOnConditionStep[Record10[ diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala new file mode 100644 index 00000000000..c77d1df9142 --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.resource.dashboard.user.workflow + +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.flatspec.AnyFlatSpec + +class WorkflowMacroCloneSpec extends AnyFlatSpec { + private val mapper = new ObjectMapper() + + "Cloning a workflow" should "rewrite canvas references and preserve pinned dependencies" in { + val content = mapper.readTree("""{ + "operators":[{"operatorID":"leaf","operatorType":"Filter"}], + "operatorPositions":{"leaf":{"x":0,"y":0}}, + "formBinding":{"fields":[{"operatorID":"leaf","propertyKey":"value"}]}, + "macroInterface":{"id":"definition","inputs":[{"targets":[{"operatorID":"leaf","portID":"input-0"}]}],"outputs":[]}, + "macroDependencies":[{"id":"child","version":8,"content":{"operators":[{"operatorID":"leaf","operatorType":"Filter"}]}}] + }""") + val cloned = mapper.readTree(WorkflowResource.assignNewOperatorIds(content.toString)) + val id = cloned.at("/operators/0/operatorID").asText() + assert(id != "leaf") + assert(cloned.path("operatorPositions").has(id)) + assert(cloned.at("/formBinding/fields/0/operatorID").asText() == id) + assert(cloned.at("/macroInterface/inputs/0/targets/0/operatorID").asText() == id) + assert(cloned.at("/macroInterface/id").asText() != "definition") + assert(cloned.path("macroDependencies") == content.path("macroDependencies")) + } + + it should "keep ordinary and empty workflows valid and reject malformed content" in { + assert( + mapper + .readTree(WorkflowResource.assignNewOperatorIds("""{"operators":[]}""")) + .path("operators") + .isEmpty + ) + intercept[Exception](WorkflowResource.assignNewOperatorIds("{}")) + intercept[Exception](WorkflowResource.assignNewOperatorIds("invalid")) + } +} diff --git a/frontend/README.md b/frontend/README.md index d1f97bbcd3d..3ed109b702e 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -53,6 +53,18 @@ Tests come first — write the failing test before the source change. The full testing reference (Vitest stack, recipes, anti-patterns, coverage troubleshooting) is in [`TESTING.md`](TESTING.md). +## Workflow macros + +Select operators, open **Operators → Macros**, enter a name, and choose **Create from selection**. +Drag saved macros from that panel into any workflow. Macros may contain other macros. +Each instance pins an immutable definition version; **Adopt latest compatible version** is explicit +and preserves connections and property values. Breaking interfaces or incompatible values require a new instance. +Double-click an instance to inspect its graph, navigate nested macros, or open its definition for editing. +**Save editable copy** creates an independent definition, including for imported macros. +In the definition's Macros panel, configure input/output mappings and exposed operator properties. +A macro definition runs through a workflow. Compilation recursively expands it into ordinary operators. +Workflow JSON exports include pinned definitions, so importing does not depend on the original deployment. + ## Project layout | Path | What lives here | diff --git a/frontend/src/app/common/type/workflow.ts b/frontend/src/app/common/type/workflow.ts index 701086e2a71..366b345e30b 100644 --- a/frontend/src/app/common/type/workflow.ts +++ b/frontend/src/app/common/type/workflow.ts @@ -19,6 +19,7 @@ import { DefaultView, WorkflowMetadata } from "../../dashboard/type/workflow-metadata.interface"; import { CommentBox, OperatorLink, OperatorPredicate, Point } from "../../workspace/types/workflow-common.interface"; +import type { MacroDefinition, MacroInterface } from "../util/workflow-macro"; export enum ExecutionMode { PIPELINED = "PIPELINED", @@ -101,6 +102,10 @@ export interface WorkflowContent /** Present once an author set up the Form View. Rides in the content (like `settings`), * so it is saved/cloned/versioned/published with the workflow for free. */ formBinding?: FormBindingConfig; + /** A reusable definition; this graph runs only as part of a workflow. */ + macroInterface?: MacroInterface; + /** Immutable, transitively bundled versions, keyed by definition id and version. */ + macroDependencies?: MacroDefinition[]; }> {} export type Workflow = { content: WorkflowContent } & WorkflowMetadata; diff --git a/frontend/src/app/common/util/workflow-macro.spec.ts b/frontend/src/app/common/util/workflow-macro.spec.ts new file mode 100644 index 00000000000..97f736e8923 --- /dev/null +++ b/frontend/src/app/common/util/workflow-macro.spec.ts @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ExecutionMode, WorkflowContent } from "../type/workflow"; +import { OperatorPredicate, OperatorLink } from "../../workspace/types/workflow-common.interface"; +import { + expandMacros, + extractMacro, + macroInstance, + macroType, + macroSchema, + adoptMacroVersion, + rollupMacroStatus, + macroResultAliases, +} from "./workflow-macro"; +import { OperatorSchema } from "../../workspace/types/operator-schema.interface"; +import { OperatorState } from "../../workspace/types/execute-workflow.interface"; + +const op = (operatorID: string): OperatorPredicate => ({ + operatorID, + operatorType: "Filter", + operatorVersion: "1", + operatorProperties: { value: "default" }, + inputPorts: [{ portID: "input-0" }], + outputPorts: [{ portID: "output-0" }], + showAdvanced: false, +}); +const link = (source: string, target: string): OperatorLink => ({ + linkID: `${source}-${target}`, + source: { operatorID: source, portID: "output-0" }, + target: { operatorID: target, portID: "input-0" }, +}); +const graph = (operators: OperatorPredicate[], links: OperatorLink[] = []): WorkflowContent => ({ + operators, + links, + operatorPositions: Object.fromEntries(operators.map(o => [o.operatorID, { x: 0, y: 0 }])), + commentBoxes: [], + settings: { dataTransferBatchSize: 10, executionMode: ExecutionMode.PIPELINED }, +}); +const definition = (id = "clean", version = 5) => ({ + id, + version, + name: "清理", + wid: 12, + content: { + ...graph([op("a"), op("b")], [link("a", "b")]), + macroInterface: { + id, + inputs: [{ portID: "input-0", displayName: "Rows", targets: [{ operatorID: "a", portID: "input-0" }] }], + outputs: [{ portID: "output-0", displayName: "Clean", targets: [{ operatorID: "b", portID: "output-0" }] }], + }, + formBinding: { fields: [{ id: "value", operatorID: "a", propertyKey: "value", displayName: "Value" }] }, + }, +}); + +describe("workflow macros", () => { + it("keeps ordinary and empty workflows unchanged", () => { + for (const content of [graph([]), graph([op("a"), op("b")], [link("a", "b")])]) { + expect(expandMacros(content).content.operators).toEqual(content.operators); + expect(expandMacros(content).content.links).toEqual(content.links); + } + }); + + it("expands repeated pinned instances with distinct, deterministic short IDs and local overrides", () => { + const d = definition(); + const first = { ...macroInstance(d, "first"), operatorProperties: { value: "override" } }; + const content = { + ...graph( + [op("source"), first, macroInstance(d, "second"), op("sink")], + [link("source", "first"), link("first", "second"), link("second", "sink")] + ), + macroDependencies: [d], + }; + const before = JSON.stringify(content); + const expanded = expandMacros(content); + expect(expanded.content.operators).toHaveLength(6); + expect(expanded.content.links).toHaveLength(5); + expect(new Set(expanded.content.operators.map(o => o.operatorID)).size).toBe(6); + expect(expanded.content.operators.every(o => !o.operatorType.startsWith("WorkflowMacro:"))).toBe(true); + const inner = expanded.content.operators.filter(o => expanded.paths[o.operatorID]?.[0] === "first"); + expect(inner.every(o => o.operatorID.length <= 40)).toBe(true); + expect(inner.find(o => o.operatorProperties.value === "override")).toBeDefined(); + expect(expandMacros(content)).toEqual(expanded); + expect(JSON.stringify(content)).toBe(before); + }); + + it("recurses through nested definitions and exposed properties and fans an input out", () => { + const child = definition(); + const nested = macroInstance(child, "nested"); + const parent = { + ...definition("outer", 8), + content: { + ...graph([nested]), + macroInterface: { + id: "outer", + inputs: [{ portID: "input-0", targets: [{ operatorID: "nested", portID: "input-0" }] }], + outputs: [{ portID: "output-0", targets: [{ operatorID: "nested", portID: "output-0" }] }], + }, + formBinding: { + fields: [{ id: "outerValue", operatorID: "nested", propertyKey: "value", displayName: "Value" }], + }, + }, + }; + const root = { ...macroInstance(parent, "root"), operatorProperties: { outerValue: "nested override" } }; + const expanded = expandMacros({ ...graph([root]), macroDependencies: [parent, child] }); + expect(expanded.content.operators.some(o => o.operatorProperties.value === "nested override")).toBe(true); + expect(Object.values(expanded.paths).every(path => path.join("/").startsWith("root/nested/"))).toBe(true); + const fan = definition(); + fan.content.macroInterface.inputs[0].targets.push({ operatorID: "b", portID: "input-0" }); + const fanout = expandMacros({ + ...graph([op("s"), macroInstance(fan, "m")], [link("s", "m")]), + macroDependencies: [fan], + }); + expect(fanout.content.links.filter(l => l.source.operatorID === "s")).toHaveLength(2); + }); + + it("rejects missing versions, dangling ports, output fan-in, duplicate IDs, recursion, and cycles", () => { + const d = definition(); + const root = graph([macroInstance(d, "m")]); + expect(() => expandMacros(root)).toThrow(/missing.*version/i); + const broken = structuredClone(d); + broken.content.macroInterface.outputs[0].targets[0].portID = "absent"; + expect(() => expandMacros({ ...root, macroDependencies: [broken] })).toThrow(/port/i); + const fanin = structuredClone(d); + fanin.content.macroInterface.outputs[0].targets.push({ operatorID: "a", portID: "output-0" }); + expect(() => expandMacros({ ...root, macroDependencies: [fanin] })).toThrow(/one.*output|output.*one/i); + expect(() => expandMacros(graph([op("a"), op("a")]))).toThrow(/duplicate/i); + const recursive = { ...d, content: { ...d.content, operators: [macroInstance(d, "a"), op("b")] } }; + expect(() => expandMacros({ ...root, macroDependencies: [recursive] })).toThrow(/recursive/i); + expect(() => expandMacros(graph([op("a"), op("b")], [link("a", "b"), link("b", "a")]))).toThrow(/cycle/i); + }); + + it("rejects malformed external definitions and properties rather than silently dropping them", () => { + const d = definition(); + const root = graph([macroInstance(d, "m")]); + for (const invalid of [ + null, + { ...d, content: null }, + { ...d, version: 0 }, + { ...d, content: { ...d.content, macroInterface: null } }, + ]) { + expect(() => expandMacros({ ...root, macroDependencies: [invalid] } as unknown as WorkflowContent)).toThrow(); + } + const instance = { ...macroInstance(d, "m"), operatorProperties: { unknown: 1 } }; + expect(() => expandMacros({ ...graph([instance]), macroDependencies: [d] })).toThrow(/property/i); + }); + + it("validates a definition's boundary even before it is instantiated", () => { + const d = definition(); + expect(() => expandMacros(d.content)).not.toThrow(); + d.content.macroInterface.inputs[0].targets[0].portID = "missing"; + expect(() => expandMacros(d.content)).toThrow(/port/i); + const invalid = definition(); + invalid.content.macroInterface.outputs[0].targets = []; + expect(() => expandMacros(invalid.content)).toThrow(/output/i); + }); + + it("carries referenced property schemas into exposed macro properties", () => { + const d = definition(); + const schema = { + jsonSchema: { + type: "object", + properties: { value: { $ref: "#/definitions/value" } }, + definitions: { value: { type: "string", minLength: 1 } }, + }, + } as unknown as OperatorSchema; + expect(macroSchema(d, () => schema).jsonSchema.properties?.["value"]).toEqual( + expect.objectContaining({ type: "string", minLength: 1 }) + ); + expect(() => macroSchema(d, () => ({ jsonSchema: { properties: {} } }) as OperatorSchema)).toThrow(/schema/i); + }); + + it("allows an exposed optional property to be filled even when its definition has no value yet", () => { + const d = definition(); + d.content.operators[0] = { ...d.content.operators[0], operatorProperties: {} }; + const instance = { ...macroInstance(d, "m"), operatorProperties: { value: "filled" } }; + expect( + expandMacros({ ...graph([instance]), macroDependencies: [d] }).content.operators[0].operatorProperties.value + ).toBe("filled"); + }); + + it("extracts a connected selection, preserves boundary links and Form View bindings", () => { + const content = { + ...graph([op("s"), op("a"), op("b"), op("t")], [link("s", "a"), link("a", "b"), link("b", "t")]), + formBinding: { fields: [{ id: "value", operatorID: "a", propertyKey: "value", displayName: "Value" }] }, + }; + const extracted = extractMacro(content, ["a", "b"], "clean"); + expect(extracted.operators.map(o => o.operatorID)).toEqual(["a", "b"]); + expect(extracted.links).toEqual([link("a", "b")]); + expect(extracted.macroInterface?.inputs[0].targets).toEqual([{ operatorID: "a", portID: "input-0" }]); + expect(extracted.macroInterface?.outputs[0].targets).toEqual([{ operatorID: "b", portID: "output-0" }]); + expect(extracted.formBinding).toEqual(content.formBinding); + expect(() => extractMacro(content, [], "empty")).toThrow(/select/i); + expect(() => extractMacro(content, ["missing"], "bad")).toThrow(/missing/i); + expect(() => extractMacro(content, ["s", "b"], "nonconvex")).toThrow(/selection/i); + }); + + it("adopts a new version explicitly, preserves values, and rejects breaking interfaces", () => { + const old = definition(); + const instance = { ...macroInstance(old, "m"), operatorProperties: { value: "mine" } }; + const next = definition("clean", 6); + const updated = adoptMacroVersion(instance, old, next); + expect(updated.operatorType).toBe(macroType(next)); + expect(updated.operatorProperties).toEqual({ value: "mine" }); + expect(instance.operatorType).toBe(macroType(old)); + expect(() => adoptMacroVersion(instance, old, definition("other", 6))).toThrow(/definition/i); + next.content.macroInterface.outputs = []; + expect(() => adoptMacroVersion(instance, old, next)).toThrow(/interface/i); + }); + + it("omits disabled instances and their links without resolving dependencies", () => { + const disabled = { ...macroInstance(definition(), "m"), isDisabled: true }; + const expanded = expandMacros(graph([op("s"), disabled], [link("s", "m")])); + expect(expanded.content.operators.map(o => o.operatorID)).toEqual(["s"]); + expect(expanded.content.links).toEqual([]); + }); + + it("reports boundary counts and aggregate state without counting internal edges twice", () => { + const d = definition(); + const content = { ...graph([macroInstance(d, "m")]), macroDependencies: [d] }; + const expanded = expandMacros(content); + const [a, b] = expanded.content.operators; + const status = { + [a.operatorID]: { + operatorState: OperatorState.Completed, + aggregatedInputRowCount: 100, + aggregatedOutputRowCount: 100, + inputPortMetrics: { "0": 100 }, + outputPortMetrics: { "0": 100 }, + numWorkers: 1, + }, + [b.operatorID]: { + operatorState: OperatorState.Running, + aggregatedInputRowCount: 100, + aggregatedOutputRowCount: 5, + inputPortMetrics: { "0": 100 }, + outputPortMetrics: { "0": 5 }, + numWorkers: 2, + }, + }; + const rolled = rollupMacroStatus(content, status); + expect(rolled.m.operatorState).toBe(OperatorState.Running); + expect(rolled.m.aggregatedInputRowCount).toBe(100); + expect(rolled.m.aggregatedOutputRowCount).toBe(5); + expect(rolled.m.numWorkers).toBe(3); + expect(rolled[a.operatorID]).toEqual(status[a.operatorID]); + expect(rollupMacroStatus(graph([]), {})).toEqual({}); + expect(rollupMacroStatus(content, {}).m).toBeUndefined(); + expect(macroResultAliases(content).m).toBe(b.operatorID); + expect(macroResultAliases(graph([]))).toEqual({}); + }); +}); diff --git a/frontend/src/app/common/util/workflow-macro.ts b/frontend/src/app/common/util/workflow-macro.ts new file mode 100644 index 00000000000..8b69e0e5208 --- /dev/null +++ b/frontend/src/app/common/util/workflow-macro.ts @@ -0,0 +1,454 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { v5 as uuid } from "uuid"; +import { cloneDeep } from "lodash-es"; +import { WorkflowContent, FormFieldBinding } from "../type/workflow"; +import { LogicalPort, OperatorPredicate, PortDescription } from "../../workspace/types/workflow-common.interface"; +import { OperatorSchema } from "../../workspace/types/operator-schema.interface"; +import { CustomJSONSchema7 } from "../../workspace/types/custom-json-schema.interface"; +import { OperatorRuntimeStatus, OperatorState } from "../../workspace/types/execute-workflow.interface"; + +export interface MacroPort extends PortDescription { + targets: LogicalPort[]; +} +export interface MacroInterface { + /** Stable across definition versions and deployments. */ + id: string; + inputs: MacroPort[]; + outputs: MacroPort[]; +} +export interface MacroDefinition { + id: string; + version: number; + name: string; + /** Source workflow, used only when this deployment's library has the same definition id. */ + wid?: number; + content: WorkflowContent; +} +export const isMacro = (op: OperatorPredicate): boolean => op.operatorType.startsWith("WorkflowMacro:"); +export const macroType = (d: MacroDefinition): string => `WorkflowMacro:${d.id}:${d.version}`; +export const portKey = (id: string, direction: "inputs" | "outputs", port: string): string => + JSON.stringify([id, direction, port]); +const require = (condition: unknown, message: string): void => { + if (!condition) throw new Error(message); +}; +const fields = (content?: WorkflowContent): FormFieldBinding[] => content?.formBinding?.fields ?? []; + +export function macroInstance(d: MacroDefinition, operatorID: string): OperatorPredicate { + require(d.content?.macroInterface, "Missing macro interface"); + return { + operatorID, + operatorType: macroType(d), + operatorVersion: String(d.version), + operatorProperties: {}, + inputPorts: d.content.macroInterface!.inputs.map(({ targets, ...p }) => p), + outputPorts: d.content.macroInterface!.outputs.map(({ targets, ...p }) => p), + showAdvanced: false, + customDisplayName: d.name, + }; +} + +export interface MacroExpansion { + content: WorkflowContent; + /** Runtime id -> instance chain followed by the definition's leaf id. */ + paths: Record; + /** Canvas boundary -> runtime ports, for result and schema lookup. */ + ports: Record; +} + +/** A pure, deterministic expansion used by both compilers and executions. Definitions stay immutable. */ +export function expandMacros(content: WorkflowContent): MacroExpansion { + const definitions = new Map(); + for (const d of content.macroDependencies ?? []) { + require(d && + typeof d.id === "string" && + d.id.length && + Number.isSafeInteger(d.version) && + d.version > 0 && + d.content?.macroInterface?.id === d.id, "Malformed macro definition"); + const previous = definitions.get(macroType(d)); + require(!previous || JSON.stringify(previous.content) === JSON.stringify(d.content), "Conflicting macro version"); + definitions.set(macroType(d), d); + } + const paths: Record = Object.create(null); + const ports: Record = Object.create(null); + const operators: OperatorPredicate[] = []; + const links: WorkflowContent["links"] = []; + const namespace = "33269d11-7448-5d6d-9aeb-68826068e94d"; + const runtimeID = (chain: string[], id: string): string => + chain.length ? uuid(JSON.stringify([...chain, id]), namespace) : id; + + const visit = (body: WorkflowContent, chain: string[], stack: string[]): Map => { + require(body && Array.isArray(body.operators) && Array.isArray(body.links), "Malformed macro graph"); + require(chain.length <= 32, "Macro nesting exceeds 32 levels"); + const ops = new Map(); + for (const op of body.operators) { + require(op && + typeof op.operatorID === "string" && + op.operatorID.length && + !ops.has(op.operatorID), "Duplicate or invalid operator ID"); + require(typeof op.operatorType === "string" && + Array.isArray(op.inputPorts) && + Array.isArray(op.outputPorts) && + op.operatorProperties && + typeof op.operatorProperties === "object" && + !Array.isArray(op.operatorProperties), "Malformed operator"); + for (const group of [op.inputPorts, op.outputPorts]) { + require(group.every(p => p && typeof p.portID === "string" && p.portID.length) && + new Set(group.map(p => p.portID)).size === group.length, "Duplicate or invalid operator port"); + } + ops.set(op.operatorID, op); + } + const resolved = new Map(); + const endpoint = (p: LogicalPort, direction: "inputs" | "outputs"): LogicalPort[] => { + const op = ops.get(p?.operatorID); + require(op && + (direction === "inputs" ? op.inputPorts : op.outputPorts).some( + port => port.portID === p.portID + ), "Missing operator or port in macro graph"); + if (op!.isDisabled) return []; + const result = resolved.get(portKey(p.operatorID, direction, p.portID)); + require(result, "Missing exposed macro port"); + return result!; + }; + for (const op of body.operators.filter(op => !op.isDisabled)) { + if (!isMacro(op)) { + const id = runtimeID(chain, op.operatorID); + require(!operators.some(o => o.operatorID === id), "Duplicate runtime operator ID"); + operators.push({ ...cloneDeep(op), operatorID: id }); + paths[id] = [...chain, op.operatorID]; + for (const direction of ["inputs", "outputs"] as const) { + for (const p of direction === "inputs" ? op.inputPorts : op.outputPorts) { + resolved.set(portKey(op.operatorID, direction, p.portID), [{ operatorID: id, portID: p.portID }]); + } + } + continue; + } + const d = definitions.get(op.operatorType); + require(d, `Missing macro version: ${op.operatorType}`); + require(op.operatorVersion === String(d!.version), "Macro version does not match its pinned definition"); + require(!stack.includes(op.operatorType), "Recursive macro reference"); + const inner = cloneDeep(d!.content); + const bindings = fields(inner); + for (const [key, value] of Object.entries(op.operatorProperties)) { + const binding = bindings.find(f => f.id === key); + require(binding, `Unknown macro property: ${key}`); + const target = inner.operators.find(o => o.operatorID === binding!.operatorID); + require(target && + (!isMacro(target) || + fields(definitions.get(target.operatorType)?.content).some( + f => f.id === binding!.propertyKey + )), "Missing exposed macro property"); + inner.operators[inner.operators.indexOf(target!)] = { + ...target!, + operatorProperties: { ...target!.operatorProperties, [binding!.propertyKey]: cloneDeep(value) }, + }; + } + const innerPorts = visit(inner, [...chain, op.operatorID], [...stack, op.operatorType]); + for (const direction of ["inputs", "outputs"] as const) { + const boundary = inner.macroInterface?.[direction]; + require(Array.isArray(boundary), "Malformed macro interface"); + const declared = direction === "inputs" ? op.inputPorts : op.outputPorts; + require(declared.length === boundary!.length && + declared.every(p => + boundary!.some(b => b.portID === p.portID) + ), "Macro instance interface differs from its pinned version"); + require(new Set(boundary!.map(p => p.portID)).size === boundary!.length, "Duplicate macro port"); + for (const p of boundary!) { + require(Array.isArray(p.targets) && + p.targets.length > 0 && + (direction === "inputs" || + p.targets.length === 1), "An output must map to one output port; inputs must have targets"); + const targets = p.targets.flatMap(t => { + const mapped = + innerPorts.get(portKey(t.operatorID, direction, t.portID)) ?? + (inner.operators.some(o => o.operatorID === t.operatorID && o.isDisabled) ? [] : undefined); + require(mapped, "Missing mapped macro port"); + return mapped!; + }); + resolved.set(portKey(op.operatorID, direction, p.portID), targets); + if (!chain.length) ports[portKey(op.operatorID, direction, p.portID)] = targets; + } + } + } + if (body.macroInterface) { + require(typeof body.macroInterface.id === "string" && + body.macroInterface.id.length, "Invalid macro interface ID"); + for (const direction of ["inputs", "outputs"] as const) { + const boundary = body.macroInterface[direction]; + require(Array.isArray(boundary) && + boundary.every(p => p && typeof p.portID === "string" && p.portID.length) && + new Set(boundary.map(p => p.portID)).size === boundary.length, "Duplicate or invalid macro port"); + boundary.forEach(p => { + require(Array.isArray(p.targets) && + p.targets.length && + (direction === "inputs" || + p.targets.length === 1), "An output must map to one output port; inputs must have targets"); + require(new Set(p.targets.map(t => JSON.stringify(t))).size === p.targets.length, "Duplicate macro target"); + p.targets.forEach(t => endpoint(t, direction)); + }); + } + require(new Set(fields(body).map(f => f.id)).size === fields(body).length, "Duplicate exposed macro property"); + fields(body).forEach(f => + require(f && + typeof f.id === "string" && + f.id.length && + typeof f.propertyKey === "string" && + ops.has(f.operatorID), "Invalid exposed macro property") + ); + } + require(new Set(body.links.map(l => l?.linkID)).size === body.links.length, "Duplicate link ID"); + for (const l of body.links) { + require(l && typeof l.linkID === "string" && l.linkID.length, "Invalid link ID"); + const sources = endpoint(l.source, "outputs"); + const targets = endpoint(l.target, "inputs"); + for (const source of sources) + for (const target of targets) { + links.push({ + linkID: + chain.length || sources.length * targets.length > 1 + ? uuid(JSON.stringify([chain, l.linkID, source, target]), namespace) + : l.linkID, + source, + target, + }); + } + } + return resolved; + }; + visit(content, [], []); + const children = new Map( + operators.map(op => [ + op.operatorID, + links.filter(l => l.source.operatorID === op.operatorID).map(l => l.target.operatorID), + ]) + ); + const visiting = new Set(), + visited = new Set(); + const checkCycle = (id: string): void => { + require(!visiting.has(id), "Workflow contains a cycle"); + if (visited.has(id)) return; + visiting.add(id); + children.get(id)!.forEach(checkCycle); + visiting.delete(id); + visited.add(id); + }; + operators.forEach(op => checkCycle(op.operatorID)); + return { + content: { ...content, operators, links, macroInterface: undefined, macroDependencies: undefined }, + paths, + ports, + }; +} + +/** Capture a convex selection; re-entering it through unselected nodes would create a macro cycle. */ +export function extractMacro(content: WorkflowContent, selected: readonly string[], id: string): WorkflowContent { + require(selected.length > 0, "Select at least one operator"); + const ids = new Set(selected); + require(selected.every(id => content.operators.some(o => o.operatorID === id)), "Missing selected operator"); + const escaped = new Set(); + const follow = (from: string): void => { + require(!ids.has(from), "Selection must not leave and re-enter the macro"); + if (escaped.has(from)) return; + escaped.add(from); + content.links.filter(l => l.source.operatorID === from).forEach(l => follow(l.target.operatorID)); + }; + content.links + .filter(l => ids.has(l.source.operatorID) && !ids.has(l.target.operatorID)) + .forEach(l => follow(l.target.operatorID)); + const operators = content.operators.filter(o => ids.has(o.operatorID)); + const links = content.links.filter(l => ids.has(l.source.operatorID) && ids.has(l.target.operatorID)); + const boundary = (direction: "inputs" | "outputs"): MacroPort[] => + operators + .flatMap(op => + (direction === "inputs" ? op.inputPorts : op.outputPorts) + .filter( + p => + !links.some(l => { + const end = direction === "inputs" ? l.target : l.source; + return end.operatorID === op.operatorID && end.portID === p.portID; + }) || + content.links.some(l => { + const end = direction === "inputs" ? l.target : l.source, + other = direction === "inputs" ? l.source : l.target; + return end.operatorID === op.operatorID && end.portID === p.portID && !ids.has(other.operatorID); + }) + ) + .map(p => ({ ...p, targets: [{ operatorID: op.operatorID, portID: p.portID }] })) + ) + .map((p, i) => ({ ...p, portID: `${direction === "inputs" ? "input" : "output"}-${i}` })); + return cloneDeep({ + ...content, + operators, + links, + operatorPositions: Object.fromEntries(operators.map(o => [o.operatorID, content.operatorPositions[o.operatorID]])), + commentBoxes: [], + macroInterface: { id, inputs: boundary("inputs"), outputs: boundary("outputs") }, + formBinding: { fields: fields(content).filter(f => ids.has(f.operatorID)) }, + }); +} + +export function adoptMacroVersion( + op: OperatorPredicate, + previous: MacroDefinition, + next: MacroDefinition +): OperatorPredicate { + require(previous.id === next.id && op.operatorType === macroType(previous), "Different macro definition"); + const signature = (d: MacroDefinition) => + JSON.stringify([ + d.content.macroInterface!.inputs.map(p => p.portID), + d.content.macroInterface!.outputs.map(p => p.portID), + fields(d.content).map(f => f.id), + ]); + require(signature(previous) === + signature(next), "Macro interface changed; insert the new version and reconnect explicitly"); + return { + ...macroInstance(next, op.operatorID), + operatorProperties: cloneDeep(op.operatorProperties), + customDisplayName: op.customDisplayName, + isDisabled: op.isDisabled, + viewResult: op.viewResult, + markedForReuse: op.markedForReuse, + }; +} + +/** Exposed Form View fields become the macro's ordinary operator properties, including nested macros. */ +export function macroSchema(d: MacroDefinition, getSchema: (type: string) => OperatorSchema): OperatorSchema { + const properties: Record = Object.create(null); + const resolve = (value: any, root: CustomJSONSchema7, refs: string[] = []): any => { + if (Array.isArray(value)) return value.map(v => resolve(v, root, refs)); + if (!value || typeof value !== "object") return value; + if (value.$ref) { + const { $ref, ...rest } = value; + require($ref.startsWith("#/") && !refs.includes($ref), "Missing or recursive exposed property schema reference"); + const target = $ref + .slice(2) + .split("/") + .reduce((node: any, key: string) => node?.[key.replace(/~1/g, "/").replace(/~0/g, "~")], root); + require(target, "Missing exposed property schema reference"); + return resolve({ ...target, ...rest }, root, [...refs, $ref]); + } + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => key !== "$id") + .map(([key, v]) => [key, resolve(v, root, refs)]) + ); + }; + for (const f of fields(d.content)) { + const op = d.content.operators.find(o => o.operatorID === f.operatorID); + require(op, "Missing exposed property operator"); + const root = getSchema(op!.operatorType).jsonSchema; + const schema = root.properties?.[f.propertyKey]; + require(schema && typeof schema !== "boolean", "Missing exposed property schema"); + properties[f.id] = { + ...resolve(schema, root), + title: f.displayName, + description: f.helpText, + default: cloneDeep( + Object.prototype.hasOwnProperty.call(op!.operatorProperties, f.propertyKey) + ? op!.operatorProperties[f.propertyKey] + : (schema as CustomJSONSchema7).default + ), + }; + } + return { + operatorType: macroType(d), + operatorVersion: String(d.version), + jsonSchema: { type: "object", properties, additionalProperties: false }, + additionalMetadata: { + userFriendlyName: `${d.name} (v${d.version})`, + operatorGroupName: "Macros", + operatorDescription: "Reusable macro. Double-click to inspect its pinned definition.", + inputPorts: d.content.macroInterface!.inputs.map(p => ({ displayName: p.displayName })), + outputPorts: d.content.macroInterface!.outputs.map(p => ({ displayName: p.displayName })), + }, + }; +} + +export function macroResultAliases(content: WorkflowContent, expanded = expandMacros(content)): Record { + const aliases: Record = Object.create(null); + for (const op of content.operators.filter(isMacro)) { + const output = op.outputPorts.flatMap(p => expanded.ports[portKey(op.operatorID, "outputs", p.portID)] ?? [])[0]; + const sink = expanded.content.operators.find( + o => expanded.paths[o.operatorID]?.[0] === op.operatorID && o.operatorType.toLowerCase().includes("sink") + ); + if (output || sink) aliases[op.operatorID] = output?.operatorID ?? sink!.operatorID; + } + return aliases; +} + +/** Count the external boundaries; summing every internal edge would multiply the row counts. */ +export function rollupMacroStatus( + content: WorkflowContent, + status: Record, + expanded = expandMacros(content) +): Record { + const result = { ...status }; + for (const op of content.operators.filter(isMacro)) { + const leaves = expanded.content.operators.filter(o => expanded.paths[o.operatorID]?.[0] === op.operatorID); + const updates = leaves.map(o => status[o.operatorID]).filter(Boolean); + if (!updates.length) continue; + const counts = (direction: "inputs" | "outputs"): Record => + Object.fromEntries( + (direction === "inputs" ? op.inputPorts : op.outputPorts).map((p, index) => [ + String(index), + Math.max( + 0, + ...(expanded.ports[portKey(op.operatorID, direction, p.portID)] ?? []).map(target => { + const leaf = leaves.find(o => o.operatorID === target.operatorID)!; + const ports = direction === "inputs" ? leaf.inputPorts : leaf.outputPorts; + const ordinal = ports.findIndex(p => p.portID === target.portID); + const update = status[target.operatorID]; + return ( + (direction === "inputs" ? update?.inputPortMetrics : update?.outputPortMetrics)?.[String(ordinal)] ?? + (ports.length === 1 + ? (direction === "inputs" ? update?.aggregatedInputRowCount : update?.aggregatedOutputRowCount) ?? 0 + : 0) + ); + }) + ), + ]) + ); + const inputs = counts("inputs"), + outputs = counts("outputs"); + const priority = [ + OperatorState.Recovering, + OperatorState.Pausing, + OperatorState.Paused, + OperatorState.Resuming, + OperatorState.Running, + OperatorState.Initializing, + OperatorState.Ready, + OperatorState.Uninitialized, + ]; + result[op.operatorID] = { + operatorState: + updates.length === leaves.length && updates.every(s => s.operatorState === OperatorState.Completed) + ? OperatorState.Completed + : priority.find(state => updates.some(s => s.operatorState === state)) ?? OperatorState.Uninitialized, + aggregatedInputRowCount: Object.values(inputs).reduce((a, b) => a + b, 0), + inputPortMetrics: inputs, + aggregatedOutputRowCount: Object.values(outputs).reduce((a, b) => a + b, 0), + outputPortMetrics: outputs, + numWorkers: updates.reduce((total, s) => total + (s.numWorkers ?? 0), 0), + }; + } + return result; +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts new file mode 100644 index 00000000000..fd39837c611 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TestBed } from "@angular/core/testing"; +import { NZ_MODAL_DATA } from "ng-zorro-antd/modal"; +import { MacroInspectorComponent } from "./macro-inspector.component"; +import { MacroService } from "../../../service/macro/macro.service"; +import { JointUIService } from "../../../service/joint-ui/joint-ui.service"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { ExecutionMode } from "../../../../common/type/workflow"; + +describe("MacroInspectorComponent", () => { + const definition = { + id: "macro", + version: 8, + wid: 12, + name: "Clean", + content: { + operators: [], + links: [], + operatorPositions: {}, + commentBoxes: [], + settings: { executionMode: ExecutionMode.PIPELINED, dataTransferBatchSize: 100 }, + macroInterface: { id: "macro", inputs: [], outputs: [] }, + }, + }; + let data: any, macros: any, notification: any; + beforeEach(() => { + data = { definition, dependencies: [], library: [definition] }; + macros = { copyDefinition: vi.fn(), library$: { value: [] } }; + notification = { error: vi.fn(), success: vi.fn() }; + TestBed.configureTestingModule({ + imports: [MacroInspectorComponent], + providers: [ + { provide: NZ_MODAL_DATA, useValue: data }, + { provide: MacroService, useValue: macros }, + { provide: JointUIService, useValue: {} }, + { provide: NotificationService, useValue: notification }, + ], + }); + }); + + it("links only to an unambiguous accessible definition and preserves its canvas on navigation", () => { + const fixture = TestBed.createComponent(MacroInspectorComponent); + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector("a").getAttribute("href")).toBe("/user/workflow/12"); + fixture.componentInstance.stack.push(definition); + fixture.componentInstance.back(); + expect(fixture.nativeElement.querySelector("svg")).toBeTruthy(); + data.library = []; + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector("a")).toBeNull(); + }); + + it("reports failed copies and resets the busy state without changing the pinned definition", async () => { + macros.copyDefinition.mockRejectedValue(new Error("save failed")); + const fixture = TestBed.createComponent(MacroInspectorComponent); + fixture.detectChanges(); + await fixture.componentInstance.copy(); + expect(notification.error).toHaveBeenCalledWith("save failed"); + expect(fixture.componentInstance.copying).toBe(false); + expect(fixture.componentInstance.definition).toBe(definition); + }); +}); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts new file mode 100644 index 00000000000..92906aa14fd --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { AfterViewInit, Component, ElementRef, Inject, OnDestroy, ViewChild } from "@angular/core"; +import { JsonPipe, NgFor, NgIf } from "@angular/common"; +import { NZ_MODAL_DATA } from "ng-zorro-antd/modal"; +import * as joint from "jointjs"; +import { isMacro, MacroDefinition, macroType } from "../../../../common/util/workflow-macro"; +import { JointUIService } from "../../../service/joint-ui/joint-ui.service"; +import { MacroService } from "../../../service/macro/macro.service"; +import { workspaceCanvasUrl } from "../../../../app-routing.constant"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; + +@Component({ + selector: "texera-macro-inspector", + imports: [NgIf, NgFor, JsonPipe], + template: ` + {{ definition.name }} · v{{ definition.version }} + Edit definition + +

+ Double-click a nested macro to inspect it. Editing a definition creates versions; existing instances stay pinned. +

+
+
+ {{ op.customDisplayName || op.operatorType }} +
{{ op.operatorProperties | json }}
+
`, + styles: [ + ` + strong, + a, + button { + margin-right: 12px; + } + pre { + max-height: 120px; + overflow: auto; + } + `, + ], +}) +export class MacroInspectorComponent implements AfterViewInit, OnDestroy { + @ViewChild("canvas", { static: true }) canvas!: ElementRef; + public definition: MacroDefinition; + public stack: MacroDefinition[] = []; + public copying = false; + private paper?: joint.dia.Paper; + public get editURL(): string | undefined { + const matches = this.data.library.filter(d => d.id === this.definition.id); + const source = matches.length === 1 ? matches[0] : undefined; + return source?.wid ? workspaceCanvasUrl(source.wid) : undefined; + } + constructor( + @Inject(NZ_MODAL_DATA) + private data: { definition: MacroDefinition; dependencies: MacroDefinition[]; library: MacroDefinition[] }, + private ui: JointUIService, + private macros: MacroService, + private notification: NotificationService + ) { + this.definition = data.definition; + } + ngAfterViewInit(): void { + this.render(); + } + ngOnDestroy(): void { + this.paper?.remove(); + } + public back(): void { + this.definition = this.stack.pop()!; + this.render(); + } + private render(): void { + this.paper?.remove(); + const canvas = document.createElement("div"); + this.canvas.nativeElement.replaceChildren(canvas); + const graph = new joint.dia.Graph(); + this.paper = new joint.dia.Paper({ el: canvas, model: graph, width: "100%", height: 380, interactive: false }); + const body = this.definition.content; + body.operators.forEach(op => { + const cell = this.ui.getJointOperatorElement(op, body.operatorPositions[op.operatorID] ?? { x: 0, y: 0 }); + cell.attr("rect.body/stroke", "#bfbfbf"); + graph.addCell(cell); + }); + body.links.forEach(l => graph.addCell(JointUIService.getJointLinkCell(l))); + if (body.operators.length) this.paper.scaleContentToFit({ padding: 30, maxScale: 1 }); + this.paper.on("element:pointerdblclick", (view: joint.dia.ElementView) => { + const op = body.operators.find(o => o.operatorID === view.model.id); + const nested = op && isMacro(op) ? this.data.dependencies.find(d => macroType(d) === op.operatorType) : undefined; + if (nested) { + this.stack.push(this.definition); + this.definition = nested; + this.render(); + } + }); + this.paper.on("blank:mousewheel", (_event: unknown, x: number, y: number, delta: number) => { + const scale = Math.max(0.15, Math.min(2, this.paper!.scale().sx + delta * 0.1)); + this.paper!.scale(scale, scale, x, y); + }); + } + public async copy(): Promise { + if (this.copying) return; + this.copying = true; + try { + this.definition = await this.macros.copyDefinition(this.definition, this.data.dependencies); + this.data.library = this.macros.library$.value; + this.stack = []; + this.render(); + this.notification.success("Macro copy saved. Open Edit definition to edit it."); + } catch (e) { + this.notification.error(e instanceof Error ? e.message : String(e)); + } finally { + this.copying = false; + } + } +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts new file mode 100644 index 00000000000..8058d3b984b --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { TestBed } from "@angular/core/testing"; +import { BehaviorSubject, of } from "rxjs"; +import { MacroPanelComponent } from "./macro-panel.component"; +import { MacroService } from "../../../service/macro/macro.service"; +import { WorkflowActionService } from "../../../service/workflow-graph/model/workflow-action.service"; +import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { NzModalService } from "ng-zorro-antd/modal"; + +describe("MacroPanelComponent", () => { + const boundary = { id: "macro", inputs: [], outputs: [] }; + let actions: any, service: any, notification: any; + beforeEach(() => { + actions = { + getMacroInterface: () => boundary, + checkWorkflowModificationEnabled: () => true, + getFormBinding: () => ({ fields: [] }), + setFormBinding: vi.fn(), + getTexeraGraph: () => ({ + getOperator: () => ({ operatorType: "Filter" }), + getAllOperators: () => [ + { + operatorID: "leaf", + customDisplayName: "Filter", + inputPorts: [{ portID: "input-0" }], + outputPorts: [{ portID: "output-0" }], + }, + ], + }), + getJointGraphWrapper: () => ({ getCurrentHighlightedOperatorIDs: () => ["leaf"] }), + setMacroInterface: vi.fn(), + }; + service = { + library$: new BehaviorSubject([]), + inspect$: new BehaviorSubject(undefined), + refresh: vi.fn(async () => []), + createFromSelection: vi.fn(async () => ({})), + adoptLatest: vi.fn(async () => {}), + }; + notification = { error: vi.fn() }; + TestBed.configureTestingModule({ + imports: [MacroPanelComponent], + providers: [ + { provide: WorkflowActionService, useValue: actions }, + { provide: MacroService, useValue: service }, + { + provide: OperatorMetadataService, + useValue: { + getOperatorSchema: vi.fn(() => ({ + jsonSchema: { properties: { value: { type: "string", title: "Value" } } }, + })), + }, + }, + { provide: NotificationService, useValue: notification }, + { provide: NzModalService, useValue: { create: vi.fn() } }, + ], + }); + }); + it("shows the definition controls and maps chosen leaf ports explicitly", () => { + const fixture = TestBed.createComponent(MacroPanelComponent); + fixture.detectChanges(); + const select = fixture.nativeElement.querySelector('select[aria-label="Input target"]') as HTMLSelectElement; + select.value = select.options[1].value; + select.dispatchEvent(new Event("change")); + fixture.detectChanges(); + (fixture.nativeElement.querySelector('[data-action="add-input"]') as HTMLButtonElement).click(); + expect(actions.setMacroInterface).toHaveBeenCalledWith( + expect.objectContaining({ + inputs: [ + expect.objectContaining({ + targets: [{ operatorID: "leaf", portID: "input-0" }], + }), + ], + }) + ); + }); + it("disables editing controls for a read-only macro and reports save failures", async () => { + actions.checkWorkflowModificationEnabled = () => false; + const fixture = TestBed.createComponent(MacroPanelComponent); + fixture.detectChanges(); + expect(fixture.nativeElement.querySelector('[data-action="create"]').disabled).toBe(true); + expect(fixture.nativeElement.querySelector('[data-action="add-input"]').disabled).toBe(true); + actions.checkWorkflowModificationEnabled = () => true; + service.createFromSelection.mockRejectedValue(new Error("save failed")); + fixture.componentInstance.name = "Clean"; + await fixture.componentInstance.create(); + expect(notification.error).toHaveBeenCalledWith("save failed"); + expect(fixture.componentInstance.busy).toBe(false); + }); + + it("exposes a chosen leaf property without requiring Form View to be enabled", () => { + const fixture = TestBed.createComponent(MacroPanelComponent); + fixture.detectChanges(); + const select = fixture.nativeElement.querySelector('select[aria-label="Property target"]') as HTMLSelectElement; + select.value = select.options[1].value; + select.dispatchEvent(new Event("change")); + fixture.detectChanges(); + fixture.nativeElement.querySelector('[data-action="add-property"]').click(); + expect(actions.setFormBinding).toHaveBeenCalledWith({ + fields: [ + expect.objectContaining({ + operatorID: "leaf", + propertyKey: "value", + displayName: "Value", + }), + ], + }); + }); +}); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts new file mode 100644 index 00000000000..bf34c9ef451 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Component } from "@angular/core"; +import { NgFor, NgIf } from "@angular/common"; +import { FormsModule } from "@angular/forms"; +import { NzModalModule, NzModalService } from "ng-zorro-antd/modal"; +import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; +import { v4 as uuid } from "uuid"; +import { MacroService } from "../../../service/macro/macro.service"; +import { WorkflowActionService } from "../../../service/workflow-graph/model/workflow-action.service"; +import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; +import { OperatorLabelComponent } from "./operator-label/operator-label.component"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { isMacro, MacroPort, macroType } from "../../../../common/util/workflow-macro"; +import { MacroInspectorComponent } from "./macro-inspector.component"; + +@UntilDestroy() +@Component({ + selector: "texera-macro-panel", + imports: [NgIf, NgFor, FormsModule, NzModalModule, OperatorLabelComponent], + template: `
+ Macros + +
+ +
+ + + + + + +
+
+ Macro interface +

Add this definition to a workflow to run it.

+
+ {{ direction === "inputs" ? "Inputs" : "Outputs" }} +
+ + {{ targetLabel(direction, target) }} + + +
+
+ + + + + Properties +
+ + +
+ + +
`, + styles: [ + ` + :host { + display: block; + padding: 10px; + } + summary { + cursor: pointer; + font-weight: 600; + } + input, + select { + width: 100%; + margin: 4px 0; + padding: 4px 7px; + } + input, + select, + button { + max-width: 100%; + font: inherit; + border: 1px solid #d9d9d9; + border-radius: 3px; + background: white; + } + button { + margin: 4px 4px 4px 0; + padding: 3px 7px; + cursor: pointer; + color: #1677ff; + white-space: normal; + } + button:disabled, + input:disabled, + select:disabled { + color: #999; + background: #fafafa; + cursor: default; + } + section { + margin-top: 12px; + } + .properties-heading { + display: block; + margin-top: 12px; + } + section span { + display: block; + font-size: 12px; + color: #666; + overflow-wrap: anywhere; + } + `, + ], +}) +export class MacroPanelComponent { + public name = ""; + public busy = false; + public inputTarget = ""; + public outputTarget = ""; + public propertyTarget = ""; + public readonly directions = ["inputs", "outputs"] as const; + public readonly type = macroType; + public get canModify(): boolean { + return this.actions.checkWorkflowModificationEnabled(); + } + public get selected(): readonly string[] { + return this.actions.getJointGraphWrapper().getCurrentHighlightedOperatorIDs(); + } + public get selectedMacro(): boolean { + const op = this.selected.length === 1 ? this.actions.getTexeraGraph().getOperator(this.selected[0]) : undefined; + return !!op && isMacro(op); + } + constructor( + public macros: MacroService, + public actions: WorkflowActionService, + public metadata: OperatorMetadataService, + private notification: NotificationService, + private modal: NzModalService + ) { + this.macros.inspect$.pipe(untilDestroyed(this)).subscribe(id => { + if (id) this.inspect(id); + }); + } + public opened(event: Event): void { + if ((event.target as HTMLDetailsElement).open) void this.refresh(); + } + public async refresh(): Promise { + await this.perform(() => this.macros.refresh()); + } + public async create(): Promise { + await this.perform(() => this.macros.createFromSelection(this.name)); + } + public async update(): Promise { + await this.perform(() => this.macros.adoptLatest(this.selected[0])); + } + private async perform(action: () => Promise): Promise { + if (this.busy) return; + this.busy = true; + try { + await action(); + } catch (e) { + this.notification.error(e instanceof Error ? e.message : String(e)); + } finally { + this.busy = false; + } + } + public async inspect(id: string): Promise { + const content = this.actions.getWorkflowContent(); + const op = content.operators.find(o => o.operatorID === id); + const definition = content.macroDependencies?.find(d => macroType(d) === op?.operatorType); + if (!definition) return; + try { + await this.macros.refresh(); + } catch (e) { + this.notification.error(e instanceof Error ? e.message : String(e)); + } + this.modal.create({ + nzTitle: "Macro definition", + nzContent: MacroInspectorComponent, + nzWidth: 900, + nzFooter: null, + nzData: { definition, dependencies: content.macroDependencies, library: this.macros.library$.value }, + }); + this.macros.inspect$.next(undefined); + } + public targets(direction: "inputs" | "outputs"): { value: string; label: string }[] { + return this.actions + .getTexeraGraph() + .getAllOperators() + .flatMap(op => + (direction === "inputs" ? op.inputPorts : op.outputPorts).map(p => ({ + value: JSON.stringify({ operatorID: op.operatorID, portID: p.portID }), + label: `${op.customDisplayName ?? this.metadata.getOperatorSchema(op.operatorType).additionalMetadata?.userFriendlyName ?? op.operatorType} / ${p.displayName ?? p.portID}`, + })) + ); + } + public targetLabel(direction: "inputs" | "outputs", target: { operatorID: string; portID: string }): string { + return this.targets(direction).find(p => p.value === JSON.stringify(target))?.label ?? "Missing target"; + } + public properties(): { value: string; label: string; title: string }[] { + return this.actions + .getTexeraGraph() + .getAllOperators() + .flatMap(op => + Object.entries(this.metadata.getOperatorSchema(op.operatorType).jsonSchema.properties ?? {}).flatMap( + ([key, schema]) => + typeof schema === "boolean" + ? [] + : [ + { + value: JSON.stringify({ operatorID: op.operatorID, propertyKey: key }), + label: `${op.customDisplayName ?? op.operatorType} / ${schema.title ?? key}`, + title: schema.title ?? key, + }, + ] + ) + ); + } + public exposeProperty(): void { + const selected = this.properties().find(p => p.value === this.propertyTarget); + if (!this.canModify || !selected) return; + const target = JSON.parse(selected.value); + const binding = this.actions.getFormBinding(); + if (binding.fields.some(f => f.operatorID === target.operatorID && f.propertyKey === target.propertyKey)) return; + this.actions.setFormBinding({ + ...binding, + fields: [...binding.fields, { id: uuid(), ...target, displayName: selected.title }], + }); + } + public renameProperty(id: string, displayName: string): void { + if (!this.canModify) return; + const binding = this.actions.getFormBinding(); + this.actions.setFormBinding({ + ...binding, + fields: binding.fields.map(f => (f.id === id ? { ...f, displayName } : f)), + }); + } + public removeProperty(id: string): void { + if (!this.canModify) return; + const binding = this.actions.getFormBinding(); + this.actions.setFormBinding({ ...binding, fields: binding.fields.filter(f => f.id !== id) }); + } + public add(direction: "inputs" | "outputs", target: string): void { + if (!this.canModify || !this.targets(direction).some(p => p.value === target)) return; + const boundary = this.actions.getMacroInterface()!; + this.actions.setMacroInterface({ + ...boundary, + [direction]: [ + ...boundary[direction], + { + portID: `${direction === "inputs" ? "input" : "output"}-${uuid()}`, + displayName: direction === "inputs" ? "Input" : "Output", + targets: [JSON.parse(target)], + }, + ], + }); + } + public remove(direction: "inputs" | "outputs", port: MacroPort): void { + if (!this.canModify) return; + const boundary = this.actions.getMacroInterface()!; + this.actions.setMacroInterface({ + ...boundary, + [direction]: boundary[direction].filter(p => p.portID !== port.portID), + }); + } + public rename(direction: "inputs" | "outputs", port: MacroPort, displayName: string): void { + if (!this.canModify) return; + const boundary = this.actions.getMacroInterface()!; + this.actions.setMacroInterface({ + ...boundary, + [direction]: boundary[direction].map(p => (p.portID === port.portID ? { ...p, displayName } : p)), + }); + } + public addTarget(port: MacroPort): void { + if (!this.canModify || !this.targets("inputs").some(p => p.value === this.inputTarget)) return; + const target = JSON.parse(this.inputTarget); + const boundary = this.actions.getMacroInterface()!; + if (port.targets.some(p => p.operatorID === target.operatorID && p.portID === target.portID)) return; + this.actions.setMacroInterface({ + ...boundary, + inputs: boundary.inputs.map(p => (p.portID === port.portID ? { ...p, targets: [...p.targets, target] } : p)), + }); + } +} diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html index ff8bb97a296..9a85406e776 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.html @@ -39,6 +39,7 @@
+ diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts index ed172ae5d13..bca9fc9d5a6 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts @@ -36,6 +36,7 @@ import { FormsModule } from "@angular/forms"; import { NgFor, NgTemplateOutlet } from "@angular/common"; import { OperatorLabelComponent } from "./operator-label/operator-label.component"; import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/collapse"; +import { MacroPanelComponent } from "./macro-panel.component"; @UntilDestroy() @Component({ @@ -43,6 +44,7 @@ import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/col templateUrl: "operator-menu.component.html", styleUrls: ["operator-menu.component.scss"], imports: [ + MacroPanelComponent, NzSpaceCompactItemDirective, NzInputDirective, FormsModule, diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts index 1fac6beeba3..1fefcbe7f3d 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts @@ -56,6 +56,8 @@ import { NgIf } from "@angular/common"; import { AgentInteractionComponent } from "../agent/agent-interaction/agent-interaction.component"; import { HeatmapLegendComponent } from "../heatmap-legend/heatmap-legend.component"; import { JupyterPanelService } from "../../service/jupyter-panel/jupyter-panel.service"; +import { MacroService } from "../../service/macro/macro.service"; +import { isMacro } from "../../../common/util/workflow-macro"; // jointjs interactive options for enabling and disabling interactivity // https://resources.jointjs.com/docs/jointjs/v3.2/joint.html#dia.Paper.prototype.options.interactive @@ -173,7 +175,8 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy private elementRef: ElementRef, private config: GuiConfigService, private agentService: AgentService, - private jupyterPanelService: JupyterPanelService + private jupyterPanelService: JupyterPanelService, + private macroService: MacroService ) { this.wrapper = this.workflowActionService.getJointGraphWrapper(); } @@ -908,7 +911,9 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy if (this.workflowActionService.getTexeraGraph().hasCommentBox(elementID)) { this.openCommentBox(elementID); } else if (this.workflowActionService.getTexeraGraph().hasOperator(elementID)) { - this.workflowActionService.openResultPanel(); + if (isMacro(this.workflowActionService.getTexeraGraph().getOperator(elementID))) { + this.macroService.inspect$.next(elementID); + } else this.workflowActionService.openResultPanel(); } } }); diff --git a/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts b/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts index 610488e05aa..9f9bd15f456 100644 --- a/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts +++ b/frontend/src/app/workspace/component/workflow-form/workflow-form.component.ts @@ -327,6 +327,7 @@ export class WorkflowFormComponent implements OnInit, OnDestroy { ) {} ngOnInit(): void { + this.workflowResultService.setMacroAliases?.(() => this.workflowActionService.getMacroResultAliases()); const wid = Number(this.route.snapshot.params.id); if (!Number.isFinite(wid)) { void this.router.navigate([USER_WORKFLOW]); diff --git a/frontend/src/app/workspace/component/workspace.component.ts b/frontend/src/app/workspace/component/workspace.component.ts index 3e43c07acd9..2af17fe6147 100644 --- a/frontend/src/app/workspace/component/workspace.component.ts +++ b/frontend/src/app/workspace/component/workspace.component.ts @@ -144,6 +144,7 @@ export class WorkspaceComponent implements AfterViewInit, OnInit, OnDestroy { ) {} ngOnInit() { + this.workflowResultService.setMacroAliases?.(() => this.workflowActionService.getMacroResultAliases()); this.workflowActionService.setHighlightingEnabled(true); // Clear session state when the user switches computing units in-canvas, so // the previous unit's status/console/results don't linger. diff --git a/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts b/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts index dae47a0088b..2c9e88607fc 100644 --- a/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts +++ b/frontend/src/app/workspace/service/compile-workflow/workflow-compiling.service.ts @@ -20,13 +20,13 @@ import { HttpClient, HttpHeaders } from "@angular/common/http"; import { Injectable } from "@angular/core"; import { JSONSchema7Definition } from "json-schema"; -import { EMPTY, merge, Observable, ReplaySubject } from "rxjs"; +import { defer, EMPTY, merge, Observable, of, ReplaySubject } from "rxjs"; import { CustomJSONSchema7 } from "src/app/workspace/types/custom-json-schema.interface"; import { AppSettings } from "../../../common/app-setting"; import { areOperatorSchemasEqual, OperatorSchema } from "../../types/operator-schema.interface"; import { ExecuteWorkflowService } from "../execute-workflow/execute-workflow.service"; import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; -import { catchError, debounceTime, mergeMap } from "rxjs/operators"; +import { catchError, debounceTime, map, mergeMap } from "rxjs/operators"; import { DynamicSchemaService } from "../dynamic-schema/dynamic-schema.service"; import { AttributeType, @@ -43,6 +43,8 @@ import { WorkflowGraphReadonly } from "../workflow-graph/model/workflow-graph"; import { serializePortIdentity } from "../../../common/util/port-identity-serde"; import { addCompilationError, areAllPortSchemasEqual } from "../../../common/util/workflow-compilation-utils"; import { parseLogicalOperatorPortID } from "../../../common/util/logical-operator-port-serde"; +import { expandMacros, isMacro, portKey } from "../../../common/util/workflow-macro"; +import { WorkflowContent } from "../../../common/type/workflow"; // endpoint for workflow compile export const WORKFLOW_COMPILATION_ENDPOINT = "compile"; @@ -90,17 +92,43 @@ export class WorkflowCompilingService { this.workflowActionService.getTexeraGraph().getOperatorAddStream(), this.workflowActionService.getTexeraGraph().getOperatorDeleteStream(), this.workflowActionService.getTexeraGraph().getOperatorPropertyChangeStream(), - this.workflowActionService.getTexeraGraph().getDisabledOperatorsChangedStream() + this.workflowActionService.getTexeraGraph().getDisabledOperatorsChangedStream(), + this.workflowActionService.macroChanged$ ?? EMPTY ) .pipe(debounceTime(WORKFLOW_COMPILATION_DEBOUNCE_TIME_MS)) .pipe( - mergeMap(() => { - const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest( - this.validationWorkflowService.getValidTexeraGraph(), - undefined - ); - return this.compile(logicalPlan); - }) + mergeMap(() => + defer(() => { + const content = this.workflowActionService.getWorkflowContent(); + const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest( + this.validationWorkflowService.getValidTexeraGraph(), + undefined, + this.workflowActionService.getMacroDependencies?.() ?? [] + ); + return this.compile(logicalPlan).pipe(map(response => this.mapMacroSchemas(response, content))); + }).pipe( + catchError((error: unknown) => { + const graph = this.workflowActionService.getTexeraGraph(); + const operatorID = + graph.getAllOperators().find(isMacro)?.operatorID ?? + graph.getAllOperators()[0]?.operatorID ?? + "workflow"; + return of({ + operatorOutputSchemas: {}, + operatorErrors: { + [operatorID]: { + message: error instanceof Error ? error.message : String(error), + details: "", + operatorId: operatorID, + workerId: "", + type: { name: "MacroValidation" }, + timestamp: { nanos: 0, seconds: 0 }, + }, + }, + } as WorkflowCompilationResponse); + }) + ) + ) ) .subscribe(response => { if (response.physicalPlan) { @@ -124,6 +152,36 @@ export class WorkflowCompilingService { return this.currentCompilationStateInfo.state; } + private mapMacroSchemas( + response: WorkflowCompilationResponse, + content: WorkflowContent + ): WorkflowCompilationResponse { + if (!content.operators.some(isMacro)) return response; + const expanded = expandMacros(content); + const schemas = { ...response.operatorOutputSchemas }; + const errors = { ...response.operatorErrors }; + for (const op of content.operators.filter(isMacro)) { + const outputs: Record = {}; + op.outputPorts.forEach((port, index) => { + const target = expanded.ports[portKey(op.operatorID, "outputs", port.portID)]?.[0]; + const leaf = expanded.content.operators.find(o => o.operatorID === target?.operatorID); + const ordinal = leaf?.outputPorts.findIndex(p => p.portID === target?.portID); + const schema = + target && ordinal !== undefined + ? schemas[target.operatorID]?.[serializePortIdentity({ id: ordinal, internal: false })] + : undefined; + if (schema) outputs[serializePortIdentity({ id: index, internal: false })] = schema; + }); + schemas[op.operatorID] = outputs; + } + for (const [id, error] of Object.entries(response.operatorErrors ?? {})) { + const path = expanded.paths[id]; + if (path?.length > 1) + errors[path[0]] = { ...error, operatorId: path[0], message: `${path.slice(1).join(" / ")}: ${error.message}` }; + } + return { ...response, operatorOutputSchemas: schemas, operatorErrors: errors }; + } + public getWorkflowCompilationErrors(): Readonly> { if ( this.currentCompilationStateInfo.state === CompilationState.Succeeded || diff --git a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts index 650a2d32a61..b55ba59b36e 100644 --- a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts +++ b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts @@ -57,6 +57,9 @@ import { TexeraWebsocketEvent, } from "../../types/workflow-websocket.interface"; import { mockScanPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { mockSentimentPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { WorkflowGraph } from "../workflow-graph/model/workflow-graph"; +import { macroInstance } from "../../../common/util/workflow-macro"; import { PAGINATION_INFO_STORAGE_KEY, ResultPaginationInfo } from "../../types/result-table.interface"; import { sessionGetObject, sessionSetObject } from "../../../common/util/storage"; @@ -111,6 +114,139 @@ describe("ExecuteWorkflowService", () => { expect(injectedService).toBeTruthy(); })); + it("sends a flat plan with leaf port ordinals and result IDs for a pinned macro", () => { + const actions = TestBed.inject(WorkflowActionService); + const d = { + id: "analysis", + version: 8, + name: "Analysis", + content: { + ...actions.getWorkflowContent(), + operators: [mockSentimentPredicate], + macroInterface: { + id: "analysis", + inputs: [ + { portID: "input-0", targets: [{ operatorID: mockSentimentPredicate.operatorID, portID: "input-0" }] }, + ], + outputs: [ + { portID: "output-0", targets: [{ operatorID: mockSentimentPredicate.operatorID, portID: "output-0" }] }, + ], + }, + }, + }; + const macro = { ...macroInstance(d, "m"), viewResult: true, markedForReuse: true }; + const graph = new WorkflowGraph( + [mockScanPredicate, macro], + [ + { + linkID: "input", + source: { + operatorID: mockScanPredicate.operatorID, + portID: "output-0", + }, + target: { operatorID: "m", portID: "input-0" }, + }, + ] + ); + try { + const plan = ExecuteWorkflowService.getLogicalPlanRequest(graph, "m", [d]); + expect(plan.operators.map(o => o.operatorType).sort()).toEqual(["NlpSentiment", "ScanSource"]); + const leaf = plan.operators.find(o => o.operatorType === "NlpSentiment")!; + expect(plan.links[0].toOpId).toBe(leaf.operatorID); + expect(plan.links[0].toPortId).toEqual({ id: 0, internal: false }); + expect(plan.opsToViewResult).toEqual([leaf.operatorID]); + expect(plan.opsToReuseResult).toEqual([leaf.operatorID]); + expect(() => ExecuteWorkflowService.getLogicalPlanRequest(graph)).toThrow(/macro version/i); + } finally { + graph.sharedModel.destroy(); + } + }); + + it("rejects a macro cycle even when the canvas has no terminal operator", () => { + const actions = TestBed.inject(WorkflowActionService); + const d = { + id: "cycle", + version: 1, + name: "Cycle", + content: { + ...actions.getWorkflowContent(), + operators: [mockSentimentPredicate], + macroInterface: { + id: "cycle", + inputs: [ + { portID: "input-0", targets: [{ operatorID: mockSentimentPredicate.operatorID, portID: "input-0" }] }, + ], + outputs: [ + { portID: "output-0", targets: [{ operatorID: mockSentimentPredicate.operatorID, portID: "output-0" }] }, + ], + }, + }, + }; + const graph = new WorkflowGraph( + [macroInstance(d, "m")], + [ + { + linkID: "cycle", + source: { operatorID: "m", portID: "output-0" }, + target: { operatorID: "m", portID: "input-0" }, + }, + ] + ); + try { + expect(() => ExecuteWorkflowService.getLogicalPlanRequest(graph, undefined, [d])).toThrow(/cycle/i); + } finally { + graph.sharedModel.destroy(); + } + }); + + it("reconfigures a paused macro using flat leaf operators and rejects missing versions before sending", () => { + const actions = TestBed.inject(WorkflowActionService); + const definition = { + id: "live", + version: 1, + name: "Live", + content: { + ...actions.getWorkflowContent(), + operators: [mockSentimentPredicate], + macroInterface: { + id: "live", + inputs: [ + { portID: "input-0", targets: [{ operatorID: mockSentimentPredicate.operatorID, portID: "input-0" }] }, + ], + outputs: [ + { portID: "output-0", targets: [{ operatorID: mockSentimentPredicate.operatorID, portID: "output-0" }] }, + ], + }, + formBinding: { + fields: [ + { + id: "attribute", + operatorID: mockSentimentPredicate.operatorID, + propertyKey: "attribute", + displayName: "Attribute", + }, + ], + }, + }, + }; + actions.setMacroDependencies([definition]); + actions.addOperator( + { ...macroInstance(definition, "m"), operatorProperties: { attribute: "new column" } }, + { x: 0, y: 0 } + ); + (service as any).currentState = { state: ExecutionState.Paused }; + const send = vi.spyOn(TestBed.inject(WorkflowWebsocketService), "send"); + service.modifyOperatorLogic("m"); + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith("ModifyLogicRequest", { + operator: expect.objectContaining({ operatorType: "NlpSentiment", attribute: "new column" }), + }); + send.mockClear(); + actions.setMacroDependencies([]); + expect(() => service.modifyOperatorLogic("m")).toThrow(/macro version/i); + expect(send).not.toHaveBeenCalled(); + }); + // A view handed a session mid-run subscribes after the last state change, and the stream carries // no current value, so it heard nothing about the run: it showed Run for a workflow that was // running. The lock is reapplied only when the state changes too, so a canvas that unlocked the diff --git a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts index c513ebe7c60..003f38e0778 100644 --- a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts +++ b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts @@ -51,6 +51,8 @@ import { ComputingUnitStatusService } from "../../../common/service/computing-un import { unavailableComputingUnitReason } from "../../../common/util/computing-unit.util"; import { WarehouseService } from "../../../common/service/warehouse/warehouse.service"; import { GuiConfigService } from "../../../common/service/gui-config.service"; +import { expandMacros, isMacro, MacroDefinition, portKey } from "../../../common/util/workflow-macro"; +import { WorkflowContent } from "../../../common/type/workflow"; // TODO: change this declaration export const FORM_DEBOUNCE_TIME_MS = 150; @@ -117,6 +119,10 @@ export class ExecuteWorkflowService { private warehouseService: WarehouseService, private config: GuiConfigService ) { + this.workflowStatusService.setMacroProvider(() => { + const expanded = this.workflowActionService.getMacroExpansion?.(); + return expanded ? { expanded, content: this.workflowActionService.getWorkflowContent() } : undefined; + }); workflowWebsocketService.websocketEvent().subscribe(event => { switch (event.type) { case "RegionUpdateEvent": @@ -196,7 +202,12 @@ export class ExecuteWorkflowService { return { state: ExecutionState.Failed, errorMessages: event.fatalErrors.map(err => { - return { ...err, message: err.message.replace("\\n", "
") }; + const path = this.workflowActionService.getMacroExpansion?.()?.paths[err.operatorId]; + return { + ...err, + operatorId: path && path.length > 1 ? path[0] : err.operatorId, + message: err.message.replace("\\n", "
"), + }; }), }; default: @@ -263,10 +274,8 @@ export class ExecuteWorkflowService { emailNotificationEnabled: boolean, targetOperatorId?: string ): void { - const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest( - this.workflowActionService.getTexeraGraph(), - targetOperatorId - ); + const logicalPlan = this.executionPlan(targetOperatorId); + if (!logicalPlan) return; const settings = this.workflowActionService.getWorkflowSettings(); if (this.refuseToRunOnUnavailableUnit()) { return; @@ -284,7 +293,8 @@ export class ExecuteWorkflowService { } public executeWorkflowWithReplay(replayExecutionInfo: ReplayExecutionInfo): void { - const logicalPlan = ExecuteWorkflowService.getLogicalPlanRequest(this.workflowActionService.getTexeraGraph()); + const logicalPlan = this.executionPlan(); + if (!logicalPlan) return; const settings = this.workflowActionService.getWorkflowSettings(); if (this.refuseToRunOnUnavailableUnit()) { return; @@ -326,6 +336,29 @@ export class ExecuteWorkflowService { return true; } + private executionPlan(targetOperatorId?: string): LogicalPlan | undefined { + if (this.workflowActionService.getMacroInterface?.()) { + this.notificationService.error("A macro runs through a workflow. Add it to a workflow to execute it."); + return; + } + try { + const dependencies = this.workflowActionService.getMacroDependencies?.() ?? []; + return dependencies.length + ? ExecuteWorkflowService.getLogicalPlanRequest( + this.workflowActionService.getTexeraGraph(), + targetOperatorId, + dependencies + ) + : targetOperatorId + ? ExecuteWorkflowService.getLogicalPlanRequest(this.workflowActionService.getTexeraGraph(), targetOperatorId) + : ExecuteWorkflowService.getLogicalPlanRequest(this.workflowActionService.getTexeraGraph()); + } catch (error) { + if (!this.workflowActionService.getMacroDependencies?.().length) throw error; + this.notificationService.error(error instanceof Error ? error.message : String(error)); + return; + } + } + /** * While the deployment requires a warehouse (#7817) and none is picked, * refuses with a toast and returns true. Checked at every public entry @@ -442,12 +475,18 @@ export class ExecuteWorkflowService { throw new Error("cannot modify logic, the current execution state is " + this.currentState.state); } const op = this.workflowActionService.getTexeraGraph().getOperator(operatorID); - const operator: LogicalOperator = { - ...op.operatorProperties, - operatorID: op.operatorID, - operatorType: op.operatorType, - }; - this.workflowWebsocketService.send("ModifyLogicRequest", { operator }); + const expanded = isMacro(op) ? expandMacros(this.workflowActionService.getWorkflowContent()) : undefined; + const operators = expanded + ? expanded.content.operators.filter(o => expanded.paths[o.operatorID][0] === operatorID) + : [op]; + for (const leaf of operators) { + const operator: LogicalOperator = { + ...leaf.operatorProperties, + operatorID: leaf.operatorID, + operatorType: leaf.operatorType, + }; + this.workflowWebsocketService.send("ModifyLogicRequest", { operator }); + } } public getExecutionStateStream(): Observable<{ @@ -540,15 +579,47 @@ export class ExecuteWorkflowService { * @param workflowGraph * @param targetOperatorId */ - public static getLogicalPlanRequest(workflowGraph: WorkflowGraphReadonly, targetOperatorId?: string): LogicalPlan { + public static getLogicalPlanRequest( + workflowGraph: WorkflowGraphReadonly, + targetOperatorId?: string, + dependencies: MacroDefinition[] = [] + ): LogicalPlan { + let subDAG = workflowGraph.getSubDAG(targetOperatorId); + let resultIDs = [...workflowGraph.getOperatorsToViewResult()]; + let reuseIDs = [...workflowGraph.getOperatorsMarkedForReuseResult()]; + if (workflowGraph.getAllOperators().some(isMacro)) { + const expanded = expandMacros({ + operators: workflowGraph.getAllOperators(), + links: workflowGraph.getAllLinks(), + macroDependencies: dependencies, + } as WorkflowContent); + const original = subDAG.operators; + const selected = new Set(original.map(o => o.operatorID)); + const operators = expanded.content.operators.filter(o => selected.has(expanded.paths[o.operatorID][0])); + const ids = new Set(operators.map(o => o.operatorID)); + subDAG = { + operators, + links: expanded.content.links.filter(l => ids.has(l.source.operatorID) && ids.has(l.target.operatorID)), + }; + resultIDs = resultIDs.flatMap(id => { + const op = original.find(o => o.operatorID === id); + return op && isMacro(op) + ? op.outputPorts.flatMap(p => expanded.ports[portKey(id, "outputs", p.portID)] ?? []).map(p => p.operatorID) + : [id]; + }); + resultIDs.push(...subDAG.operators.filter(o => o.viewResult).map(o => o.operatorID)); + reuseIDs = subDAG.operators + .filter(o => o.markedForReuse || reuseIDs.includes(expanded.paths[o.operatorID]?.[0])) + .map(o => o.operatorID); + } + const byID = new Map(subDAG.operators.map(op => [op.operatorID, op])); const getInputPortOrdinal = (operatorID: string, inputPortID: string): number => { - return workflowGraph.getOperator(operatorID).inputPorts.findIndex(port => port.portID === inputPortID); + return byID.get(operatorID)!.inputPorts.findIndex(port => port.portID === inputPortID); }; const getOutputPortOrdinal = (operatorID: string, outputPortID: string): number => { - return workflowGraph.getOperator(operatorID).outputPorts.findIndex(port => port.portID === outputPortID); + return byID.get(operatorID)!.outputPorts.findIndex(port => port.portID === outputPortID); }; - const subDAG = workflowGraph.getSubDAG(targetOperatorId); const operators: LogicalOperator[] = subDAG.operators.map(op => ({ ...op.operatorProperties, @@ -571,16 +642,24 @@ export class ExecuteWorkflowService { const operatorIds = new Set(subDAG.operators.map(op => op.operatorID)); - const opsToViewResult: string[] = Array.from(intersection(operatorIds, workflowGraph.getOperatorsToViewResult())); + const opsToViewResult: string[] = Array.from(intersection(operatorIds, new Set(resultIDs))); - const opsToReuseResult: string[] = Array.from( - intersection(operatorIds, workflowGraph.getOperatorsMarkedForReuseResult()) - ); + const opsToReuseResult: string[] = Array.from(intersection(operatorIds, new Set(reuseIDs))); return { operators, links, opsToViewResult, opsToReuseResult }; } public getWorkerIds(operatorId: string): ReadonlyArray { + const expanded = this.workflowActionService.getMacroExpansion?.(); + if ( + expanded && + this.workflowActionService.getTexeraGraph().hasOperator(operatorId) && + isMacro(this.workflowActionService.getTexeraGraph().getOperator(operatorId)) + ) { + return Object.entries(expanded.paths) + .filter(([, path]) => path[0] === operatorId) + .flatMap(([id]) => this.assignedWorkerIds.get(id) ?? []); + } return this.assignedWorkerIds.get(operatorId) || []; } } diff --git a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts index 7d51e3a6199..286bbb1bbc6 100644 --- a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts +++ b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts @@ -27,6 +27,7 @@ import { fromEventPattern, Observable } from "rxjs"; import { Coeditor } from "../../../common/type/user"; import { OperatorResultCacheStatus } from "../../types/workflow-websocket.interface"; import { HEATMAP_NO_DATA_COLOR, scoreToColor } from "../heatmap/heatmap-color"; +import { isMacro } from "../../../common/util/workflow-macro"; /** * Defines the SVG path for the delete button @@ -289,7 +290,9 @@ export class JointUIService { public getJointOperatorElement(operator: OperatorPredicate, point: Point): joint.dia.Element { // check if the operatorType exists in the operator metadata - const operatorSchema = this.operatorSchemas.find(op => op.operatorType === operator.operatorType); + const operatorSchema = isMacro(operator) + ? this.operatorMetadataService.getOperatorSchema(operator.operatorType) + : this.operatorSchemas.find(op => op.operatorType === operator.operatorType); if (operatorSchema === undefined) { throw new Error(`operator type ${operator.operatorType} doesn't exist`); } @@ -395,24 +398,24 @@ export class JointUIService { // Absent map: no per-port information in this snapshot, so leave the labels alone. // Empty map: every port measured zero, so write the zeros. if (inputMetrics !== undefined) { - inPorts.forEach(portDef => { + inPorts.forEach((portDef, index) => { const portId = portDef.id; if (portId != null) { const parts = portId.split("-"); const numericSuffix = parts.length > 1 ? parts[1] : portId; - const count: number = inputMetrics[numericSuffix] ?? 0; + const count: number = inputMetrics[numericSuffix] ?? inputMetrics[String(index)] ?? 0; element.portProp(portId, "attrs/.port-label/text", count.toLocaleString()); } }); } if (outputMetrics !== undefined) { - outPorts.forEach(portDef => { + outPorts.forEach((portDef, index) => { const portId = portDef.id; if (portId != null) { const parts = portId.split("-"); const numericSuffix = parts.length > 1 ? parts[1] : portId; - const count: number = outputMetrics[numericSuffix] ?? 0; + const count: number = outputMetrics[numericSuffix] ?? outputMetrics[String(index)] ?? 0; element.portProp(portId, "attrs/.port-label/text", count.toLocaleString()); } }); @@ -934,7 +937,9 @@ export class JointUIService { visibility: "hidden", }, ".texera-operator-icon": { - "xlink:href": "assets/operator_images/" + operatorType + ".png", + "xlink:href": operatorType.startsWith("WorkflowMacro:") + ? "assets/operator_images/SubWorkflow.svg" + : "assets/operator_images/" + operatorType + ".png", width: 35, height: 35, "ref-x": 0.5, diff --git a/frontend/src/app/workspace/service/macro/macro.service.spec.ts b/frontend/src/app/workspace/service/macro/macro.service.spec.ts new file mode 100644 index 00000000000..551f3e78511 --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro.service.spec.ts @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { of, Subject, throwError } from "rxjs"; +import { MacroService } from "./macro.service"; +import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; +import { WorkflowPersistService } from "../../../common/service/workflow-persist/workflow-persist.service"; +import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; +import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; +import { ExecutionMode, Workflow, WorkflowContent } from "../../../common/type/workflow"; +import { OperatorPredicate } from "../../types/workflow-common.interface"; +import { expandMacros, macroType } from "../../../common/util/workflow-macro"; + +describe("MacroService", () => { + const op: OperatorPredicate = { + operatorID: "leaf", + operatorType: "Filter", + operatorVersion: "1", + inputPorts: [{ portID: "input-0" }], + outputPorts: [{ portID: "output-0" }], + operatorProperties: { value: "original" }, + showAdvanced: false, + }; + let content: WorkflowContent; + let actions: any, persist: any, versions: any, metadata: any, service: MacroService; + beforeEach(() => { + content = { + operators: [op], + links: [], + operatorPositions: { leaf: { x: 25, y: 40 } }, + commentBoxes: [], + settings: { dataTransferBatchSize: 10, executionMode: ExecutionMode.PIPELINED }, + formBinding: { fields: [{ id: "value", operatorID: "leaf", propertyKey: "value", displayName: "Value" }] }, + }; + actions = { + checkWorkflowModificationEnabled: () => true, + getWorkflowContent: () => content, + bundleMacroActions: (fn: () => void) => fn(), + getWorkflow: () => ({ wid: 1, name: "Workflow", content }), + getJointGraphWrapper: () => ({ getCurrentHighlightedOperatorIDs: () => ["leaf"] }), + getMacroDependencies: () => content.macroDependencies ?? [], + getTexeraGraph: () => ({ + bundleActions: (fn: () => void) => fn(), + getOperator: (id: string) => content.operators.find(o => o.operatorID === id), + }), + deleteOperatorsAndLinks: vi.fn(() => { + content = { ...content, operators: [], links: [] }; + }), + addOperatorsAndLinks: vi.fn((ops: { op: OperatorPredicate }[], links: WorkflowContent["links"]) => { + content = { ...content, operators: ops.map(o => o.op), links }; + }), + setMacroDependencies: vi.fn(d => { + content = { ...content, macroDependencies: d }; + }), + setFormBinding: vi.fn(f => { + content = { ...content, formBinding: f }; + }), + }; + persist = { + createWorkflow: vi.fn((c: WorkflowContent, name: string) => of({ workflow: { wid: 12, name, content: c } })), + retrieveWorkflowsBySessionUser: vi.fn(() => of([])), + retrieveWorkflow: vi.fn(), + }; + versions = { + retrieveVersionsOfWorkflow: vi.fn(() => of([{ vId: 5 }])), + retrieveWorkflowByVersion: vi.fn((wid: number) => + of({ wid, name: "Clean", content: persist.createWorkflow.mock.calls.at(-1)[0] }) + ), + }; + metadata = { + registerMacros: vi.fn(), + getOperatorSchema: vi.fn(() => ({ jsonSchema: { properties: { value: { type: "string" } } } })), + }; + service = new MacroService( + actions as WorkflowActionService, + persist as WorkflowPersistService, + versions as WorkflowVersionService, + metadata as OperatorMetadataService + ); + }); + + it("creates an independent version before collapsing, preserving property bindings", async () => { + const d = await service.createFromSelection("Clean"); + expect(d.version).toBe(5); + expect(d.wid).toBe(12); + expect(persist.createWorkflow).toHaveBeenCalledOnce(); + expect(content.operators).toHaveLength(1); + expect(content.operators[0].operatorType).toBe(macroType(d)); + expect(content.formBinding!.fields[0].operatorID).toBe(content.operators[0].operatorID); + expect(content.formBinding!.fields[0].propertyKey).toBe("value"); + expect(expandMacros(content).content.operators[0].operatorProperties).toEqual(op.operatorProperties); + }); + + it("leaves the selection intact on save failure or while locked", async () => { + persist.createWorkflow.mockReturnValue(throwError(() => new Error("save failed"))); + await expect(service.createFromSelection("Clean")).rejects.toThrow("save failed"); + expect(actions.deleteOperatorsAndLinks).not.toHaveBeenCalled(); + actions.checkWorkflowModificationEnabled = () => false; + await expect(service.createFromSelection("Clean")).rejects.toThrow(/read.only/i); + await expect(service.createFromSelection(" ")).rejects.toThrow(); + }); + + it("never overwrites changes made while the creation request is in flight", async () => { + const response = new Subject<{ workflow: Workflow }>(); + persist.createWorkflow.mockReturnValue(response); + const creating = service.createFromSelection("Clean"); + const saved = persist.createWorkflow.mock.calls[0][0]; + content = { ...content, operators: [{ ...op, operatorProperties: { value: "new edit" } }] }; + response.next({ workflow: { wid: 12, name: "Clean", content: saved } as Workflow }); + response.complete(); + await expect(creating).rejects.toThrow(/changed/i); + expect(actions.deleteOperatorsAndLinks).not.toHaveBeenCalled(); + expect(content.operators[0].operatorProperties.value).toBe("new edit"); + }); + + it("discovers only accessible definitions and resolves the exact latest version", async () => { + const created = await service.createFromSelection("Clean"); + const macroWorkflow = { wid: 12, name: "Clean", content: created.content }; + persist.retrieveWorkflowsBySessionUser.mockReturnValue(of([{ workflow: { wid: 12 } }, { workflow: { wid: 99 } }])); + persist.retrieveWorkflow.mockImplementation((wid: number) => + of(wid === 12 ? macroWorkflow : { wid, content: { ...created.content, macroInterface: undefined } }) + ); + versions.retrieveWorkflowByVersion.mockReturnValue(of(macroWorkflow)); + const library = await service.refresh(); + expect(library).toHaveLength(1); + expect(versions.retrieveWorkflowByVersion).toHaveBeenCalledWith(12, 5); + expect(library[0].id).toBe(created.id); + }); + + it("forks a portable definition into a newly versioned editable copy", async () => { + const original = await service.createFromSelection("Clean"); + const copy = await service.copyDefinition(original, []); + expect(copy.id).not.toBe(original.id); + expect(copy.content.operators).toEqual(original.content.operators); + expect(copy.content.macroInterface?.id).toBe(copy.id); + expect(service.library$.value).toContain(copy); + expect(original.content.macroInterface?.id).toBe(original.id); + }); + + it("adopts compatible versions explicitly and rejects overrides invalid under a new property schema", async () => { + const previous = await service.createFromSelection("Clean"); + const latest = { ...previous, version: 6 }; + content = { ...content, operators: [{ ...content.operators[0], operatorProperties: { value: "mine" } }] }; + vi.spyOn(service, "refresh").mockResolvedValue([latest]); + await service.adoptLatest(content.operators[0].operatorID); + expect(content.operators[0].operatorType).toBe(macroType(latest)); + expect(content.operators[0].operatorProperties).toEqual({ value: "mine" }); + vi.mocked(service.refresh).mockResolvedValue([{ ...latest, version: 7 }]); + metadata.getOperatorSchema.mockReturnValue({ jsonSchema: { properties: { value: { type: "number" } } } }); + const before = JSON.stringify(content); + await expect(service.adoptLatest(content.operators[0].operatorID)).rejects.toThrow(/properties.*incompatible/i); + expect(JSON.stringify(content)).toBe(before); + }); +}); diff --git a/frontend/src/app/workspace/service/macro/macro.service.ts b/frontend/src/app/workspace/service/macro/macro.service.ts new file mode 100644 index 00000000000..f672eda73ff --- /dev/null +++ b/frontend/src/app/workspace/service/macro/macro.service.ts @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Injectable } from "@angular/core"; +import { BehaviorSubject, firstValueFrom } from "rxjs"; +import { v4 as uuid } from "uuid"; +import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; +import { WorkflowPersistService } from "../../../common/service/workflow-persist/workflow-persist.service"; +import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; +import { OperatorMetadataService } from "../operator-metadata/operator-metadata.service"; +import { Workflow } from "../../../common/type/workflow"; +import { + adoptMacroVersion, + expandMacros, + extractMacro, + MacroDefinition, + macroInstance, + macroSchema, + macroType, +} from "../../../common/util/workflow-macro"; +import { WorkflowUtilService } from "../workflow-graph/util/workflow-util.service"; +import { LogicalPort } from "../../types/workflow-common.interface"; +import Ajv from "ajv"; + +@Injectable({ providedIn: "root" }) +export class MacroService { + public readonly library$ = new BehaviorSubject([]); + public readonly inspect$ = new BehaviorSubject(undefined); + private refreshing?: Promise; + + constructor( + private actions: WorkflowActionService, + private persist: WorkflowPersistService, + private versions: WorkflowVersionService, + private metadata: OperatorMetadataService + ) {} + + /** The list endpoint omits content; bound concurrent reads rather than firing one request per saved workflow at once. */ + public refresh(): Promise { + return (this.refreshing ??= this.loadLibrary().finally(() => { + this.refreshing = undefined; + })); + } + + private async loadLibrary(): Promise { + const entries = await firstValueFrom(this.persist.retrieveWorkflowsBySessionUser()); + const definitions: MacroDefinition[] = []; + for (let i = 0; i < entries.length; i += 6) { + const batch = await Promise.all( + entries.slice(i, i + 6).map(async entry => { + const workflow = await firstValueFrom(this.persist.retrieveWorkflow(entry.workflow.wid!)); + return workflow.content.macroInterface ? this.pin(workflow) : undefined; + }) + ); + definitions.push(...batch.filter((d): d is MacroDefinition => d !== undefined)); + } + definitions.sort((a, b) => a.name.localeCompare(b.name)); + this.metadata.registerMacros(definitions.flatMap(d => [d, ...(d.content.macroDependencies ?? [])])); + this.library$.next(definitions); + return definitions; + } + + private async pin(workflow: Workflow): Promise { + const versions = await firstValueFrom(this.versions.retrieveVersionsOfWorkflow(workflow.wid!)); + const version = Math.max(...versions.map(v => v.vId)); + if (!Number.isSafeInteger(version) || version <= 0) throw new Error("Macro has no saved version"); + const snapshot = await firstValueFrom(this.versions.retrieveWorkflowByVersion(workflow.wid!, version)); + const content = WorkflowUtilService.parseWorkflowInfo(snapshot).content; + if (!content.macroInterface) throw new Error("Saved version is not a macro definition"); + expandMacros(content); + const definition = { id: content.macroInterface.id, version, name: workflow.name, wid: workflow.wid, content }; + this.metadata.registerMacros(content.macroDependencies ?? []); + macroSchema(definition, type => this.metadata.getOperatorSchema(type)); + return definition; + } + + public async copyDefinition(definition: MacroDefinition, dependencies: MacroDefinition[]): Promise { + const content = { + ...definition.content, + macroDependencies: dependencies, + macroInterface: { ...definition.content.macroInterface!, id: uuid() }, + }; + expandMacros(content); + const created = await firstValueFrom(this.persist.createWorkflow(content, definition.name + " copy")); + const copy = await this.pin(WorkflowUtilService.parseWorkflowInfo(created.workflow)); + this.metadata.registerMacros([copy, ...dependencies]); + this.library$.next([...this.library$.value, copy]); + return copy; + } + + public async createFromSelection(name: string): Promise { + if (!this.actions.checkWorkflowModificationEnabled()) throw new Error("Workflow is read-only"); + if (!name.trim()) throw new Error("Enter a macro name"); + const content = this.actions.getWorkflowContent(); + const before = JSON.stringify(content); + const selected = [...this.actions.getJointGraphWrapper().getCurrentHighlightedOperatorIDs()]; + const body = extractMacro(content, selected, uuid()); + expandMacros(body); + const created = await firstValueFrom(this.persist.createWorkflow(body, name.trim())); + const d = await this.pin(WorkflowUtilService.parseWorkflowInfo(created.workflow)); + this.metadata.registerMacros([d, ...(d.content.macroDependencies ?? [])]); + this.library$.next([...this.library$.value, d]); + if ( + !this.actions.checkWorkflowModificationEnabled() || + JSON.stringify(this.actions.getWorkflowContent()) !== before + ) { + throw new Error("The workflow changed while saving. The macro was saved; add it from the Macros panel."); + } + const instance = macroInstance(d, `WorkflowMacro-${uuid()}`); + const ids = new Set(selected); + const boundary = (p: LogicalPort, direction: "inputs" | "outputs"): LogicalPort => { + if (!ids.has(p.operatorID)) return p; + const port = body.macroInterface![direction].find(b => + b.targets.some(t => t.operatorID === p.operatorID && t.portID === p.portID) + ); + if (!port) throw new Error("Missing boundary mapping"); + return { operatorID: instance.operatorID, portID: port.portID }; + }; + const links = content.links + .filter(l => ids.has(l.source.operatorID) !== ids.has(l.target.operatorID)) + .map(l => ({ ...l, source: boundary(l.source, "outputs"), target: boundary(l.target, "inputs") })); + this.actions.bundleMacroActions(() => { + this.actions.setMacroDependencies([...this.actions.getMacroDependencies(), d]); + this.actions.deleteOperatorsAndLinks(selected); + this.actions.addOperatorsAndLinks([{ op: instance, pos: content.operatorPositions[selected[0]] }], links); + if (content.formBinding) + this.actions.setFormBinding({ + ...content.formBinding, + fields: content.formBinding.fields.map(f => + ids.has(f.operatorID) ? { ...f, operatorID: instance.operatorID, propertyKey: f.id } : f + ), + ...(content.formBinding.shownResultIds === undefined + ? {} + : { + shownResultIds: [ + ...new Set(content.formBinding.shownResultIds.map(id => (ids.has(id) ? instance.operatorID : id))), + ], + }), + }); + }); + return d; + } + + public async adoptLatest(operatorID: string): Promise { + if (!this.actions.checkWorkflowModificationEnabled()) throw new Error("Workflow is read-only"); + const content = this.actions.getWorkflowContent(); + const op = content.operators.find(o => o.operatorID === operatorID); + const previous = content.macroDependencies?.find(d => macroType(d) === op?.operatorType); + if (!previous) throw new Error("Missing pinned macro definition"); + const matches = (await this.refresh()).filter(d => d.id === previous.id); + if (matches.length > 1) + throw new Error( + "Multiple definitions share this identity. Insert the intended version from the library explicitly." + ); + const latest = matches[0]; + if (!latest) throw new Error("Definition is not in this deployment's accessible macro library"); + const updated = adoptMacroVersion(op!, previous, latest); + if ( + !new Ajv({ strict: false }).validate( + macroSchema(latest, type => this.metadata.getOperatorSchema(type)).jsonSchema, + updated.operatorProperties + ) + ) { + throw new Error("Macro properties are incompatible with the new version; insert it explicitly."); + } + if ( + !this.actions.checkWorkflowModificationEnabled() || + JSON.stringify(this.actions.getWorkflowContent()) !== JSON.stringify(content) + ) { + throw new Error("The workflow changed while checking versions. Try again."); + } + if (latest.version === previous.version) return; + const links = content.links.filter(l => l.source.operatorID === operatorID || l.target.operatorID === operatorID); + this.actions.bundleMacroActions(() => { + const definitions = [...this.actions.getMacroDependencies(), latest, ...(latest.content.macroDependencies ?? [])]; + this.actions.setMacroDependencies([...new Map(definitions.map(d => [macroType(d), d])).values()]); + this.actions.deleteOperatorsAndLinks([operatorID]); + this.actions.addOperatorsAndLinks([{ op: updated, pos: content.operatorPositions[operatorID] }], links); + }); + } +} diff --git a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts index 0f1282a4a5f..83b2eaafd19 100644 --- a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts +++ b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.spec.ts @@ -39,6 +39,7 @@ import { import { CommentBox } from "../../types/workflow-common.interface"; import { NotificationService } from "../../../common/service/notification/notification.service"; import { ExecuteWorkflowService } from "../execute-workflow/execute-workflow.service"; +import { expandMacros, macroInstance } from "../../../common/util/workflow-macro"; import { Subscription } from "rxjs"; describe("OperatorMenuService", () => { @@ -360,6 +361,38 @@ describe("OperatorMenuService", () => { expect(errorSpy).toHaveBeenCalledWith("Copy failed. You don't have the permission to write to the clipboard."); }); + + it("bundles pinned definitions when copying a macro to another workflow", async () => { + const d = { + id: "clipboard-macro", + version: 4, + name: "Scan", + content: { + ...workflowActionService.getWorkflowContent(), + operators: [mockScanPredicate], + operatorPositions: { [mockScanPredicate.operatorID]: mockPoint }, + macroInterface: { id: "clipboard-macro", inputs: [], outputs: [] }, + }, + }; + workflowActionService.setMacroDependencies([d]); + const instance = macroInstance(d, "macro-instance"); + workflowActionService.addOperator(instance, mockPoint); + workflowActionService.getJointGraphWrapper().highlightOperators(instance.operatorID); + service.saveHighlightedElements(); + const saved = writeText.mock.calls[0][0]; + expect(JSON.parse(saved).macroDependencies).toEqual([d]); + workflowActionService.clearWorkflow(); + Object.defineProperty(navigator, "clipboard", { + value: { readText: vi.fn().mockResolvedValue(saved) }, + configurable: true, + }); + service.performPasteOperation(); + await flushAsync(); + const content = workflowActionService.getWorkflowContent(); + expect(content.operators).toHaveLength(1); + expect(content.operators[0].operatorID).not.toBe(instance.operatorID); + expect(expandMacros(content).content.operators[0].operatorType).toBe(mockScanPredicate.operatorType); + }); }); describe("performPasteOperation", () => { @@ -426,6 +459,23 @@ describe("OperatorMenuService", () => { expect(workflowActionService.getTexeraGraph().getAllOperators().length).toBe(0); }); + it("refuses a clipboard macro with a missing definition before changing the graph", async () => { + const invalid = { ...mockScanPredicate, operatorType: "WorkflowMacro:missing:1" }; + readText.mockResolvedValue( + JSON.stringify({ + operators: [invalid], + links: [], + commentBoxes: [], + operatorPositions: { [invalid.operatorID]: mockPoint }, + }) + ); + const error = vi.spyOn(TestBed.inject(NotificationService), "error").mockImplementation(() => {}); + service.performPasteOperation(); + await flushAsync(); + expect(error).toHaveBeenCalledWith(expect.stringMatching(/macro version/i)); + expect(workflowActionService.getTexeraGraph().getAllOperators()).toEqual([]); + }); + it("notifies the user when reading the clipboard is rejected", async () => { readText.mockRejectedValue(new Error("blocked")); const notificationService = TestBed.inject(NotificationService); diff --git a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts index a5f9224a522..b097adce96c 100644 --- a/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts +++ b/frontend/src/app/workspace/service/operator-menu/operator-menu.service.ts @@ -26,6 +26,7 @@ import { WorkflowUtilService } from "../workflow-graph/util/workflow-util.servic import { NotificationService } from "src/app/common/service/notification/notification.service"; import { ExecuteWorkflowService } from "../execute-workflow/execute-workflow.service"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; +import { expandMacros, isMacro, MacroDefinition, macroType } from "../../../common/util/workflow-macro"; type OperatorPositions = { [key: string]: Point; @@ -42,6 +43,7 @@ type SerializedString = { operatorPositions: OperatorPositions; links: OperatorLink[]; commentBoxes: CommentBox[]; + macroDependencies?: MacroDefinition[]; }; /** @@ -210,6 +212,8 @@ export class OperatorMenuService { }); serializedString.operators = operatorsCopy; + if (operatorsCopy.some(isMacro)) + serializedString.macroDependencies = this.workflowActionService.getMacroDependencies(); serializedString.operatorPositions = operatorPositionsCopy; // get all the highlighted links, and sort them by their layers @@ -296,6 +300,26 @@ export class OperatorMenuService { ) as OperatorPositions; // get all the operators from the clipboard, which are already sorted by their layers let copiedOps: OperatorPredicate[] = elementsInClipboard.get("operators") as OperatorPredicate[]; + let dependencies: MacroDefinition[] | undefined; + if (copiedOps.some(isMacro)) { + try { + const bundled = [ + ...this.workflowActionService.getMacroDependencies(), + ...(elementsInClipboard.get("macroDependencies") ?? []), + ]; + expandMacros({ + ...this.workflowActionService.getWorkflowContent(), + operators: copiedOps, + links: linksInClipboard, + macroInterface: undefined, + macroDependencies: bundled, + }); + dependencies = [...new Map(bundled.map(d => [macroType(d), d])).values()]; + } catch (error) { + this.notificationService.error(error instanceof Error ? error.message : String(error)); + return; + } + } let linksCopy: LinkWithID = {}; copiedOps.forEach(copiedOperator => { @@ -352,7 +376,12 @@ export class OperatorMenuService { // actually add all operators and links to the workflow try { - this.workflowActionService.addOperatorsAndLinks(operatorsAndPositions, links); + const add = () => { + if (dependencies) this.workflowActionService.setMacroDependencies(dependencies); + this.workflowActionService.addOperatorsAndLinks(operatorsAndPositions, links); + }; + if (dependencies) this.workflowActionService.bundleMacroActions(add); + else add(); } catch (e) { this.notificationService.info( "Some of the links that you selected don't have operators attached to both ends of them. These links won't be pasted, since links can't exist without operators." diff --git a/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts b/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts index ce564a302ce..0d5ac4c2c32 100644 --- a/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts +++ b/frontend/src/app/workspace/service/operator-metadata/operator-metadata.service.ts @@ -23,6 +23,7 @@ import { Observable } from "rxjs"; import { AppSettings } from "../../../common/app-setting"; import { OperatorMetadata, OperatorSchema } from "../../types/operator-schema.interface"; import { shareReplay } from "rxjs/operators"; +import { MacroDefinition, macroSchema, macroType } from "../../../common/util/workflow-macro"; export const OPERATOR_METADATA_ENDPOINT = "resources/operator-metadata"; @@ -46,6 +47,19 @@ export type IOperatorMetadataService = Pick(); + private schemaStack = new Set(); + private macroProvider: () => MacroDefinition[] = () => []; + + public setMacroProvider(provider: () => MacroDefinition[]): void { + this.macroProvider = provider; + } + public registerMacros(definitions: MacroDefinition[]): void { + definitions.forEach(d => this.macros.set(macroType(d), d)); + } + public getMacroDefinition(type: string): MacroDefinition | undefined { + return this.macroProvider().find(d => macroType(d) === type) ?? this.macros.get(type); + } // holds the current version of operator metadata private currentOperatorMetadata: OperatorMetadata | undefined; @@ -54,7 +68,7 @@ export class OperatorMetadataService { .pipe(shareReplay(1)); constructor(private httpClient: HttpClient) { - this.getOperatorMetadata().subscribe(data => { + this.operatorMetadataObservable.subscribe(data => { this.currentOperatorMetadata = data; }); } @@ -75,6 +89,16 @@ export class OperatorMetadataService { } const operatorSchema = this.currentOperatorMetadata.operators.find(schema => schema.operatorType === operatorType); if (!operatorSchema) { + const definition = this.getMacroDefinition(operatorType); + if (definition) { + if (this.schemaStack.has(operatorType)) throw new Error("Recursive macro property schema"); + this.schemaStack.add(operatorType); + try { + return macroSchema(definition, type => this.getOperatorSchema(type)); + } finally { + this.schemaStack.delete(operatorType); + } + } throw new Error(`can\'t find operator schema of type ${operatorType}`); } return operatorSchema; @@ -94,6 +118,7 @@ export class OperatorMetadataService { userFriendlyNameFilter: boolean = false, caseInsensitive: boolean = false ): boolean { + if (this.getMacroDefinition(operatorType)) return true; if (!this.currentOperatorMetadata) { return false; } diff --git a/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts b/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts index 382d1f87db8..275199f9acb 100644 --- a/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts +++ b/frontend/src/app/workspace/service/operator-metadata/stub-operator-metadata.service.ts @@ -18,33 +18,14 @@ */ import { Injectable } from "@angular/core"; -import { Observable, of } from "rxjs"; +import { of } from "rxjs"; import { mockOperatorMetaData } from "./mock-operator-metadata.data"; -import { OperatorMetadata, OperatorSchema } from "../../types/operator-schema.interface"; -import { IOperatorMetadataService } from "./operator-metadata.service"; -import { shareReplay } from "rxjs/operators"; +import { OperatorMetadataService } from "./operator-metadata.service"; +import { HttpClient } from "@angular/common/http"; @Injectable() -export class StubOperatorMetadataService implements IOperatorMetadataService { - private operatorMetadataObservable = of(mockOperatorMetaData).pipe(shareReplay(1)); - - public getOperatorSchema(operatorType: string): OperatorSchema { - const operatorSchema = mockOperatorMetaData.operators.find(schema => schema.operatorType === operatorType); - if (!operatorSchema) { - throw new Error(`can\'t find operator schema of type ${operatorType}`); - } - return operatorSchema; - } - - public getOperatorMetadata(): Observable { - return this.operatorMetadataObservable; - } - - public operatorTypeExists(operatorType: string): boolean { - const operator = mockOperatorMetaData.operators.filter(op => op.operatorType === operatorType); - if (operator.length === 0) { - return false; - } - return true; +export class StubOperatorMetadataService extends OperatorMetadataService { + constructor() { + super({ get: () => of(mockOperatorMetaData) } as unknown as HttpClient); } } diff --git a/frontend/src/app/workspace/service/validation/validation-workflow.service.ts b/frontend/src/app/workspace/service/validation/validation-workflow.service.ts index afad4493139..76f39858445 100644 --- a/frontend/src/app/workspace/service/validation/validation-workflow.service.ts +++ b/frontend/src/app/workspace/service/validation/validation-workflow.service.ts @@ -303,7 +303,9 @@ export class ValidationWorkflowService { throw new Error(`operator with ID ${operatorID} doesn't exist`); } - const operatorSchema = this.operatorSchemaList.find(schema => schema.operatorType === operator.operatorType); + const operatorSchema = + this.operatorSchemaList.find(schema => schema.operatorType === operator.operatorType) ?? + this.operatorMetadataService.getOperatorSchema(operator.operatorType); if (operatorSchema === undefined) { throw new Error("operatorSchema doesn't exist"); } diff --git a/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts b/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts index c3179d6d5ae..ba6f494ed77 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/shared-model.ts @@ -29,6 +29,7 @@ import { } from "../../../types/workflow-common.interface"; import { CoeditorState, User } from "../../../../common/type/user"; import { FormBindingConfig, WorkflowSettings } from "../../../../common/type/workflow"; +import { MacroDefinition, MacroInterface } from "../../../../common/util/workflow-macro"; import { getWebsocketUrl } from "../../../../common/util/url"; import { v4 as uuid } from "uuid"; import { YType } from "../../../types/shared-editing.interface"; @@ -38,10 +39,10 @@ import { YType } from "../../../types/shared-editing.interface"; * the shared doc. Keyed "settings" (WorkflowSettings) and "formBinding" (the Form View * definition); the union keeps the map off `unknown` and rejects unrelated values. */ -export type ContentMetaValue = WorkflowSettings | FormBindingConfig; +export type ContentMetaValue = WorkflowSettings | FormBindingConfig | MacroInterface | MacroDefinition[]; /** The keys `contentMetaMap` holds. */ -export type ContentMetaKey = "settings" | "formBinding"; +export type ContentMetaKey = "settings" | "formBinding" | "macroInterface" | "macroDependencies"; /** * SharedModel encapsulates everything related to real-time shared editing for the current workflow. @@ -91,7 +92,7 @@ export class SharedModel { // Initialize Y-undo manager by aggregating intended Y-structures. Only structures included here will be undoable. this.undoManager = new Y.UndoManager( - [this.operatorIDMap, this.elementPositionMap, this.operatorLinkMap, this.commentBoxMap], + [this.operatorIDMap, this.elementPositionMap, this.operatorLinkMap, this.commentBoxMap, this.contentMetaMap], { captureTimeout: 100, } diff --git a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts index 984ff04ff9c..551332b8049 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.spec.ts @@ -94,6 +94,38 @@ describe("WorkflowActionService", () => { expect(injectedService).toBeTruthy(); })); + it("undoes macro metadata together with its graph transaction", () => { + syncSharedDoc(); + const boundary = { id: "macro", inputs: [], outputs: [] }; + texeraGraph.bundleActions(() => { + service.setMacroInterface(boundary); + service.addOperator(mockScanPredicate, mockPoint); + }); + expect(service.getWorkflowContent().macroInterface).toEqual(boundary); + undoRedo.undoAction(); + expect(texeraGraph.getAllOperators()).toEqual([]); + expect(service.getMacroInterface()).toBeUndefined(); + undoRedo.redoAction(); + expect(service.getMacroInterface()).toEqual(boundary); + expect(texeraGraph.getAllOperators()).toHaveLength(1); + }); + + it("undoes and redoes macro bindings while the shared document is waiting to sync", () => { + const original = { fields: [{ id: "field", operatorID: "old", propertyKey: "value", displayName: "Value" }] }; + service.setFormBinding(original); + service.bundleMacroActions(() => { + service.setFormBinding({ fields: [{ ...original.fields[0], operatorID: "new" }] }); + service.addOperator(mockScanPredicate, mockPoint); + }); + undoRedo.undoAction(); + expect(service.getFormBinding()).toEqual(original); + undoRedo.redoAction(); + expect(service.getFormBinding().fields[0].operatorID).toBe("new"); + syncSharedDoc(); + undoRedo.undoAction(); + expect(service.getFormBinding()).toEqual(original); + }); + // The operator canvas and the Form View hand one open workflow between them; the arriving view // asks this before loading, because seeding a second document for the same workflow would leave // the co-editing room and rejoin it. diff --git a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts index f73b97c89af..bb265d2ceb4 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/workflow-action.service.ts @@ -21,7 +21,7 @@ import { Injectable } from "@angular/core"; import * as joint from "jointjs"; import * as Y from "yjs"; -import { isEqual } from "lodash-es"; +import { cloneDeep, isEqual } from "lodash-es"; import { BehaviorSubject, merge, Observable, Subject } from "rxjs"; import { ExecutionMode, @@ -54,6 +54,15 @@ import { User } from "../../../../common/type/user"; import { SharedModelChangeHandler } from "./shared-model-change-handler"; import { ContentMetaKey, ContentMetaValue } from "./shared-model"; import { GuiConfigService } from "../../../../common/service/gui-config.service"; +import { + expandMacros, + isMacro, + macroResultAliases, + macroType, + MacroDefinition, + MacroExpansion, + MacroInterface, +} from "../../../../common/util/workflow-macro"; /** A content-meta seed waiting to land (see seedContentMeta): its identity, and how to stop * listening for the sync it waits on once it is superseded or cancelled. */ @@ -112,6 +121,8 @@ export class WorkflowActionService { public readonly resultPanelOpen$: Observable = this.resultPanelOpenSubject.asObservable(); private workflowResetSubject = new Subject(); + public readonly macroChanged$ = new Subject(); + private macroExpansionCache?: { key: string; expanded: MacroExpansion }; // The Form View definition. Presentation, not structure, but it still lives in the shared // doc (shared-model contentMetaMap, key "formBinding") so a co-editor sees it live and no @@ -148,6 +159,7 @@ export class WorkflowActionService { private config: GuiConfigService ) { this.texeraGraph = new WorkflowGraph(); + this.operatorMetadataService.setMacroProvider(() => this.getMacroDependencies()); this.jointGraph = new joint.dia.Graph(); this.jointGraphWrapper = new JointGraphWrapper(this.jointGraph); @@ -204,6 +216,13 @@ export class WorkflowActionService { if (event.changes.keys.has("formBinding") && this.announces("formBinding")) { this.formBindingChangeSubject.next(this.getFormBinding()); } + if ( + ["macroInterface", "macroDependencies"].some( + key => event.changes.keys.has(key) && this.announces(key as ContentMetaKey) + ) + ) { + this.macroChanged$.next(); + } // Settings are announced the same way, so a co-editor's change reaches the settings panel, // which refreshes from this stream, instead of sitting stale until an unrelated edit. Not // through workflowChanged, though: the panel persists a settings change itself, and the @@ -214,6 +233,20 @@ export class WorkflowActionService { }; contentMetaMap.observe(this.contentMetaObserver); this.observedContentMetaMap = contentMetaMap; + const shared = this.texeraGraph.sharedModel; + shared.undoManager.on("stack-item-popped", ({ stackItem, type }) => { + const snapshots = stackItem.meta.get("macroContent"); + if (!snapshots) return; + shared.yDoc.transact(() => { + for (const [key, value] of snapshots[type === "undo" ? "before" : "after"]) { + this.seedContentMeta(key, value, true); + } + }, "macro-content-undo"); + this.formBindingChangeSubject.next(this.getFormBinding()); + this.macroChanged$.next(); + const destination = type === "undo" ? shared.undoManager.redoStack : shared.undoManager.undoStack; + destination.at(-1)?.meta.set("macroContent", snapshots); + }); } /** Whether a change to a key in the shared map is announced: not a seed's own write, which is an @@ -300,6 +333,13 @@ export class WorkflowActionService { * @param point */ public addOperator(operator: OperatorPredicate, point: Point): void { + this.texeraGraph.assertOperatorNotExists(operator.operatorID); + let dependencies: MacroDefinition[] | undefined; + if (isMacro(operator) && !this.getMacroDependencies().some(d => macroType(d) === operator.operatorType)) { + const definition = this.operatorMetadataService.getMacroDefinition(operator.operatorType); + if (!definition) throw new Error("Missing pinned macro version"); + dependencies = [...this.getMacroDependencies(), definition, ...(definition.content.macroDependencies ?? [])]; + } // turn off multiselect since there's only one operator added this.jointGraphWrapper.setMultiSelectMode(false); // check that the operator doesn't exist @@ -310,6 +350,7 @@ export class WorkflowActionService { } this.texeraGraph.bundleActions(() => { + if (dependencies) this.setMacroDependencies([...new Map(dependencies.map(d => [macroType(d), d])).values()]); // add operator to texera graph this.texeraGraph.addOperator(operator); this.texeraGraph.sharedModel.elementPositionMap?.set(operator.operatorID, point); @@ -801,6 +842,8 @@ export class WorkflowActionService { const workflowContent: WorkflowContent = workflow.content; this.hydrateSettings(workflowContent.settings); this.hydrateFormBinding(workflowContent.formBinding); + this.seedContentMeta("macroInterface", workflowContent.macroInterface); + this.seedContentMeta("macroDependencies", workflowContent.macroDependencies); let operatorsAndPositions: { op: OperatorPredicate; pos: Point }[] = []; workflowContent.operators.forEach(op => { @@ -855,6 +898,7 @@ export class WorkflowActionService { this.getTexeraGraph().getPortDisplayNameChangedSubject(), this.getTexeraGraph().getPortPropertyChangedStream(), this.formBindingChanged$, + this.macroChanged$, this.workflowResetSubject.asObservable() ); } @@ -976,7 +1020,7 @@ export class WorkflowActionService { this.seedingContentMetaKeys.add(key); try { if (!isEqual(shared.contentMetaMap.get(key), stored)) { - shared.contentMetaMap.set(key, stored); + shared.yDoc.transact(() => shared.contentMetaMap.set(key, stored), "content-meta-seed"); } } finally { this.seedingContentMetaKeys.delete(key); @@ -1064,6 +1108,59 @@ export class WorkflowActionService { return this.workflowMetadata; } + public getMacroDependencies(): MacroDefinition[] { + return (this.readContentMeta("macroDependencies") as MacroDefinition[]) ?? []; + } + + public getMacroExpansion(): MacroExpansion | undefined { + const content = this.getWorkflowContent(); + if (!content.operators.some(isMacro)) return undefined; + const key = JSON.stringify([content.operators, content.links, content.macroDependencies]); + if (this.macroExpansionCache?.key === key) return this.macroExpansionCache.expanded; + try { + const expanded = expandMacros(content); + this.macroExpansionCache = { key, expanded }; + return expanded; + } catch { + return undefined; + } // Compilation reports malformed edits; keep the runtime streams alive meanwhile. + } + + public getMacroResultAliases(): Record { + const expanded = this.getMacroExpansion(); + return expanded ? macroResultAliases(this.getWorkflowContent(), expanded) : {}; + } + + public setMacroDependencies(definitions: MacroDefinition[]): void { + this.operatorMetadataService.registerMacros(definitions); + this.writeContentMeta("macroDependencies", definitions, () => this.macroChanged$.next()); + } + + /** Pending metadata has not entered Yjs yet; attach it to the same undo item as the collapse. */ + public bundleMacroActions(action: () => void): void { + const keys: ContentMetaKey[] = ["formBinding", "macroDependencies", "macroInterface"]; + const shared = this.texeraGraph.sharedModel; + const pending = shared.wsProvider.shouldConnect && !shared.wsProvider.synced; + const before = keys.map(key => [key, cloneDeep(this.readContentMeta(key))]); + shared.undoManager.stopCapturing(); + this.texeraGraph.bundleActions(action); + if (pending) + shared.undoManager.undoStack.at(-1)?.meta.set("macroContent", { + before, + after: keys.map(key => [key, cloneDeep(this.readContentMeta(key))]), + }); + shared.undoManager.stopCapturing(); + } + + public getMacroInterface(): MacroInterface | undefined { + return this.readContentMeta("macroInterface") as MacroInterface | undefined; + } + + public setMacroInterface(boundary: MacroInterface): void { + if (!this.checkWorkflowModificationEnabled()) return; + this.writeContentMeta("macroInterface", boundary, () => this.macroChanged$.next()); + } + public getWorkflowContent(): WorkflowContent { // collect workflow content const texeraGraph = this.getTexeraGraph(); @@ -1089,6 +1186,8 @@ export class WorkflowActionService { links, commentBoxes, settings, + ...(this.getMacroInterface() ? { macroInterface: this.getMacroInterface() } : {}), + ...(this.getMacroDependencies().length ? { macroDependencies: this.getMacroDependencies() } : {}), // Carry formBinding only when the workflow has one (opened with it, or an author // populated it since), so a plain workflow's content is unchanged and its save cuts no // needless version. hasContentMeta stands in for the old "loaded" flag: hydrate sets the diff --git a/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts b/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts index 8b0c54fd593..45034da2fb3 100644 --- a/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts +++ b/frontend/src/app/workspace/service/workflow-graph/util/workflow-util.service.ts @@ -27,6 +27,7 @@ import { import { OperatorMetadataService } from "../../operator-metadata/operator-metadata.service"; import { InputPortInfo, OperatorSchema, OutputPortInfo } from "../../../types/operator-schema.interface"; import { Injectable } from "@angular/core"; +import { isMacro, macroInstance } from "../../../../common/util/workflow-macro"; import { v4 as uuid } from "uuid"; import Ajv from "ajv"; @@ -85,6 +86,12 @@ export class WorkflowUtilService { * @returns a new OperatorPredicate of the operatorType */ public getNewOperatorPredicate(operatorType: string, customDisplayName?: string): OperatorPredicate { + const macro = this.operatorMetadataService.getMacroDefinition?.(operatorType); + if (macro) + return { + ...macroInstance(macro, `WorkflowMacro-${this.getOperatorRandomUUID()}`), + customDisplayName: customDisplayName ?? macro.name, + }; const operatorSchema = this.operatorSchemaList.find(schema => schema.operatorType === operatorType); if (operatorSchema === undefined) { throw new Error(`operatorType ${operatorType} doesn't exist in operator metadata`); @@ -184,6 +191,7 @@ export class WorkflowUtilService { } public updateOperatorVersion(op: OperatorPredicate) { + if (isMacro(op)) return op; const operatorType = op.operatorType; const operatorSchema = this.operatorSchemaList.find(schema => schema.operatorType === operatorType); if (operatorSchema === undefined) { diff --git a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts index e53de2c11fc..c0f3806f849 100644 --- a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts +++ b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.spec.ts @@ -151,6 +151,22 @@ describe("WorkflowResultService", () => { expect(initiated).toEqual(["pagOp", "dataOp"]); }); + it("resolves a macro result to the existing leaf service and forwards boundary updates", () => { + service.setMacroAliases(() => ({ macro: "leaf" })); + const updates: Record[] = []; + service.getResultUpdateStream().subscribe(update => updates.push(update)); + pushWsEvent(TestBed.inject(WorkflowWebsocketService), { + type: "WebResultUpdateEvent", + updates: { leaf: snapshotUpdate([{ a: 1 }]) }, + tableStats: {}, + }); + expect(service.getResultService("macro")).toBe(service.getResultService("leaf")); + expect(service.hasNonEmptyResult("macro")).toBe(true); + expect(updates[0]["macro"]).toBe(updates[0]["leaf"]); + service.clearResults(); + expect(service.hasAnyResult("macro")).toBe(false); + }); + it("creates no result service for a cleared operator entry but still forwards the update record", () => { const ws = TestBed.inject(WorkflowWebsocketService); const updateEvents: Record[] = []; diff --git a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts index e7879ba15e8..1161c9a2989 100644 --- a/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts +++ b/frontend/src/app/workspace/service/workflow-result/workflow-result.service.ts @@ -42,6 +42,17 @@ import { SchemaAttribute } from "../../types/workflow-compiling.interface"; providedIn: "root", }) export class WorkflowResultService { + private macroAliases: () => Record = () => ({}); + public setMacroAliases(provider: () => Record): void { + this.macroAliases = provider; + } + private withMacroAliases(event: Record): Record { + const aliases = this.macroAliases(); + if (!Object.keys(aliases).length) return event; + const result = { ...event }; + for (const [alias, id] of Object.entries(aliases)) if (event[id] !== undefined) result[alias] = event[id]; + return result; + } private paginatedResultServices = new Map(); private operatorResultServices = new Map(); @@ -109,11 +120,11 @@ export class WorkflowResultService { } public getPaginatedResultService(operatorID: string): OperatorPaginationResultService | undefined { - return this.paginatedResultServices.get(operatorID); + return this.paginatedResultServices.get(this.macroAliases()[operatorID] ?? operatorID); } public getResultService(operatorID: string): OperatorResultService | undefined { - return this.operatorResultServices.get(operatorID); + return this.operatorResultServices.get(this.macroAliases()[operatorID] ?? operatorID); } /** @@ -143,7 +154,7 @@ export class WorkflowResultService { this.paginatedResultServices.delete(operatorID); } }); - this.resultUpdateStream.next(event); + this.resultUpdateStream.next(this.withMacroAliases(event)); } private handleTableStatsUpdate(event: WorkflowResultTableStats): void { @@ -151,7 +162,7 @@ export class WorkflowResultService { const paginatedResultService = this.getOrInitPaginatedResultService(operatorID); paginatedResultService.handleStatsUpdate(event[operatorID]); }); - this.resultTableStats.next(event); + this.resultTableStats.next(this.withMacroAliases(event)); } private getOrInitPaginatedResultService(operatorID: string): OperatorPaginationResultService { diff --git a/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts b/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts index 0e20939d1b7..5370b9d011c 100644 --- a/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts +++ b/frontend/src/app/workspace/service/workflow-status/workflow-status.service.ts @@ -22,11 +22,17 @@ import { BehaviorSubject, Observable, Subject } from "rxjs"; import { OperatorRuntimeStatus, OperatorState, OperatorStatistics } from "../../types/execute-workflow.interface"; import { WorkflowWebsocketService } from "../workflow-websocket/workflow-websocket.service"; import { OperatorPerformanceMetrics, extractPerformanceMetrics } from "./performance-metrics"; +import { MacroExpansion, rollupMacroStatus } from "../../../common/util/workflow-macro"; +import { WorkflowContent } from "../../../common/type/workflow"; @Injectable({ providedIn: "root", }) export class WorkflowStatusService { + private macroProvider?: () => { content: WorkflowContent; expanded: MacroExpansion } | undefined; + public setMacroProvider(provider: typeof this.macroProvider): void { + this.macroProvider = provider; + } // The engine streams operator state and operator statistics bundled in one // wire object (OperatorRuntimeStatus); this service splits them into two // separate sub-concepts, each with its own stream and snapshot. Derived @@ -70,6 +76,8 @@ export class WorkflowStatusService { /** Splits a bundled runtime-status map into the two sub-concept emissions. */ private ingestRuntimeStatus(runtimeStatus: Record): void { + const macro = this.macroProvider?.(); + if (macro) runtimeStatus = rollupMacroStatus(macro.content, runtimeStatus, macro.expanded); const state: Record = {}; const statistics: Record = {}; for (const [operatorId, update] of Object.entries(runtimeStatus)) { diff --git a/frontend/src/assets/operator_images/SubWorkflow.svg b/frontend/src/assets/operator_images/SubWorkflow.svg new file mode 100644 index 00000000000..9501a671af3 --- /dev/null +++ b/frontend/src/assets/operator_images/SubWorkflow.svg @@ -0,0 +1,16 @@ + + + + + + + From 46cd2041710fa4b485997eb5951e3cff936ce3cc Mon Sep 17 00:00:00 2001 From: carloea2 Date: Thu, 8 Oct 2026 00:07:27 -0600 Subject: [PATCH 02/11] fix(frontend): pin macros after the saved history delta --- .../workspace/service/macro/macro.service.spec.ts | 13 ++++++++++--- .../app/workspace/service/macro/macro.service.ts | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/workspace/service/macro/macro.service.spec.ts b/frontend/src/app/workspace/service/macro/macro.service.spec.ts index 551f3e78511..9de2ea4c02e 100644 --- a/frontend/src/app/workspace/service/macro/macro.service.spec.ts +++ b/frontend/src/app/workspace/service/macro/macro.service.spec.ts @@ -78,8 +78,9 @@ describe("MacroService", () => { }; versions = { retrieveVersionsOfWorkflow: vi.fn(() => of([{ vId: 5 }])), - retrieveWorkflowByVersion: vi.fn((wid: number) => - of({ wid, name: "Clean", content: persist.createWorkflow.mock.calls.at(-1)[0] }) + // History applies each save's inverse patch inclusively: the creation version returns {}. + retrieveWorkflowByVersion: vi.fn((wid: number, vid: number) => + of({ wid, name: "Clean", content: vid > 5 ? persist.createWorkflow.mock.calls.at(-1)[0] : {} }) ), }; metadata = { @@ -115,6 +116,12 @@ describe("MacroService", () => { await expect(service.createFromSelection(" ")).rejects.toThrow(); }); + it("rejects a saved snapshot without a macro interface without changing the selection", async () => { + versions.retrieveWorkflowByVersion.mockReturnValue(of({ wid: 12, content: {} })); + await expect(service.createFromSelection("Clean")).rejects.toThrow("Saved version is not a macro definition"); + expect(actions.deleteOperatorsAndLinks).not.toHaveBeenCalled(); + }); + it("never overwrites changes made while the creation request is in flight", async () => { const response = new Subject<{ workflow: Workflow }>(); persist.createWorkflow.mockReturnValue(response); @@ -138,7 +145,7 @@ describe("MacroService", () => { versions.retrieveWorkflowByVersion.mockReturnValue(of(macroWorkflow)); const library = await service.refresh(); expect(library).toHaveLength(1); - expect(versions.retrieveWorkflowByVersion).toHaveBeenCalledWith(12, 5); + expect(versions.retrieveWorkflowByVersion).toHaveBeenCalledWith(12, 6); expect(library[0].id).toBe(created.id); }); diff --git a/frontend/src/app/workspace/service/macro/macro.service.ts b/frontend/src/app/workspace/service/macro/macro.service.ts index f672eda73ff..de8076692ae 100644 --- a/frontend/src/app/workspace/service/macro/macro.service.ts +++ b/frontend/src/app/workspace/service/macro/macro.service.ts @@ -79,7 +79,8 @@ export class MacroService { const versions = await firstValueFrom(this.versions.retrieveVersionsOfWorkflow(workflow.wid!)); const version = Math.max(...versions.map(v => v.vId)); if (!Number.isSafeInteger(version) || version <= 0) throw new Error("Macro has no saved version"); - const snapshot = await firstValueFrom(this.versions.retrieveWorkflowByVersion(workflow.wid!, version)); + // History applies inverse patches inclusively; skip this save's patch to pin its resulting content. + const snapshot = await firstValueFrom(this.versions.retrieveWorkflowByVersion(workflow.wid!, version + 1)); const content = WorkflowUtilService.parseWorkflowInfo(snapshot).content; if (!content.macroInterface) throw new Error("Saved version is not a macro definition"); expandMacros(content); From bc80a5b7013fd376595fb2763719f45d9611b101 Mon Sep 17 00:00:00 2001 From: carloea2 Date: Wed, 14 Oct 2026 17:03:21 -0600 Subject: [PATCH 03/11] feat(frontend): refine macros and validate portable imports Related to #8913 --- .../app/common/util/workflow-macro.spec.ts | 25 +- .../user-workflow.component.spec.ts | 28 ++ .../user-workflow/user-workflow.component.ts | 15 +- .../user/download/download.service.spec.ts | 11 + .../macro-inspector.component.html | 77 +++++ .../macro-inspector.component.spec.ts | 9 + .../macro-inspector.component.ts | 48 +-- .../operator-menu/macro-panel.component.html | 225 ++++++++++++++ .../macro-panel.component.spec.ts | 46 ++- .../operator-menu/macro-panel.component.ts | 286 ++++-------------- .../operator-menu/macro.component.scss | 150 +++++++++ .../workflow-editor.component.spec.ts | 13 + .../workflow-editor.component.ts | 7 +- .../execute-workflow.service.spec.ts | 22 ++ .../execute-workflow.service.ts | 15 +- .../model/shared-model-change-handler.spec.ts | 19 ++ .../model/shared-model-change-handler.ts | 13 +- 17 files changed, 722 insertions(+), 287 deletions(-) create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.html create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html create mode 100644 frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss diff --git a/frontend/src/app/common/util/workflow-macro.spec.ts b/frontend/src/app/common/util/workflow-macro.spec.ts index 97f736e8923..af0a03448f2 100644 --- a/frontend/src/app/common/util/workflow-macro.spec.ts +++ b/frontend/src/app/common/util/workflow-macro.spec.ts @@ -17,7 +17,8 @@ * under the License. */ -import { ExecutionMode, WorkflowContent } from "../type/workflow"; +import { ExecutionMode, WorkflowContent, exportedWorkflow } from "../type/workflow"; +import { DefaultView } from "../../dashboard/type/workflow-metadata.interface"; import { OperatorPredicate, OperatorLink } from "../../workspace/types/workflow-common.interface"; import { expandMacros, @@ -77,6 +78,28 @@ describe("workflow macros", () => { } }); + it("round-trips nested pinned definitions without their source workflow references", () => { + const child = definition(); + const parent = { + ...definition("outer", 8), + content: { + ...graph([macroInstance(child, "nested")]), + macroInterface: { + id: "outer", + inputs: [{ portID: "input-0", targets: [{ operatorID: "nested", portID: "input-0" }] }], + outputs: [{ portID: "output-0", targets: [{ operatorID: "nested", portID: "output-0" }] }], + }, + }, + }; + const content = { ...graph([macroInstance(parent, "root")]), macroDependencies: [parent, child] }; + const { defaultView, ...uploaded } = JSON.parse(JSON.stringify(exportedWorkflow(content, DefaultView.FORM))); + expect(defaultView).toBe(DefaultView.FORM); + uploaded.macroDependencies.forEach((d: { wid?: number }) => delete d.wid); + expect(expandMacros(uploaded)).toEqual(expandMacros(content)); + uploaded.macroDependencies.pop(); + expect(() => expandMacros(uploaded)).toThrow(/missing.*version/i); + }); + it("expands repeated pinned instances with distinct, deterministic short IDs and local overrides", () => { const d = definition(); const first = { ...macroInstance(d, "first"), operatorProperties: { value: "override" } }; diff --git a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts index 26373628c6b..5f180fc96a1 100644 --- a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.spec.ts @@ -970,6 +970,34 @@ describe("SavedWorkflowSectionComponent", () => { }); describe("workflow uploads", () => { + it("rejects a macro file missing its pinned definition before saving", async () => { + const persist = TestBed.inject(WorkflowPersistService) as any; + persist.createWorkflow = vi.fn(); + const errorSpy = vi + .spyOn(TestBed.inject(NotificationService), "error") + .mockImplementation(() => undefined as any); + const content = { + ...testWorkflowContent([]), + operators: [ + { + operatorID: "missing", + operatorType: "WorkflowMacro:missing:1", + operatorVersion: "1", + operatorProperties: {}, + inputPorts: [], + outputPorts: [], + showAdvanced: false, + }, + ], + }; + await expect( + firstValueFrom( + component.onClickUploadExistingWorkflowFromLocal(new File([JSON.stringify(content)], "broken.json") as any) + ) + ).rejects.toThrow(/Missing macro version/); + expect(persist.createWorkflow).not.toHaveBeenCalled(); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("Missing macro version")); + }); it("imports a valid .json workflow, appending an entry and toasting success", async () => { const persist = TestBed.inject(WorkflowPersistService) as any; persist.createWorkflow = vi.fn().mockReturnValue(of(makeDashboardWorkflow(42, "wf"))); diff --git a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts index 06f498a80f7..89aaf6ff49d 100644 --- a/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts +++ b/frontend/src/app/dashboard/component/user/user-workflow/user-workflow.component.ts @@ -30,6 +30,7 @@ import { UserService } from "../../../../common/service/user/user.service"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { NotificationService } from "../../../../common/service/notification/notification.service"; import { ExecutionMode, ExportedWorkflow, WorkflowContent } from "../../../../common/type/workflow"; +import { expandMacros, isMacro } from "../../../../common/util/workflow-macro"; import { NzUploadFile, NzUploadComponent } from "ng-zorro-antd/upload"; import JSZip from "jszip"; import { FiltersComponent } from "../filters/filters.component"; @@ -637,6 +638,7 @@ export class UserWorkflowComponent implements AfterViewInit, OnDestroy { let reader = new FileReader(); reader.readAsText(file); reader.onload = () => { + let validatingMacro = false; try { const result = reader.result; if (typeof result !== "string") { @@ -645,6 +647,15 @@ export class UserWorkflowComponent implements AfterViewInit, OnDestroy { // The landing view rides as a sibling key next to the content (see exportedWorkflow); // pull it back out so it is stored on the workflow row, not inside content. const { defaultView, ...workflowContent } = JSON.parse(result) as ExportedWorkflow; + validatingMacro = !!( + workflowContent.macroInterface || + workflowContent.macroDependencies || + workflowContent.operators?.some(isMacro) + ); + if (validatingMacro) { + expandMacros(workflowContent); + } + validatingMacro = false; this.workflowPersistService .createWorkflow(workflowContent, this.deriveWorkflowName(name), defaultView) .pipe(untilDestroyed(this)) @@ -663,7 +674,9 @@ export class UserWorkflowComponent implements AfterViewInit, OnDestroy { }); } catch (error) { this.notificationService.error( - "An error occurred when importing the workflow. Please import a workflow json file." + validatingMacro && error instanceof Error + ? error.message + : "An error occurred when importing the workflow. Please import a workflow json file." ); observer.error(error); } diff --git a/frontend/src/app/dashboard/service/user/download/download.service.spec.ts b/frontend/src/app/dashboard/service/user/download/download.service.spec.ts index 0d1c3ac734f..1ee54f283ba 100644 --- a/frontend/src/app/dashboard/service/user/download/download.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/download/download.service.spec.ts @@ -246,6 +246,17 @@ describe("DownloadService", () => { reader.readAsText(blob); }); + it("includes pinned macro dependencies in the downloaded file", async () => { + const content = { + operators: [], + links: [], + macroDependencies: [{ id: "nested", version: 8, content: { operators: [] } }], + }; + workflowPersistServiceSpy.retrieveWorkflow.mockReturnValue(of({ content } as any)); + const result = await firstValueFrom(downloadService.downloadWorkflow(42, "Macros")); + expect(JSON.parse(await readBlob(result.blob)).macroDependencies).toEqual(content.macroDependencies); + }); + it("carries the default view into the downloaded workflow json so it round-trips on upload", async () => { const content = { operators: [], links: [] }; workflowPersistServiceSpy.retrieveWorkflow.mockReturnValue(of({ content, defaultView: "FORM" } as any)); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.html new file mode 100644 index 00000000000..e7b29d8edd6 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.html @@ -0,0 +1,77 @@ + +
+
+ +

{{ definition.name }}

+ Pinned version · v{{ definition.version }} +
+
+ Edit latest definition + +
+
+

+ Double-click a nested macro to explore it. Changes to a definition apply when you choose to update an instance. +

+
+ Operators · {{ definition.content.operators.length }} + Inputs · {{ definition.content.macroInterface?.inputs?.length || 0 }} + Outputs · {{ definition.content.macroInterface?.outputs?.length || 0 }} +
+
+
+ Operator properties +

+ No operators in this definition +

+
+ {{ op.customDisplayName || op.operatorType }} +
{{ op.operatorProperties | json }}
+
+
diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts index fd39837c611..954e44c70e5 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.spec.ts @@ -77,4 +77,13 @@ describe("MacroInspectorComponent", () => { expect(fixture.componentInstance.copying).toBe(false); expect(fixture.componentInstance.definition).toBe(definition); }); + + it("identifies the pinned snapshot and labels its graph and empty properties", () => { + const fixture = TestBed.createComponent(MacroInspectorComponent); + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("Pinned version"); + expect(fixture.nativeElement.textContent).toContain("No operators in this definition"); + expect(fixture.nativeElement.querySelector('[aria-label="Macro graph"]')).toBeTruthy(); + expect(fixture.nativeElement.querySelector("a").textContent).toContain("Edit latest definition"); + }); }); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts index 92906aa14fd..a54108bb370 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-inspector.component.ts @@ -19,6 +19,7 @@ import { AfterViewInit, Component, ElementRef, Inject, OnDestroy, ViewChild } from "@angular/core"; import { JsonPipe, NgFor, NgIf } from "@angular/common"; import { NZ_MODAL_DATA } from "ng-zorro-antd/modal"; +import { NzButtonModule } from "ng-zorro-antd/button"; import * as joint from "jointjs"; import { isMacro, MacroDefinition, macroType } from "../../../../common/util/workflow-macro"; import { JointUIService } from "../../../service/joint-ui/joint-ui.service"; @@ -28,50 +29,9 @@ import { NotificationService } from "../../../../common/service/notification/not @Component({ selector: "texera-macro-inspector", - imports: [NgIf, NgFor, JsonPipe], - template: ` - {{ definition.name }} · v{{ definition.version }} - Edit definition - -

- Double-click a nested macro to inspect it. Editing a definition creates versions; existing instances stay pinned. -

-
-
- {{ op.customDisplayName || op.operatorType }} -
{{ op.operatorProperties | json }}
-
`, - styles: [ - ` - strong, - a, - button { - margin-right: 12px; - } - pre { - max-height: 120px; - overflow: auto; - } - `, - ], + imports: [NgIf, NgFor, JsonPipe, NzButtonModule], + templateUrl: "./macro-inspector.component.html", + styleUrls: ["./macro.component.scss"], }) export class MacroInspectorComponent implements AfterViewInit, OnDestroy { @ViewChild("canvas", { static: true }) canvas!: ElementRef; diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html new file mode 100644 index 00000000000..2e3e650e970 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html @@ -0,0 +1,225 @@ + +
+ Macros {{ macros.library$.value.length }} +
+ Saved macros + +
+

+ No saved macros yet. Create one from a selection below. +

+

+ Drag a macro onto the canvas. +

+
+ +
+
+ +

+ {{ selectionHint }} +

+ + +
+
+ Selected macro +

This instance keeps its pinned version until you update it.

+
+ + +
+
+
+
+ Macro interface +

Choose what workflows can connect to and configure.

+
+
+ {{ direction === 'inputs' ? 'Inputs' : 'Outputs' }} + {{ boundary[direction].length }} +
+

+ No {{ direction }} exposed. +

+
+ + {{ targetLabel(direction, target) }} +
+ + +
+
+ + +
+
+
+ Exposed properties{{ binding.fields.length }} +
+

Set these values on each macro instance.

+
+ + +
+ + +
+
diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts index 8058d3b984b..5f1abb34748 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.spec.ts @@ -55,7 +55,7 @@ describe("MacroPanelComponent", () => { createFromSelection: vi.fn(async () => ({})), adoptLatest: vi.fn(async () => {}), }; - notification = { error: vi.fn() }; + notification = { error: vi.fn(), success: vi.fn() }; TestBed.configureTestingModule({ imports: [MacroPanelComponent], providers: [ @@ -124,4 +124,48 @@ describe("MacroPanelComponent", () => { ], }); }); + + it("explains empty selection and read-only state without hiding the library", () => { + actions.getJointGraphWrapper = () => ({ getCurrentHighlightedOperatorIDs: () => [] }); + const fixture = TestBed.createComponent(MacroPanelComponent); + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("No saved macros yet"); + expect(fixture.nativeElement.textContent).toContain("Select operators on the canvas"); + expect(fixture.nativeElement.querySelector('[data-action="create"]').disabled).toBe(true); + actions.checkWorkflowModificationEnabled = () => false; + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("Read-only workflow"); + expect(fixture.nativeElement.querySelector("summary").textContent).toContain("Macros"); + }); + + it("submits a named selection from the keyboard and clears the name only after success", async () => { + const fixture = TestBed.createComponent(MacroPanelComponent); + fixture.componentInstance.name = "清理 · café"; + fixture.detectChanges(); + expect(fixture.nativeElement.textContent).toContain("1 operator selected"); + fixture.nativeElement.querySelector("form").dispatchEvent(new Event("submit", { cancelable: true })); + await fixture.whenStable(); + fixture.detectChanges(); + expect(service.createFromSelection).toHaveBeenCalledWith("清理 · café"); + expect(fixture.componentInstance.name).toBe(""); + expect(notification.success).toHaveBeenCalledWith("Macro created"); + service.createFromSelection.mockRejectedValue(new Error("save failed")); + fixture.componentInstance.name = "Try again"; + await fixture.componentInstance.create(); + expect(fixture.componentInstance.name).toBe("Try again"); + }); + + it("keeps a port name focused when shared metadata is reread", () => { + actions.getMacroInterface = () => + structuredClone({ + ...boundary, + inputs: [{ portID: "input-0", displayName: "Rows", targets: [{ operatorID: "leaf", portID: "input-0" }] }], + }); + const fixture = TestBed.createComponent(MacroPanelComponent); + fixture.detectChanges(); + const input = fixture.nativeElement.querySelector('[aria-label="Port name input-0"]') as HTMLInputElement; + input.focus(); + fixture.detectChanges(); + expect(document.activeElement).toBe(input); + }); }); diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts index bf34c9ef451..b8df4a5ea09 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.ts @@ -20,6 +20,8 @@ import { Component } from "@angular/core"; import { NgFor, NgIf } from "@angular/common"; import { FormsModule } from "@angular/forms"; import { NzModalModule, NzModalService } from "ng-zorro-antd/modal"; +import { NzButtonModule } from "ng-zorro-antd/button"; +import { NzInputModule } from "ng-zorro-antd/input"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { v4 as uuid } from "uuid"; import { MacroService } from "../../../service/macro/macro.service"; @@ -27,208 +29,30 @@ import { WorkflowActionService } from "../../../service/workflow-graph/model/wor import { OperatorMetadataService } from "../../../service/operator-metadata/operator-metadata.service"; import { OperatorLabelComponent } from "./operator-label/operator-label.component"; import { NotificationService } from "../../../../common/service/notification/notification.service"; -import { isMacro, MacroPort, macroType } from "../../../../common/util/workflow-macro"; +import { isMacro, MacroDefinition, MacroPort, macroType } from "../../../../common/util/workflow-macro"; +import { FormFieldBinding } from "../../../../common/type/workflow"; +import { LogicalPort } from "../../../types/workflow-common.interface"; import { MacroInspectorComponent } from "./macro-inspector.component"; @UntilDestroy() @Component({ selector: "texera-macro-panel", - imports: [NgIf, NgFor, FormsModule, NzModalModule, OperatorLabelComponent], - template: `
- Macros - -
- -
- - - - - - -
-
- Macro interface -

Add this definition to a workflow to run it.

-
- {{ direction === "inputs" ? "Inputs" : "Outputs" }} -
- - {{ targetLabel(direction, target) }} - - -
-
- - - - - Properties -
- - -
- - -
`, - styles: [ - ` - :host { - display: block; - padding: 10px; - } - summary { - cursor: pointer; - font-weight: 600; - } - input, - select { - width: 100%; - margin: 4px 0; - padding: 4px 7px; - } - input, - select, - button { - max-width: 100%; - font: inherit; - border: 1px solid #d9d9d9; - border-radius: 3px; - background: white; - } - button { - margin: 4px 4px 4px 0; - padding: 3px 7px; - cursor: pointer; - color: #1677ff; - white-space: normal; - } - button:disabled, - input:disabled, - select:disabled { - color: #999; - background: #fafafa; - cursor: default; - } - section { - margin-top: 12px; - } - .properties-heading { - display: block; - margin-top: 12px; - } - section span { - display: block; - font-size: 12px; - color: #666; - overflow-wrap: anywhere; - } - `, - ], + imports: [NgIf, NgFor, FormsModule, NzModalModule, NzButtonModule, NzInputModule, OperatorLabelComponent], + templateUrl: "./macro-panel.component.html", + styleUrls: ["./macro.component.scss"], }) export class MacroPanelComponent { public name = ""; public busy = false; - public inputTarget = ""; - public outputTarget = ""; + public portTargets = { inputs: "", outputs: "" }; public propertyTarget = ""; public readonly directions = ["inputs", "outputs"] as const; public readonly type = macroType; + public readonly trackDefinition = (_: number, d: MacroDefinition) => macroType(d); + public readonly trackPort = (_: number, p: MacroPort) => p.portID; + public readonly trackField = (_: number, f: FormFieldBinding) => f.id; + public readonly trackOption = (_: number, p: { value: string }) => p.value; + public readonly trackTarget = (_: number, p: LogicalPort) => JSON.stringify([p.operatorID, p.portID]); public get canModify(): boolean { return this.actions.checkWorkflowModificationEnabled(); } @@ -239,6 +63,11 @@ export class MacroPanelComponent { const op = this.selected.length === 1 ? this.actions.getTexeraGraph().getOperator(this.selected[0]) : undefined; return !!op && isMacro(op); } + public get selectionHint(): string { + if (!this.canModify) return "Read-only workflow"; + const count = this.selected.length; + return count ? `${count} operator${count === 1 ? "" : "s"} selected` : "Select operators on the canvas"; + } constructor( public macros: MacroService, public actions: WorkflowActionService, @@ -257,7 +86,11 @@ export class MacroPanelComponent { await this.perform(() => this.macros.refresh()); } public async create(): Promise { - await this.perform(() => this.macros.createFromSelection(this.name)); + await this.perform(async () => { + await this.macros.createFromSelection(this.name); + this.name = ""; + this.notification.success("Macro created"); + }); } public async update(): Promise { await this.perform(() => this.macros.adoptLatest(this.selected[0])); @@ -287,6 +120,7 @@ export class MacroPanelComponent { nzTitle: "Macro definition", nzContent: MacroInspectorComponent, nzWidth: 900, + nzStyle: { maxWidth: "calc(100vw - 32px)" }, nzFooter: null, nzData: { definition, dependencies: content.macroDependencies, library: this.macros.library$.value }, }); @@ -303,7 +137,7 @@ export class MacroPanelComponent { })) ); } - public targetLabel(direction: "inputs" | "outputs", target: { operatorID: string; portID: string }): string { + public targetLabel(direction: "inputs" | "outputs", target: LogicalPort): string { return this.targets(direction).find(p => p.value === JSON.stringify(target))?.label ?? "Missing target"; } public properties(): { value: string; label: string; title: string }[] { @@ -331,63 +165,47 @@ export class MacroPanelComponent { const target = JSON.parse(selected.value); const binding = this.actions.getFormBinding(); if (binding.fields.some(f => f.operatorID === target.operatorID && f.propertyKey === target.propertyKey)) return; - this.actions.setFormBinding({ - ...binding, - fields: [...binding.fields, { id: uuid(), ...target, displayName: selected.title }], - }); + this.updateFields(fields => [...fields, { id: uuid(), ...target, displayName: selected.title }]); } public renameProperty(id: string, displayName: string): void { - if (!this.canModify) return; - const binding = this.actions.getFormBinding(); - this.actions.setFormBinding({ - ...binding, - fields: binding.fields.map(f => (f.id === id ? { ...f, displayName } : f)), - }); + this.updateFields(fields => fields.map(f => (f.id === id ? { ...f, displayName } : f))); } public removeProperty(id: string): void { + this.updateFields(fields => fields.filter(f => f.id !== id)); + } + private updateFields(update: (fields: FormFieldBinding[]) => FormFieldBinding[]): void { if (!this.canModify) return; const binding = this.actions.getFormBinding(); - this.actions.setFormBinding({ ...binding, fields: binding.fields.filter(f => f.id !== id) }); + this.actions.setFormBinding({ ...binding, fields: update(binding.fields) }); } public add(direction: "inputs" | "outputs", target: string): void { if (!this.canModify || !this.targets(direction).some(p => p.value === target)) return; - const boundary = this.actions.getMacroInterface()!; - this.actions.setMacroInterface({ - ...boundary, - [direction]: [ - ...boundary[direction], - { - portID: `${direction === "inputs" ? "input" : "output"}-${uuid()}`, - displayName: direction === "inputs" ? "Input" : "Output", - targets: [JSON.parse(target)], - }, - ], - }); + this.updatePorts(direction, ports => [ + ...ports, + { + portID: `${direction === "inputs" ? "input" : "output"}-${uuid()}`, + displayName: direction === "inputs" ? "Input" : "Output", + targets: [JSON.parse(target)], + }, + ]); } public remove(direction: "inputs" | "outputs", port: MacroPort): void { - if (!this.canModify) return; - const boundary = this.actions.getMacroInterface()!; - this.actions.setMacroInterface({ - ...boundary, - [direction]: boundary[direction].filter(p => p.portID !== port.portID), - }); + this.updatePorts(direction, ports => ports.filter(p => p.portID !== port.portID)); } public rename(direction: "inputs" | "outputs", port: MacroPort, displayName: string): void { - if (!this.canModify) return; - const boundary = this.actions.getMacroInterface()!; - this.actions.setMacroInterface({ - ...boundary, - [direction]: boundary[direction].map(p => (p.portID === port.portID ? { ...p, displayName } : p)), - }); + this.updatePorts(direction, ports => ports.map(p => (p.portID === port.portID ? { ...p, displayName } : p))); } public addTarget(port: MacroPort): void { - if (!this.canModify || !this.targets("inputs").some(p => p.value === this.inputTarget)) return; - const target = JSON.parse(this.inputTarget); - const boundary = this.actions.getMacroInterface()!; + if (!this.canModify || !this.targets("inputs").some(p => p.value === this.portTargets.inputs)) return; + const target = JSON.parse(this.portTargets.inputs); if (port.targets.some(p => p.operatorID === target.operatorID && p.portID === target.portID)) return; - this.actions.setMacroInterface({ - ...boundary, - inputs: boundary.inputs.map(p => (p.portID === port.portID ? { ...p, targets: [...p.targets, target] } : p)), - }); + this.updatePorts("inputs", ports => + ports.map(p => (p.portID === port.portID ? { ...p, targets: [...p.targets, target] } : p)) + ); + } + private updatePorts(direction: "inputs" | "outputs", update: (ports: MacroPort[]) => MacroPort[]): void { + if (!this.canModify) return; + const boundary = this.actions.getMacroInterface(); + if (boundary) this.actions.setMacroInterface({ ...boundary, [direction]: update(boundary[direction]) }); } } diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss b/frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss new file mode 100644 index 00000000000..f2ee8b1e427 --- /dev/null +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro.component.scss @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +:host { + display: block; + min-width: 0; +} +.macro-library, +.macro-interface { + padding: 12px; + border-bottom: 1px solid #f0f0f0; +} +summary { + cursor: pointer; + font-weight: 600; +} +summary .badge { + float: right; +} +.badge { + display: inline-block; + padding: 1px 8px; + border-radius: 12px; + background: #f0f5ff; + color: #597ef7; + font-size: 12px; + font-weight: 400; + white-space: nowrap; +} +.section-header, +.actions { + display: flex; + align-items: center; + gap: 8px; +} +.section-header { + justify-content: space-between; +} +.actions { + flex-wrap: wrap; +} +.content-section { + margin-top: 16px; + padding-top: 12px; + border-top: 1px solid #f0f0f0; +} +.muted, +.empty-state, +.target { + color: #8c8c8c; + font-size: 12px; + line-height: 1.6; +} +p { + margin: 8px 0; +} +.empty-state { + padding: 12px; + border: 1px dashed #d9d9d9; + border-radius: 6px; + background: #fafafa; +} +.library-entry { + padding: 0 8px; + margin: 6px 0; + border: 1px solid #e6e9f0; + border-radius: 6px; + cursor: grab; +} +.library-entry:hover { + border-color: #91caff; + background: #f0f5ff; +} +label { + display: block; + font-weight: 600; +} +input, +select { + width: 100%; + min-width: 0; + margin: 8px 0; +} +form button { + margin-top: 4px; +} +.mapping { + margin: 8px 0; + padding: 8px; + background: #fafafa; + border: 1px solid #f0f0f0; + border-radius: 6px; +} +.target { + display: block; + margin-bottom: 8px; + overflow-wrap: anywhere; +} +.inspector-header { + align-items: flex-start; + flex-wrap: wrap; +} +h3 { + margin: 0 0 6px; + font-size: 20px; + overflow-wrap: anywhere; +} +.graph-summary { + color: #595959; + font-size: 12px; + margin: 12px 0; +} +.graph-summary span + span { + border-left: 1px solid #d9d9d9; + padding-left: 8px; +} +.macro-graph { + height: 380px; + overflow: hidden; + border: 1px solid #e6e9f0; + border-radius: 8px; + background: radial-gradient(#e0e5ed 1px, transparent 1px) 0 0 / 16px 16px #fafbfc; +} +.operator-properties { + padding: 10px 12px; + margin: 8px 0; + border: 1px solid #f0f0f0; + border-radius: 6px; +} +pre { + margin: 12px 0 0; + max-height: 180px; + overflow: auto; + font-size: 12px; +} diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts index 3b419e683a3..83e51ec854d 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.spec.ts @@ -1920,6 +1920,19 @@ describe("WorkflowEditorComponent editor wiring", () => { }); describe("region overlays", () => { + it("handles generated operators absent from the canvas and empty regions", fakeAsync(() => { + addRegions([[1, ["visible"]]]); + (executeWorkflowService as any).regionUpdateStream.next({ + regions: [ + [1, ["visible", "generated"]], + [2, []], + ], + }); + tick(); + expect(component.paper.getModelById("region-1").attr("body/d")).toBeTruthy(); + expect(component.paper.getModelById("region-2").attr("body/d")).toBeFalsy(); + })); + it("reshapes the region drawn around an operator that moved", () => { // Two regions so the handler's filter is exercised on both sides. Nothing can be asserted // about the untouched one: its outline is recomputed from its own, unmoved operator, so a diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts index 1fefcbe7f3d..f175e5f04ed 100644 --- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts +++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts @@ -675,7 +675,7 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy regionMap = event.regions.map(([id, region]) => { const element = new Region({ id: "region-" + id }); - const ops = region.map(id => this.paper.getModelById(id)); + const ops = region.map(id => this.paper.getModelById(id)).filter(Boolean); this.paper.model.addCell(element); this.updateRegionElement(element, ops); return { regionElement: element, operators: ops }; @@ -728,7 +728,10 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy [x + width + padding, y + height + padding + 10], ]; }); - regionElement.attr("body/d", line().curve(curveCatmullRomClosed)(concaveman(points, 2, 0) as [number, number][])); + regionElement.attr( + "body/d", + points.length ? line().curve(curveCatmullRomClosed)(concaveman(points, 2, 0) as [number, number][]) : null + ); } /** diff --git a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts index b55ba59b36e..da028b163ae 100644 --- a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts +++ b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.spec.ts @@ -488,6 +488,28 @@ describe("ExecuteWorkflowService", () => { expect(service.getWorkerIds("opX")).toEqual(["w1", "w2"]); }); + it("draws execution regions around the outer macro once while preserving ordinary and empty regions", () => { + const actions = TestBed.inject(WorkflowActionService); + vi.spyOn(actions, "getMacroExpansion").mockReturnValue({ + content: actions.getWorkflowContent(), + paths: { leafA: ["macro", "nested", "a"], leafB: ["macro", "nested", "b"] }, + ports: {}, + }); + const updates: RegionUpdateEvent[] = []; + service.getRegionUpdateStream().subscribe(event => updates.push(event)); + emitWsEvent({ + type: "RegionUpdateEvent", + regions: [ + [1, ["source", "leafA", "leafB"]], + [2, []], + ], + }); + expect(updates[0].regions).toEqual([ + [1, ["source", "macro"]], + [2, []], + ]); + }); + it("routes execution-status websocket events into the execution-state stream", () => { const states: ExecutionState[] = []; service.getExecutionStateStream().subscribe(event => states.push(event.current.state)); diff --git a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts index 003f38e0778..d433a4ce5e5 100644 --- a/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts +++ b/frontend/src/app/workspace/service/execute-workflow/execute-workflow.service.ts @@ -18,7 +18,7 @@ */ import { Inject, Injectable, DOCUMENT } from "@angular/core"; -import { BehaviorSubject, interval, Observable, Subject, Subscription } from "rxjs"; +import { BehaviorSubject, interval, map, Observable, Subject, Subscription } from "rxjs"; import { WorkflowActionService } from "../workflow-graph/model/workflow-action.service"; import { WorkflowGraphReadonly } from "../workflow-graph/model/workflow-graph"; import { @@ -497,7 +497,18 @@ export class ExecuteWorkflowService { } public getRegionUpdateStream(): Observable { - return this.regionUpdateStream.asObservable(); + return this.regionUpdateStream.pipe( + map(event => { + const paths = this.workflowActionService.getMacroExpansion?.()?.paths ?? {}; + return { + ...event, + regions: event.regions.map(([id, ops]): [number, string[]] => [ + id, + [...new Set(ops.map(op => paths[op]?.[0] ?? op))], + ]), + }; + }) + ); } public getRegionStateStream(): Observable { diff --git a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts index 72e1902acf5..001a5d2212a 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.spec.ts @@ -98,6 +98,25 @@ describe("SharedModelChangeHandler", () => { } describe("operator add and delete", () => { + it("updates a replaced operator in one transaction without losing its links", () => { + addOperatorWithPosition(mockScanPredicate); + addOperatorWithPosition(mockResultPredicate); + texeraGraph.addLink(mockScanResultLink); + const link = jointGraph.getCell(mockScanResultLink.linkID); + const updated = { ...mockSentimentPredicate, operatorID: mockScanPredicate.operatorID }; + const added: string[] = []; + const subscription = texeraGraph.getOperatorAddStream().subscribe(op => added.push(op.operatorID)); + texeraGraph.bundleActions(() => { + texeraGraph.deleteOperator(updated.operatorID); + texeraGraph.addOperator(updated); + }); + expect((jointGraph.getCell(updated.operatorID) as joint.dia.Element).getPorts().length).toBe( + updated.inputPorts.length + updated.outputPorts.length + ); + expect(jointGraph.getCell(mockScanResultLink.linkID)).toBe(link); + expect(added).toEqual([updated.operatorID]); + subscription.unsubscribe(); + }); it("should emit operatorAddSubject and add a joint element when an operator is added", () => { let added: OperatorPredicate | undefined; const sub = texeraGraph.getOperatorAddStream().subscribe(op => (added = op)); diff --git a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts index e5eab7a812e..8009045fdfc 100644 --- a/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts +++ b/frontend/src/app/workspace/service/workflow-graph/model/shared-model-change-handler.ts @@ -81,14 +81,23 @@ export class SharedModelChangeHandler { const jointElementsToAdd: joint.dia.Element[] = []; const newOpIDs: string[] = []; event.changes.keys.forEach((change, key) => { - if (change.action === "add") { + if (change.action === "add" || change.action === "update") { const newOperator = this.texeraGraph.sharedModel.operatorIDMap.get(key) as YType; // Also find its position if (this.texeraGraph.sharedModel.elementPositionMap?.has(key)) { const newPos = this.texeraGraph.sharedModel.elementPositionMap?.get(key) as Point; // Add the operator into joint graph const jointOperator = this.jointUIService.getJointOperatorElement(newOperator.toJSON(), newPos); - jointElementsToAdd.push(jointOperator); + if (change.action === "update") { + this.texeraGraph.setSyncTexeraGraph(false); + try { + this.jointGraph.getCell(key).set(jointOperator.attributes); + } finally { + this.texeraGraph.setSyncTexeraGraph(true); + } + } else { + jointElementsToAdd.push(jointOperator); + } newOpIDs.push(key); } else { throw new Error(`operator with key ${key} does not exist in position map`); From f0a891028cd54dcb6ce82ca7549400c1e41bb965 Mon Sep 17 00:00:00 2001 From: carloea2 Date: Wed, 14 Oct 2026 17:17:13 -0600 Subject: [PATCH 04/11] test(frontend): cover nested macros in form view Related to #8913 --- .../form-binding/form-binding.service.spec.ts | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts b/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts index cd32c277957..d2721615e08 100644 --- a/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts +++ b/frontend/src/app/workspace/service/form-binding/form-binding.service.spec.ts @@ -27,6 +27,7 @@ import { OperatorMetadataService } from "../operator-metadata/operator-metadata. import { StubOperatorMetadataService } from "../operator-metadata/stub-operator-metadata.service"; import { commonTestProviders } from "../../../common/testing/test-utils"; import { mockPoint, mockScanPredicate, mockSentimentPredicate } from "../workflow-graph/model/mock-workflow-data"; +import { expandMacros, MacroDefinition, macroInstance } from "../../../common/util/workflow-macro"; describe("FormBindingService", () => { let service: FormBindingService; @@ -51,6 +52,65 @@ describe("FormBindingService", () => { const scanId = mockScanPredicate.operatorID; + it("resolves and edits three nested macro levels without changing another instance or the pinned definitions", () => { + const definitions: MacroDefinition[] = []; + let inner = { ...mockScanPredicate, operatorID: "leaf", operatorProperties: { tableName: "initial" } }; + for (let level = 0; level < 3; level++) { + const id = `form-level-${level}`; + const definition: MacroDefinition = { + id, + version: 1, + name: `Level ${level}`, + content: { + ...workflowActionService.getWorkflowContent(), + operators: [inner], + links: [], + macroInterface: { + id, + inputs: [], + outputs: [{ portID: "output-0", targets: [{ operatorID: inner.operatorID, portID: "output-0" }] }], + }, + formBinding: { + fields: [ + { + id: "exposed", + operatorID: inner.operatorID, + propertyKey: level ? "exposed" : "tableName", + displayName: "Table", + }, + ], + }, + }, + }; + definitions.push(definition); + inner = macroInstance(definition, `nested-${level}`) as typeof inner; + } + workflowActionService.setMacroDependencies(definitions); + for (const id of ["first", "second"]) { + workflowActionService.addOperator( + { ...inner, operatorID: id, operatorProperties: { exposed: "initial" } }, + mockPoint + ); + service.addBinding(id, "exposed"); + } + const before = JSON.stringify(definitions); + const [first, second] = service.resolveFields(); + expect(first.schema?.title).toBe("Table"); + expect(first.brokenReason).toBeUndefined(); + for (const value of ["東京 café", ""]) { + service.writeValue(first.binding, value); + expect(service.resolveFields().map(f => f.value)).toEqual([value, "initial"]); + const expanded = expandMacros(workflowActionService.getWorkflowContent()); + const leaf = (id: string) => expanded.content.operators.find(op => expanded.paths[op.operatorID]?.[0] === id)!; + expect(leaf("first").operatorProperties["tableName"]).toBe(value); + expect(leaf("second").operatorProperties["tableName"]).toBe("initial"); + } + expect(JSON.stringify(definitions)).toBe(before); + service.updateBinding(first.binding.id, { propertyKey: "missing" }); + expect(service.resolveFields()[0].brokenReason).toContain("not a setting"); + expect(service.readValue(second.binding.operatorID, second.binding.propertyKey)).toBe("initial"); + }); + it("starts with nothing exposed and no result list, so the final steps show", () => { expect(service.getConfig().fields).toEqual([]); expect(service.getConfig().shownResultIds).toBeUndefined(); From dc5cf916f43043c810e7fe53af2d51df9a3d9dfb Mon Sep 17 00:00:00 2001 From: carloea2 Date: Wed, 14 Oct 2026 18:21:25 -0600 Subject: [PATCH 05/11] fix(frontend, amber): harden nested macro composition --- .../user/workflow/WorkflowResource.scala | 43 ++++--- .../workflow/WorkflowMacroCloneSpec.scala | 24 ++++ .../app/common/util/workflow-macro.spec.ts | 56 +++++++++ .../src/app/common/util/workflow-macro.ts | 73 ++++++------ .../operator-menu/macro-panel.component.html | 4 +- .../macro-panel.component.spec.ts | 17 ++- .../operator-menu/macro-panel.component.ts | 62 ++++------ .../execute-workflow.service.spec.ts | 42 ++++++- .../execute-workflow.service.ts | 17 +-- .../service/joint-ui/joint-ui.service.spec.ts | 19 +++ .../service/joint-ui/joint-ui.service.ts | 4 +- .../service/macro/macro.service.spec.ts | 79 +++++++++++++ .../workspace/service/macro/macro.service.ts | 110 ++++++++++++------ .../operator-menu/operator-menu.service.ts | 13 +-- .../operator-metadata.service.spec.ts | 46 ++++++++ .../operator-metadata.service.ts | 8 +- .../model/workflow-action.service.ts | 6 +- 17 files changed, 466 insertions(+), 157 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala index d3901548aa4..1bdc0da7e3a 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowResource.scala @@ -19,8 +19,8 @@ package org.apache.texera.web.resource.dashboard.user.workflow -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.typesafe.scalalogging.LazyLogging import io.dropwizard.auth.Auth @@ -247,8 +247,6 @@ object WorkflowResource { def assignNewOperatorIds(workflowContent: String): String = { val objectMapper = new ObjectMapper().registerModule(DefaultScalaModule) val canvas = objectMapper.readTree(workflowContent).asInstanceOf[ObjectNode] - // Pinned dependency graphs keep their definition IDs, even when a canvas ID is identical. - val dependencies = canvas.remove("macroDependencies") val operators = Option(canvas.get("operators")) .filter(_.isArray) .getOrElse(throw new IllegalArgumentException("Missing operators array")) @@ -261,22 +259,41 @@ object WorkflowResource { val operatorType = operator.get("operatorType").asText() // operator id in frontend: operatorSchema.operatorType + "-operator-" + uuid(); // v4 = UUID.randomUUID().toString val newOperatorId = s"$operatorType-operator-${UUID.randomUUID()}" + operator.asInstanceOf[ObjectNode].put("operatorID", newOperatorId) oldOperatorId -> newOperatorId }) .toMap - // replace all old operator ids with new operator ids - val rewritten = operatorIdMap.foldLeft(canvas.toString) { - case (updatedContent, (oldId, newId)) => - updatedContent.replace(oldId, newId) + // Only graph references change; pinned snapshots and user values stay untouched. + Seq("/links", "/formBinding/fields", "/macroInterface/inputs", "/macroInterface/outputs") + .foreach { path => + canvas.at(path).findParents("operatorID").asScala.foreach { node => + val ref = node.asInstanceOf[ObjectNode] + operatorIdMap + .get(ref.path("operatorID").asText()) + .foreach(id => ref.put("operatorID", id)) + } + } + Option(canvas.get("operatorPositions")).collect { case p: ObjectNode => p }.foreach { + positions => + operatorIdMap.foreach { + case (oldId, newId) => + Option(positions.remove(oldId)).foreach(value => positions.set[JsonNode](newId, value)) + } + } + val shown = canvas.at("/formBinding/shownResultIds") + shown.elements().asScala.zipWithIndex.foreach { + case (entry, index) => + operatorIdMap + .get(entry.asText()) + .foreach(id => + shown.asInstanceOf[ArrayNode].set(index, objectMapper.getNodeFactory.textNode(id)) + ) } - val result = objectMapper.readTree(rewritten).asInstanceOf[ObjectNode] - if (dependencies != null) - result.set[com.fasterxml.jackson.databind.JsonNode]("macroDependencies", dependencies) - Option(result.get("macroInterface")).filter(_.isObject).foreach { boundary => + Option(canvas.get("macroInterface")).filter(_.isObject).foreach { boundary => boundary.asInstanceOf[ObjectNode].put("id", UUID.randomUUID().toString) } - result.toString + canvas.toString } def baseWorkflowSelect(): SelectOnConditionStep[Record10[ diff --git a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala index c77d1df9142..d645ea687a2 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/dashboard/user/workflow/WorkflowMacroCloneSpec.scala @@ -53,4 +53,28 @@ class WorkflowMacroCloneSpec extends AnyFlatSpec { intercept[Exception](WorkflowResource.assignNewOperatorIds("{}")) intercept[Exception](WorkflowResource.assignNewOperatorIds("invalid")) } + + it should "rewrite only graph references when IDs occur in macro types and user values" in { + val content = mapper.readTree("""{ + "operators":[{"operatorID":"leaf","operatorType":"WorkflowMacro:leaf-definition:8","operatorProperties":{"code":"leaf + 1","value":"leaf"},"customDisplayName":"leaf"}], + "operatorPositions":{"leaf":{"x":0,"y":0}}, + "links":[{"source":{"operatorID":"leaf","portID":"output-0"},"target":{"operatorID":"external","portID":"input-0"}}], + "formBinding":{"fields":[{"id":"leaf-field","operatorID":"leaf","propertyKey":"leaf-key"}],"shownResultIds":["leaf"]}, + "macroDependencies":[{"id":"leaf-definition","version":8,"content":{"operators":[]}}] + }""") + val cloned = mapper.readTree(WorkflowResource.assignNewOperatorIds(content.toString)) + val id = cloned.at("/operators/0/operatorID").asText() + assert(id != "leaf") + assert(cloned.at("/operators/0/operatorType") == content.at("/operators/0/operatorType")) + assert( + cloned.at("/operators/0/operatorProperties") == content.at("/operators/0/operatorProperties") + ) + assert(cloned.at("/operators/0/customDisplayName").asText() == "leaf") + assert(cloned.at("/formBinding/fields/0/id").asText() == "leaf-field") + assert(cloned.at("/formBinding/fields/0/propertyKey").asText() == "leaf-key") + assert(cloned.at("/formBinding/shownResultIds/0").asText() == id) + assert(cloned.at("/links/0/source/operatorID").asText() == id) + assert(cloned.at("/links/0/target/operatorID").asText() == "external") + assert(cloned.path("macroDependencies") == content.path("macroDependencies")) + } } diff --git a/frontend/src/app/common/util/workflow-macro.spec.ts b/frontend/src/app/common/util/workflow-macro.spec.ts index af0a03448f2..8e69a4c88ad 100644 --- a/frontend/src/app/common/util/workflow-macro.spec.ts +++ b/frontend/src/app/common/util/workflow-macro.spec.ts @@ -100,6 +100,19 @@ describe("workflow macros", () => { expect(() => expandMacros(uploaded)).toThrow(/missing.*version/i); }); + it("deduplicates equivalent definitions regardless of JSON key order but rejects differing content", () => { + const d = definition(); + const equivalent = { ...d, content: Object.fromEntries(Object.entries(d.content).reverse()) as WorkflowContent }; + const content = { ...graph([macroInstance(d, "m")]), macroDependencies: [d, equivalent] }; + expect(() => expandMacros(content)).not.toThrow(); + const conflicting = structuredClone(d); + conflicting.content.operators[0] = { + ...conflicting.content.operators[0], + operatorProperties: { value: "different" }, + }; + expect(() => expandMacros({ ...content, macroDependencies: [d, conflicting] })).toThrow(/conflicting/i); + }); + it("expands repeated pinned instances with distinct, deterministic short IDs and local overrides", () => { const d = definition(); const first = { ...macroInstance(d, "first"), operatorProperties: { value: "override" } }; @@ -176,6 +189,7 @@ describe("workflow macros", () => { null, { ...d, content: null }, { ...d, version: 0 }, + { ...d, name: 42 }, { ...d, content: { ...d.content, macroInterface: null } }, ]) { expect(() => expandMacros({ ...root, macroDependencies: [invalid] } as unknown as WorkflowContent)).toThrow(); @@ -218,6 +232,21 @@ describe("workflow macros", () => { ).toBe("filled"); }); + it("maps exposed attribute pickers to the macro's boundary port instead of the inner port ordinal", () => { + const d = definition(); + d.content.operators[0].inputPorts.push({ portID: "input-1" }); + d.content.macroInterface.inputs = [ + { portID: "input-0", displayName: "Right", targets: [{ operatorID: "a", portID: "input-1" }] }, + { portID: "input-1", displayName: "Left", targets: [{ operatorID: "a", portID: "input-0" }] }, + ]; + const schema = { + jsonSchema: { properties: { value: { type: "string", autofill: "attributeName", autofillAttributeOnPort: 1 } } }, + } as unknown as OperatorSchema; + expect((macroSchema(d, () => schema).jsonSchema.properties!.value as any).autofillAttributeOnPort).toBe(0); + d.content.macroInterface.inputs = []; + expect((macroSchema(d, () => schema).jsonSchema.properties!.value as any).autofillAttributeOnPort).toBe(-1); + }); + it("extracts a connected selection, preserves boundary links and Form View bindings", () => { const content = { ...graph([op("s"), op("a"), op("b"), op("t")], [link("s", "a"), link("a", "b"), link("b", "t")]), @@ -247,6 +276,33 @@ describe("workflow macros", () => { expect(() => adoptMacroVersion(instance, old, next)).toThrow(/interface/i); }); + it("accepts property and port reordering when the exposed identities are unchanged", () => { + const previous = definition(); + previous.content.formBinding.fields.push({ + id: "second", + operatorID: "b", + propertyKey: "value", + displayName: "Second", + }); + previous.content.macroInterface.inputs.push({ + portID: "input-1", + displayName: "Other", + targets: [{ operatorID: "b", portID: "input-0" }], + }); + const next = structuredClone(previous); + next.version++; + next.content.formBinding.fields.reverse(); + next.content.macroInterface.inputs.reverse(); + const instance = { ...macroInstance(previous, "m"), operatorProperties: { value: "mine" } }; + expect(adoptMacroVersion(instance, previous, next).operatorProperties).toEqual({ value: "mine" }); + }); + + it("rejects duplicate boundary targets even when their JSON keys are ordered differently", () => { + const d = definition(); + d.content.macroInterface.inputs[0].targets.push({ portID: "input-0", operatorID: "a" }); + expect(() => expandMacros(d.content)).toThrow(/duplicate.*target/i); + }); + it("omits disabled instances and their links without resolving dependencies", () => { const disabled = { ...macroInstance(definition(), "m"), isDisabled: true }; const expanded = expandMacros(graph([op("s"), disabled], [link("s", "m")])); diff --git a/frontend/src/app/common/util/workflow-macro.ts b/frontend/src/app/common/util/workflow-macro.ts index 8b69e0e5208..cc6129276d1 100644 --- a/frontend/src/app/common/util/workflow-macro.ts +++ b/frontend/src/app/common/util/workflow-macro.ts @@ -18,7 +18,7 @@ */ import { v5 as uuid } from "uuid"; -import { cloneDeep } from "lodash-es"; +import { cloneDeep, groupBy, isEqual } from "lodash-es"; import { WorkflowContent, FormFieldBinding } from "../type/workflow"; import { LogicalPort, OperatorPredicate, PortDescription } from "../../workspace/types/workflow-common.interface"; import { OperatorSchema } from "../../workspace/types/operator-schema.interface"; @@ -73,20 +73,27 @@ export interface MacroExpansion { ports: Record; } -/** A pure, deterministic expansion used by both compilers and executions. Definitions stay immutable. */ -export function expandMacros(content: WorkflowContent): MacroExpansion { +/** One immutable snapshot per identity/version, shared by imports, libraries and clipboard operations. */ +export function mergeMacroDefinitions(...bundles: MacroDefinition[][]): MacroDefinition[] { const definitions = new Map(); - for (const d of content.macroDependencies ?? []) { + for (const d of bundles.flat()) { require(d && typeof d.id === "string" && + typeof d.name === "string" && d.id.length && Number.isSafeInteger(d.version) && d.version > 0 && d.content?.macroInterface?.id === d.id, "Malformed macro definition"); const previous = definitions.get(macroType(d)); - require(!previous || JSON.stringify(previous.content) === JSON.stringify(d.content), "Conflicting macro version"); + require(!previous || isEqual(previous.content, d.content), "Conflicting macro version"); definitions.set(macroType(d), d); } + return [...definitions.values()]; +} + +/** A pure, deterministic expansion used by both compilers and executions. Definitions stay immutable. */ +export function expandMacros(content: WorkflowContent): MacroExpansion { + const definitions = new Map(mergeMacroDefinitions(content.macroDependencies ?? []).map(d => [macroType(d), d])); const paths: Record = Object.create(null); const ports: Record = Object.create(null); const operators: OperatorPredicate[] = []; @@ -131,7 +138,7 @@ export function expandMacros(content: WorkflowContent): MacroExpansion { for (const op of body.operators.filter(op => !op.isDisabled)) { if (!isMacro(op)) { const id = runtimeID(chain, op.operatorID); - require(!operators.some(o => o.operatorID === id), "Duplicate runtime operator ID"); + require(!Object.hasOwn(paths, id), "Duplicate runtime operator ID"); operators.push({ ...cloneDeep(op), operatorID: id }); paths[id] = [...chain, op.operatorID]; for (const direction of ["inputs", "outputs"] as const) { @@ -163,26 +170,15 @@ export function expandMacros(content: WorkflowContent): MacroExpansion { } const innerPorts = visit(inner, [...chain, op.operatorID], [...stack, op.operatorType]); for (const direction of ["inputs", "outputs"] as const) { - const boundary = inner.macroInterface?.[direction]; - require(Array.isArray(boundary), "Malformed macro interface"); + const boundary = inner.macroInterface![direction]; const declared = direction === "inputs" ? op.inputPorts : op.outputPorts; require(declared.length === boundary!.length && declared.every(p => boundary!.some(b => b.portID === p.portID) ), "Macro instance interface differs from its pinned version"); - require(new Set(boundary!.map(p => p.portID)).size === boundary!.length, "Duplicate macro port"); for (const p of boundary!) { - require(Array.isArray(p.targets) && - p.targets.length > 0 && - (direction === "inputs" || - p.targets.length === 1), "An output must map to one output port; inputs must have targets"); - const targets = p.targets.flatMap(t => { - const mapped = - innerPorts.get(portKey(t.operatorID, direction, t.portID)) ?? - (inner.operators.some(o => o.operatorID === t.operatorID && o.isDisabled) ? [] : undefined); - require(mapped, "Missing mapped macro port"); - return mapped!; - }); + // visit already validated each boundary target; disabled operators have no runtime port. + const targets = p.targets.flatMap(t => innerPorts.get(portKey(t.operatorID, direction, t.portID)) ?? []); resolved.set(portKey(op.operatorID, direction, p.portID), targets); if (!chain.length) ports[portKey(op.operatorID, direction, p.portID)] = targets; } @@ -201,7 +197,8 @@ export function expandMacros(content: WorkflowContent): MacroExpansion { p.targets.length && (direction === "inputs" || p.targets.length === 1), "An output must map to one output port; inputs must have targets"); - require(new Set(p.targets.map(t => JSON.stringify(t))).size === p.targets.length, "Duplicate macro target"); + require(new Set(p.targets.map(t => portKey(t?.operatorID, direction, t?.portID))).size === + p.targets.length, "Duplicate macro target"); p.targets.forEach(t => endpoint(t, direction)); }); } @@ -234,19 +231,14 @@ export function expandMacros(content: WorkflowContent): MacroExpansion { return resolved; }; visit(content, [], []); - const children = new Map( - operators.map(op => [ - op.operatorID, - links.filter(l => l.source.operatorID === op.operatorID).map(l => l.target.operatorID), - ]) - ); + const children = groupBy(links, l => l.source.operatorID); const visiting = new Set(), visited = new Set(); const checkCycle = (id: string): void => { require(!visiting.has(id), "Workflow contains a cycle"); if (visited.has(id)) return; visiting.add(id); - children.get(id)!.forEach(checkCycle); + if (Object.hasOwn(children, id)) children[id].forEach(link => checkCycle(link.target.operatorID)); visiting.delete(id); visited.add(id); }; @@ -313,9 +305,11 @@ export function adoptMacroVersion( require(previous.id === next.id && op.operatorType === macroType(previous), "Different macro definition"); const signature = (d: MacroDefinition) => JSON.stringify([ - d.content.macroInterface!.inputs.map(p => p.portID), - d.content.macroInterface!.outputs.map(p => p.portID), - fields(d.content).map(f => f.id), + d.content.macroInterface!.inputs.map(p => p.portID).sort(), + d.content.macroInterface!.outputs.map(p => p.portID).sort(), + fields(d.content) + .map(f => f.id) + .sort(), ]); require(signature(previous) === signature(next), "Macro interface changed; insert the new version and reconnect explicitly"); @@ -332,8 +326,8 @@ export function adoptMacroVersion( /** Exposed Form View fields become the macro's ordinary operator properties, including nested macros. */ export function macroSchema(d: MacroDefinition, getSchema: (type: string) => OperatorSchema): OperatorSchema { const properties: Record = Object.create(null); - const resolve = (value: any, root: CustomJSONSchema7, refs: string[] = []): any => { - if (Array.isArray(value)) return value.map(v => resolve(v, root, refs)); + const resolve = (value: any, root: CustomJSONSchema7, op: OperatorPredicate, refs: string[] = []): any => { + if (Array.isArray(value)) return value.map(v => resolve(v, root, op, refs)); if (!value || typeof value !== "object") return value; if (value.$ref) { const { $ref, ...rest } = value; @@ -343,12 +337,19 @@ export function macroSchema(d: MacroDefinition, getSchema: (type: string) => Ope .split("/") .reduce((node: any, key: string) => node?.[key.replace(/~1/g, "/").replace(/~0/g, "~")], root); require(target, "Missing exposed property schema reference"); - return resolve({ ...target, ...rest }, root, [...refs, $ref]); + return resolve({ ...target, ...rest }, root, op, [...refs, $ref]); } return Object.fromEntries( Object.entries(value) .filter(([key]) => key !== "$id") - .map(([key, v]) => [key, resolve(v, root, refs)]) + .map(([key, v]) => [ + key, + key === "autofillAttributeOnPort" && Number.isInteger(v) + ? d.content.macroInterface!.inputs.findIndex(p => + p.targets.some(t => t.operatorID === op.operatorID && t.portID === op.inputPorts[v as number]?.portID) + ) + : resolve(v, root, op, refs), + ]) ); }; for (const f of fields(d.content)) { @@ -358,7 +359,7 @@ export function macroSchema(d: MacroDefinition, getSchema: (type: string) => Ope const schema = root.properties?.[f.propertyKey]; require(schema && typeof schema !== "boolean", "Missing exposed property schema"); properties[f.id] = { - ...resolve(schema, root), + ...resolve(schema, root, op!), title: f.displayName, description: f.helpText, default: cloneDeep( diff --git a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html index 2e3e650e970..7194c1016b1 100644 --- a/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html +++ b/frontend/src/app/workspace/component/left-panel/operator-menu/macro-panel.component.html @@ -188,13 +188,13 @@ nz-input [attr.aria-label]="'Property name ' + field.id" [ngModel]="field.displayName" - (ngModelChange)="renameProperty(field.id, $event)" + (ngModelChange)="canModify && forms.updateBinding(field.id, { displayName: $event })" [disabled]="!canModify" />