From 4e516603aaa3ccde215c97bd9c526a60898a3fe8 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Sat, 19 Sep 2026 11:59:26 -0400 Subject: [PATCH] Add finite Repeat-until workflows (M4C-3) Implement typed state, frozen finite batch policy, durable manual continuation, mixed iteration identities, authorized inspection, and V2 authoring. Preserve the existing exact saved-record export and publication contracts. Bump the application to 0.261.120. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 26 + application/single_app/config.py | 2 +- application/single_app/functions_settings.py | 10 + .../single_app/functions_workflow_editor.py | 14 +- .../functions_workflow_execution_history.py | 59 +- .../single_app/functions_workflow_flow.py | 253 ++++- .../functions_workflow_flow_runner.py | 57 +- .../single_app/functions_workflow_identity.py | 57 +- .../functions_workflow_iterations.py | 126 ++- .../single_app/functions_workflow_journal.py | 198 +++- .../single_app/functions_workflow_limits.py | 43 + .../functions_workflow_loop_runners.py | 6 +- .../functions_workflow_node_results.py | 575 ++++++++--- .../functions_workflow_repeat_execution.py | 265 +++++ .../functions_workflow_repeat_history.py | 163 +++ .../functions_workflow_repeat_state.py | 521 ++++++++++ .../single_app/functions_workflow_results.py | 6 +- .../single_app/functions_workflow_runner.py | 27 +- .../single_app/functions_workflow_runtime.py | 25 +- .../functions_workflow_runtime_store.py | 65 +- ...functions_workflow_structured_execution.py | 22 + .../single_app/route_backend_workflows.py | 58 +- .../route_frontend_admin_settings.py | 13 + .../templates/admin/_panes/workflow.html | 23 + .../workflows/WorkflowConditionEditor.tsx | 173 ++-- .../workflows/WorkflowEditorDialog.tsx | 5 +- .../workflows/WorkflowExecutionHistory.tsx | 208 +++- .../workflows/WorkflowLoopFields.tsx | 35 +- .../workflows/WorkflowRepeatFields.tsx | 236 +++++ .../workflows/WorkflowRepeatProgress.tsx | 24 + .../workflows/WorkflowRuntimePanel.tsx | 102 +- .../workflows/WorkflowStructuredList.tsx | 35 +- application/v2_ui/src/lib/workflowEditor.ts | 154 ++- .../v2_ui/src/lib/workflowExecutionHistory.ts | 187 +++- application/v2_ui/src/lib/workflowFlow.ts | 277 ++++- docs/admin/workflow.md | 31 +- .../features/WORKFLOW_DURABLE_EXECUTION.md | 34 +- .../features/WORKFLOW_FOR_EACH_COLLECT.md | 43 +- .../WORKFLOW_PUBLICATION_COMPLETION.md | 9 + .../features/WORKFLOW_REPEAT_UNTIL.md | 529 ++++++++++ .../features/WORKFLOW_RESULT_READERS.md | 33 + .../WORKFLOW_SAVED_OUTPUT_PUBLICATION.md | 33 +- .../WORKFLOW_STRUCTURED_CONTROL_FLOW.md | 36 +- docs/guides/create-a-workflow.md | 49 +- docs/guides/trigger-a-workflow.md | 36 + .../test_workflow_execution_journal_policy.py | 4 +- .../test_workflow_repeat_policy.py | 127 +++ .../test_v2_admin_workflow_parity.py | 13 +- functional_tests/test_workflow_loop_limits.py | 81 +- functional_tests/test_workflow_loop_schema.py | 11 +- .../test_workflow_repeat_dispatcher.py | 157 +++ .../test_workflow_repeat_editor_options.py | 223 ++++ .../test_workflow_repeat_execution.py | 266 +++++ .../test_workflow_repeat_limits.py | 52 + .../test_workflow_repeat_publication.py | 193 ++++ .../test_workflow_repeat_recovery.py | 511 ++++++++++ .../test_workflow_repeat_schema.py | 456 +++++++++ .../test_workflow_repeat_state.py | 314 ++++++ ui_tests/fixtures/workflow_admin_limits.py | 6 +- ui_tests/fixtures/workflow_repeat_until.py | 544 ++++++++++ ui_tests/test_v2_workflow_loops.py | 7 +- ui_tests/test_v2_workflow_repeat_until.py | 962 ++++++++++++++++++ ui_tests/test_workflow_loop_admin_limits.py | 102 +- 63 files changed, 8296 insertions(+), 616 deletions(-) create mode 100644 application/single_app/functions_workflow_repeat_execution.py create mode 100644 application/single_app/functions_workflow_repeat_history.py create mode 100644 application/single_app/functions_workflow_repeat_state.py create mode 100644 application/v2_ui/src/components/workflows/WorkflowRepeatFields.tsx create mode 100644 application/v2_ui/src/components/workflows/WorkflowRepeatProgress.tsx create mode 100644 docs/explanation/features/WORKFLOW_REPEAT_UNTIL.md create mode 100644 functional_tests/route_tests/test_workflow_repeat_policy.py create mode 100644 functional_tests/test_workflow_repeat_dispatcher.py create mode 100644 functional_tests/test_workflow_repeat_editor_options.py create mode 100644 functional_tests/test_workflow_repeat_execution.py create mode 100644 functional_tests/test_workflow_repeat_limits.py create mode 100644 functional_tests/test_workflow_repeat_publication.py create mode 100644 functional_tests/test_workflow_repeat_recovery.py create mode 100644 functional_tests/test_workflow_repeat_schema.py create mode 100644 functional_tests/test_workflow_repeat_state.py create mode 100644 ui_tests/fixtures/workflow_repeat_until.py create mode 100644 ui_tests/test_v2_workflow_repeat_until.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 153412d83..4425a8419 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -136,8 +136,12 @@ WORKFLOW_LOOP_ITEMS_DEFAULT, WORKFLOW_LOOP_ITEMS_MAX, WORKFLOW_LOOP_ITEMS_MIN, + WORKFLOW_REPEAT_ITERATIONS_DEFAULT, + WORKFLOW_REPEAT_ITERATIONS_MAX, + WORKFLOW_REPEAT_ITERATIONS_MIN, WorkflowLoopLimitError, validate_workflow_max_loop_items, + validate_workflow_max_repeat_iterations, ) HEX_COLOR_PATTERN = re.compile(r"^#[0-9a-fA-F]{6}$") @@ -4008,6 +4012,22 @@ "max": WORKFLOW_LOOP_ITEMS_MAX, "step": 1, }, + { + "key": "workflow_max_repeat_iterations", + "type": "number", + "label": "Workflow Repeat Iteration Limit", + "help": ( + "Maximum rounds allowed in one automatic Repeat until batch in a new " + "personal or group workflow run. Authors must choose a per-block maximum; " + "new runs above this ceiling are rejected, never shortened. Active runs " + "and manual continuation keep their admitted limit. Another batch does " + "not reset the run's execution-admission budget or elapsed deadline." + ), + "default": WORKFLOW_REPEAT_ITERATIONS_DEFAULT, + "min": WORKFLOW_REPEAT_ITERATIONS_MIN, + "max": WORKFLOW_REPEAT_ITERATIONS_MAX, + "step": 1, + }, ], # --- Agents & Actions ------------------------------------------------- # @@ -6582,6 +6602,12 @@ def _normalize_field_value(key, value, field): except WorkflowLoopLimitError as error: return None, error.public_message, None + if key == "workflow_max_repeat_iterations": + try: + return validate_workflow_max_repeat_iterations(value), None, None + except WorkflowLoopLimitError as error: + return None, error.public_message, None + if field_type == "switch": return _coerce_bool(value), None, None diff --git a/application/single_app/config.py b/application/single_app/config.py index 906d30955..0fa86a0ad 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.119" +VERSION = "0.261.120" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 89fcf9486..0d605925f 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -63,7 +63,9 @@ from functions_service_health import get_default_service_health from functions_workflow_limits import ( WORKFLOW_LOOP_ITEMS_DEFAULT, + WORKFLOW_REPEAT_ITERATIONS_DEFAULT, validate_workflow_max_loop_items, + validate_workflow_max_repeat_iterations, ) import admin_settings_secret_utils as _secret_utils import app_settings_cache @@ -1355,6 +1357,7 @@ def get_settings(use_cosmos=False, include_source=False): 'require_member_of_workflow_user': False, 'workflow_max_tasks': 50, 'workflow_max_loop_items': WORKFLOW_LOOP_ITEMS_DEFAULT, + 'workflow_max_repeat_iterations': WORKFLOW_REPEAT_ITERATIONS_DEFAULT, 'allow_group_workflows': False, 'require_group_assignment_for_group_workflows': False, 'group_workflow_allowed_group_ids': [], @@ -2156,6 +2159,13 @@ def update_settings(new_settings): new_settings['workflow_max_loop_items'] ), } + if isinstance(new_settings, dict) and 'workflow_max_repeat_iterations' in new_settings: + new_settings = { + **new_settings, + 'workflow_max_repeat_iterations': validate_workflow_max_repeat_iterations( + new_settings['workflow_max_repeat_iterations'] + ), + } screening_write = isinstance(new_settings, dict) and 'enable_content_screening' in new_settings try: # The guard imports storage clients only when a settings write is requested. diff --git a/application/single_app/functions_workflow_editor.py b/application/single_app/functions_workflow_editor.py index 1a785c37d..32ab928de 100644 --- a/application/single_app/functions_workflow_editor.py +++ b/application/single_app/functions_workflow_editor.py @@ -9,14 +9,19 @@ from functions_workflow_flow import FLOW_LIMITS from functions_workflow_limits import ( WORKFLOW_LOOP_ITEMS_DEFAULT, + WORKFLOW_REPEAT_ITERATIONS_DEFAULT, + WORKFLOW_REPEAT_ITERATIONS_MAX, get_workflow_max_loop_items, + get_workflow_max_repeat_iterations, validate_workflow_max_loop_items, + validate_workflow_max_repeat_iterations, ) def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks, agents, endpoints, default_model=None, - max_loop_items=WORKFLOW_LOOP_ITEMS_DEFAULT): + max_loop_items=WORKFLOW_LOOP_ITEMS_DEFAULT, + max_repeat_iterations=WORKFLOW_REPEAT_ITERATIONS_DEFAULT): if scope_type not in {"personal", "group"}: raise ValueError("Unsupported workflow editor scope.") agent_options = [ @@ -58,10 +63,10 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks return { "definition_version": WORKFLOW_DEFINITION_VERSION, "supported_definition_versions": [1, 2, 3], - "supported_node_kinds": ["task", "if", "route", "for_each", "collect"], + "supported_node_kinds": ["task", "if", "route", "for_each", "collect", "repeat_until"], "supported_iterable_kinds": ["input", "documents", "workspace_query"], "supported_query_modes": ["all_matches", "best_n"], - "supported_binding_sources": ["node_output", "loop_item"], + "supported_binding_sources": ["node_output", "loop_item", "repeat_state"], "supported_input_processing_modes": sorted(WORKFLOW_INPUT_PROCESSING_MODES), "supported_publication_completion_policies": list(WORKFLOW_PUBLICATION_COMPLETION_POLICIES), "publication_source_capabilities": [ @@ -73,6 +78,8 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks "flow_limits": { **FLOW_LIMITS, "max_loop_items": validate_workflow_max_loop_items(max_loop_items), + "max_repeat_iterations": validate_workflow_max_repeat_iterations(max_repeat_iterations), + "hard_repeat_iterations": WORKFLOW_REPEAT_ITERATIONS_MAX, }, "scope": {"type": scope_type, "id": str(scope_id)}, "can_manage": bool(can_manage), @@ -124,4 +131,5 @@ def get_workflow_editor_options(user_id, settings, *, group_id=""): can_manage=can_manage, max_tasks=get_workflow_max_tasks(settings), agents=agents, endpoints=endpoints, default_model=_build_default_model_summary(settings), max_loop_items=get_workflow_max_loop_items(settings), + max_repeat_iterations=get_workflow_max_repeat_iterations(settings), ) diff --git a/application/single_app/functions_workflow_execution_history.py b/application/single_app/functions_workflow_execution_history.py index c2c2341db..f2291eaee 100644 --- a/application/single_app/functions_workflow_execution_history.py +++ b/application/single_app/functions_workflow_execution_history.py @@ -3,50 +3,50 @@ from functions_analysis_access import authorize_analysis_sources, build_analysis_access from functions_workflow_identity import workflow_node_identity -from functions_workflow_node_results import authorize_workflow_node_result_read, result_selectors +from functions_workflow_limits import WORKFLOW_MAX_EXECUTION_ADMISSIONS +from functions_workflow_node_results import ( + WorkflowLineageAuthorization, authorize_workflow_node_result_read, load_node_result, result_selectors, +) from functions_workflow_result_store import read_workflow_node_result_page from functions_workflow_runtime_store import workflow_runtime_store -def authorize_execution_payload(workflow, run_id, payload, *, reader_user_id): +def authorize_execution_payload(workflow, run_id, payload, *, reader_user_id, authorization=None): + authorization = authorization or WorkflowLineageAuthorization( + workflow, run_id, reader_user_id=reader_user_id, store=workflow_runtime_store(workflow, run_id), + ) if payload.get("iteration_path"): - from functions_workflow_iterations import authorize_iteration_path - - authorize_iteration_path( - workflow, run_id, payload, reader_user_id=reader_user_id, - receipts=payload.get("iteration_inputs") or [], - ) + authorization.walk([("path", payload, payload.get("iteration_inputs") or [])]) if payload.get("node_kind") == "for_each": - from functions_workflow_iterations import authorize_frozen_loop - - store = workflow_runtime_store(workflow, run_id) + store = authorization.store loop = store.journal_read("loop", payload["execution_id"]) if loop: - authorize_frozen_loop( - workflow, run_id, {"producer": loop["payload"]["identity"], "manifest_ref": loop["payload"]["manifest_ref"]}, - reader_user_id=reader_user_id, store=store, - ) + authorization.walk([("frozen", { + "producer": loop["payload"]["identity"], "manifest_ref": loop["payload"]["manifest_ref"], + })]) + if payload.get("node_kind") == "repeat_until" or payload.get("decision_kind") == "repeat_transition" or payload.get("repeat"): + loop = authorization.store.journal_read("loop", payload["execution_id"]) + if loop: + head = loop["payload"] + reference = payload.get("after_state_ref") or head["current_state_ref"] + authorization.authorize_repeat(head["identity"], reference) references = payload.get("reference_sources") or [] if references: policy = build_analysis_access(references) authorize_analysis_sources(reader_user_id, policy["sources"]) summary = payload.get("workflow_result") or {} if summary.get("result_ref"): - authorize_workflow_node_result_read( - workflow, run_id, summary["producer"], summary["result_ref"], reader_user_id=reader_user_id, - ) - for receipt in payload.get("consumed_inputs") or []: - authorize_workflow_node_result_read( - workflow, run_id, receipt["producer"], receipt["result_ref"], reader_user_id=reader_user_id, - ) + authorization.authorize_result(summary["producer"], summary["result_ref"]) + authorization.walk(("receipt", receipt) for receipt in payload.get("consumed_inputs") or []) def workflow_execution_history(workflow, run_id, *, reader_user_id, kind="execution", execution_id=None, - cursor=None, limit=50): + cursor=None, limit=50, authorization=None): store = workflow_runtime_store(workflow, run_id) if store.read().get("schema_version") != 2: raise ValueError("Execution history is available only for structured workflow runs.") workflow = store.run_definition() + authorization = authorization or WorkflowLineageAuthorization(workflow, run_id, reader_user_id=reader_user_id, store=store) if execution_id: execution = store.journal_read("execution", execution_id) if execution is None: @@ -55,7 +55,10 @@ def workflow_execution_history(workflow, run_id, *, reader_user_id, kind="execut # Read the bound internal records too: safe decision projections intentionally omit source references. for item in page["items"]: if kind == "decision": - key = ["gate", item["gate_id"]] if item.get("gate_id") else ["control", item["execution_id"]] + key = ( + ["repeat-transition", item["execution_id"], item["iteration"]] if item.get("decision_kind") == "repeat_transition" + else ["gate", item["gate_id"]] if item.get("gate_id") else ["control", item["execution_id"]] + ) row = store.journal_read("decision", key) payload = row["payload"] else: @@ -64,7 +67,7 @@ def workflow_execution_history(workflow, run_id, *, reader_user_id, kind="execut if row is None: raise LookupError("The execution journal changed while it was being read.") payload = row["payload"] - authorize_execution_payload(workflow, run_id, payload, reader_user_id=reader_user_id) + authorize_execution_payload(workflow, run_id, payload, reader_user_id=reader_user_id, authorization=authorization) name = {"execution": "executions", "attempt": "attempts", "decision": "decisions"}[kind] result = {name: page["items"], "next_cursor": page["next_cursor"]} if kind == "execution": @@ -96,14 +99,12 @@ def workflow_execution_result_page(workflow, run_id, execution_id, attempt, *, r if not reference: raise LookupError("The selected output was not produced.") descriptor = (manifest.get("outputs") or {}).get(name) or {} - for _ in range(256): + for _ in range(WORKFLOW_MAX_EXECUTION_ADMISSIONS): selected = descriptor.get("selected_producer") if not selected or name == "manifest": break identity = selected["producer"] - manifest, _ = authorize_workflow_node_result_read( - workflow, run_id, identity, selected["result_ref"], reader_user_id=reader_user_id, - ) + manifest = load_node_result(workflow, run_id, identity, selected["result_ref"]) descriptor = (manifest.get("outputs") or {}).get(selected["output_name"]) or {} if descriptor.get("result_ref") != selected["output_ref"]: raise ValueError("The selected producer output changed.") diff --git a/application/single_app/functions_workflow_flow.py b/application/single_app/functions_workflow_flow.py index ca4f676ab..2e4fb7084 100644 --- a/application/single_app/functions_workflow_flow.py +++ b/application/single_app/functions_workflow_flow.py @@ -12,11 +12,12 @@ validate_workflow_publication_completion, workflow_output_kind_matches, ) from functions_workflow_loop_schema import WORKFLOW_DOCUMENT_ITEM_SCHEMA, normalize_workflow_iterable +from functions_workflow_limits import WORKFLOW_MAX_EXECUTION_ADMISSIONS, WORKFLOW_REPEAT_ITERATIONS_MAX FLOW_LIMITS = { "max_nodes": 256, "max_depth": 4, "max_predicate_nodes": 100, - "max_predicate_depth": 8, "max_executions": 5000, "deadline_seconds": 86400, + "max_predicate_depth": 8, "max_executions": WORKFLOW_MAX_EXECUTION_ADMISSIONS, "deadline_seconds": 86400, } MISSING = object() _ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z") @@ -41,7 +42,7 @@ def normalize_flow_bindings(values): names.add(name) source = binding.get("source") if not isinstance(source, dict) or source.get("scope", "current") != "current": - raise WorkflowDefinitionError("Bindings require a node_output or enclosing loop_item in the current scope.") + raise WorkflowDefinitionError("Bindings require a node_output, enclosing loop_item or repeat_state in the current scope.") if source.get("kind") == "node_output": _object(source, {"kind", "node_id", "output", "scope"}, "Binding source") output = source.get("output", "authoritative") @@ -53,8 +54,14 @@ def normalize_flow_bindings(values): elif source.get("kind") == "loop_item": _object(source, {"kind", "loop_id", "scope"}, "Loop item source") normalized_source = {"kind": "loop_item", "loop_id": _id(source.get("loop_id")), "scope": "current"} + elif source.get("kind") == "repeat_state": + _object(source, {"kind", "loop_id", "state_name", "scope"}, "Repeat state source") + normalized_source = { + "kind": "repeat_state", "loop_id": _id(source.get("loop_id")), + "state_name": _name(source.get("state_name"), "Repeat state name"), "scope": "current", + } else: - raise WorkflowDefinitionError("Bindings require a node_output or enclosing loop_item in the current scope.") + raise WorkflowDefinitionError("Bindings require a node_output, enclosing loop_item or repeat_state in the current scope.") kind = binding.get("expected_kind", "json" if source["kind"] == "loop_item" else "any") if not isinstance(kind, str) or kind not in WORKFLOW_OUTPUT_KINDS: raise WorkflowDefinitionError("Unsupported binding output kind.") @@ -275,9 +282,10 @@ def region(raw, depth, *, root=False, body=False, parent=None, loop_ids=()): "route": {"id", "kind", "inputs", "condition", "target"}, "for_each": {"id", "kind", "inputs", "iterable", "item_key", "max_items", "body"}, "collect": {"id", "kind", "source", "output_contract"}, + "repeat_until": {"id", "kind", "max_iterations", "state", "body", "until", "exports"}, } if not isinstance(kind, str) or kind not in allowed: - raise WorkflowDefinitionError("Only task, if, route, for_each and collect nodes are executable.") + raise WorkflowDefinitionError("Only task, if, route, for_each, collect and repeat_until nodes are executable.") _object(child, allowed[kind], "Flow node") node = {"id": register(child.get("id")), "kind": kind} nodes[node["id"]] = {"node": node, "region_id": result["id"]} @@ -306,6 +314,54 @@ def region(raw, depth, *, root=False, body=False, parent=None, loop_ids=()): node["body"] = region( child.get("body"), depth + 1, body=True, parent=node["id"], loop_ids=(*loop_ids, node["id"]), ) + elif kind == "repeat_until": + maximum = child.get("max_iterations") + if type(maximum) is not int or not 1 <= maximum <= WORKFLOW_REPEAT_ITERATIONS_MAX: + raise WorkflowDefinitionError( + f"Repeat max_iterations must be an explicit integer from 1 to {WORKFLOW_REPEAT_ITERATIONS_MAX:,} per automatic batch." + ) + node["max_iterations"] = maximum + state = child.get("state") + if not isinstance(state, list) or not 1 <= len(state) <= 100: + raise WorkflowDefinitionError("Repeat requires one to 100 named state slots.") + node["state"], names = [], set() + for slot in state: + _object(slot, {"name", "initial", "next", "output_contract"}, "Repeat state slot") + name = _name(slot.get("name"), "Repeat state name") + if name in names: + raise WorkflowDefinitionError("Repeat state names must be unique.") + names.add(name) + contract = normalize_workflow_output_contract(slot.get("output_contract")) + if contract["kind"] == "any": + raise WorkflowDefinitionError("Repeat state requires an explicit text, json, records or document_results kind.") + initial = normalize_flow_bindings([{ + "name": name, "source": slot.get("initial"), + "expected_kind": contract["kind"], "allow_partial": contract["allow_partial"], + }])[0]["source"] + if initial["kind"] not in {"node_output", "repeat_state"}: + raise WorkflowDefinitionError("Initial Repeat state must select a saved node output or enclosing Repeat state.") + node["state"].append({ + "name": name, "initial": initial, + "next": _name(slot.get("next"), "Next Repeat body output"), + "output_contract": contract, + }) + node["until"] = normalize_predicate(child.get("until"), node["state"]) + node["body"] = region( + child.get("body"), depth + 1, body=True, parent=node["id"], loop_ids=(*loop_ids, node["id"]), + ) + exports = child.get("exports") + if not isinstance(exports, list) or len(exports) > 100: + raise WorkflowDefinitionError("Repeat requires an explicit exports list of at most 100 entries.") + node["exports"], names = [], set() + for export in exports: + _object(export, {"name", "output"}, "Repeat export") + name = _name(export.get("name"), "Repeat export name") + if name in names: + raise WorkflowDefinitionError("Repeat export names must be unique.") + names.add(name) + node["exports"].append({ + "name": name, "output": _name(export.get("output"), "Repeat body output"), + }) elif kind == "collect": source = _object(child.get("source"), {"loop_id", "output"}, "Collect source") node["source"] = { @@ -371,7 +427,28 @@ def collection_keys(node): kind = node["output_contract"]["kind"] return {(node["id"], "authoritative"), (node["id"], "records" if kind == "records" else "documents")} - def descriptor(node_id, output): + def state_slot(source): + loop = nodes.get(source.get("loop_id"), {}).get("node", {}) + if loop.get("kind") != "repeat_until": + raise WorkflowDefinitionError("A repeat_state input must select an enclosing Repeat until block.") + slot = next((entry for entry in loop["state"] if entry["name"] == source.get("state_name")), None) + if slot is None: + raise WorkflowDefinitionError("The selected Repeat state slot is not declared.") + return slot + + def repeat_export_binding(node, output): + export = next((entry for entry in node["exports"] if entry["name"] == output), None) + if export is None: + raise WorkflowDefinitionError("The selected Repeat output is not declared.") + binding = next((entry for entry in node["body"]["outputs"] if entry["name"] == export["output"]), None) + if binding is None: + raise WorkflowDefinitionError("Repeat exports must select declared body outputs.") + return binding + + def descriptor(node_id, output, active=()): + key = (node_id, output) + if key in active: + raise WorkflowDefinitionError("Producer exports must not contain cycles.") entry = nodes.get(node_id) if not entry: raise WorkflowDefinitionError("A binding references a missing producer node.") @@ -385,8 +462,15 @@ def descriptor(node_id, output): if (node_id, output) not in collection_keys(node): raise WorkflowDefinitionError("Collect exposes only its exact collection kind and authoritative output.") return node["output_contract"]["kind"] + if node["kind"] == "repeat_until": + source = repeat_export_binding(node, output)["source"] + if source["kind"] == "repeat_state": + return state_slot(source)["output_contract"]["kind"] + if source["kind"] != "node_output": + raise WorkflowDefinitionError("Repeat exports require saved outputs or explicitly retained state.") + return descriptor(source["node_id"], source["output"], (*active, key)) if node["kind"] != "task" or output not in WORKFLOW_BINDABLE_OUTPUTS: - raise WorkflowDefinitionError("Only task final representations, Collect outputs and declared join exports can supply inputs.") + raise WorkflowDefinitionError("Only task final representations, Collect outputs and declared join or Repeat exports can supply inputs.") contract = catalogue[node["task_id"]].get("output_contract") or {} declared = contract.get("kind", "any") if output not in {"authoritative", "text"} and declared not in { @@ -404,12 +488,20 @@ def check_bindings(bindings, definite, possible, consumer): source = binding["source"] if source.get("kind") == "loop_item": loop_id = source["loop_id"] - if loop_id not in node_loop_ids[consumer]: + if loop_id not in node_loop_ids[consumer] or nodes[loop_id]["node"]["kind"] != "for_each": raise WorkflowDefinitionError("A loop_item binding must select an enclosing For each loop.") if not workflow_output_kind_matches("json", binding["expected_kind"]): raise WorkflowDefinitionError("A loop_item binding supplies a JSON object with value, key and index.") dependencies[consumer].append((loop_id, "loop_item")) continue + if source.get("kind") == "repeat_state": + if source["loop_id"] not in node_loop_ids[consumer]: + raise WorkflowDefinitionError("A repeat_state binding must select a declared enclosing Repeat until block.") + slot = state_slot(source) + if not workflow_output_kind_matches(slot["output_contract"]["kind"], binding["expected_kind"]): + raise WorkflowDefinitionError("The selected Repeat state does not match the declared input kind.") + dependencies[consumer].append((source["loop_id"], f"state:{slot['name']}")) + continue key = (source["node_id"], source["output"]) kind = descriptor(*key) if key not in possible: @@ -426,32 +518,54 @@ def task_keys(node): selectors.update({"json": {"json"}, "records": {"records"}, "document_results": {"documents"}}.get(declared, set())) return {(node["id"], output) for output in selectors} - leaf_cache = {} + # Deduplicate by source identity so shared join chains cannot expand exponentially. + contract_cache = {} - def output_leaves(node_id, output, active=()): + def output_contracts(node_id, output, active=()): key = (node_id, output) - if key in leaf_cache: - return leaf_cache[key] + if key in contract_cache: + return contract_cache[key] if key in active: raise WorkflowDefinitionError("Producer exports must not contain cycles.") - node = nodes[node_id]["node"] + entry = nodes.get(node_id) + if entry is None: + raise WorkflowDefinitionError("A binding references a missing producer node.") + node = entry["node"] if node["kind"] == "join": - export = next(item for item in node["exports"] if item["name"] == output) - leaves = tuple(dict.fromkeys( - leaf for branch in ("then", "else") - for leaf in output_leaves(export[branch]["node_id"], export[branch]["output"], (*active, key)) - )) + export = next((item for item in node["exports"] if item["name"] == output), None) + if export is None: + raise WorkflowDefinitionError("The selected join output is not declared.") + contracts = {} + for branch in ("then", "else"): + contracts.update(output_contracts( + export[branch]["node_id"], export[branch]["output"], (*active, key), + )) + elif node["kind"] == "repeat_until": + contracts = source_contracts(repeat_export_binding(node, output)["source"], (*active, key)) + elif node["kind"] in {"task", "collect"}: + descriptor(node_id, output) + contracts = {("node_output", node_id, output): (output_contract(node), output)} else: - leaves = (key,) - leaf_cache[key] = leaves - return leaves + raise WorkflowDefinitionError("The selected engine node does not declare final data.") + contract_cache[key] = contracts + return contracts + + def source_contracts(source, active=()): + if source.get("kind") == "repeat_state": + contract = state_slot(source)["output_contract"] + selector = {"document_results": "documents"}.get(contract["kind"], contract["kind"]) + return {("repeat_state", source["loop_id"], source["state_name"]): (contract, selector)} + if source.get("kind", "node_output") != "node_output": + raise WorkflowDefinitionError("This operation requires a saved output or declared Repeat state.") + return output_contracts(source["node_id"], source["output"], active) + + def contract_kind(contract, selector): + return {"text": "text", "json": "json", "records": "records", "documents": "document_results"}.get( + selector, contract.get("kind", "any"), + ) def structured_output(node_id, output): - for producer_id, selector in output_leaves(node_id, output): - node = nodes[producer_id]["node"] - if node["kind"] not in {"task", "collect"}: - return False - contract = output_contract(node) + for contract, selector in output_contracts(node_id, output).values(): schema = contract.get("schema") or {} root_types = schema.get("type") root_types = {root_types} if isinstance(root_types, str) else set(root_types or []) @@ -464,13 +578,10 @@ def structured_output(node_id, output): return True def output_schemas(node_id, output): - return [output_contract(nodes[producer_id]["node"]).get("schema") or {} - for producer_id, _ in output_leaves(node_id, output)] + return [contract.get("schema") or {} for contract, _ in output_contracts(node_id, output).values()] def collection_kind(source): - selected = {descriptor(node_id, output) for node_id, output in output_leaves( - source["node_id"], source["output"], - )} + selected = {contract_kind(contract, selector) for contract, selector in source_contracts(source).values()} if len(selected) != 1 or not selected <= {"records", "document_results"}: raise WorkflowDefinitionError("Loop collections require one exact records or document_results kind on every producer.") return next(iter(selected)) @@ -489,12 +600,14 @@ def prepare_loop(node): iterable = node["iterable"] if iterable["kind"] == "input": binding = next((item for item in node["inputs"] if item["name"] == iterable["name"]), None) - if binding is None or binding["source"]["kind"] != "node_output" or not binding["required"] or binding["allow_partial"]: - raise WorkflowDefinitionError("A saved iterable must select a required, nonpartial node-output input.") + if ( + binding is None or binding["source"]["kind"] not in {"node_output", "repeat_state"} + or not binding["required"] or binding["allow_partial"] + ): + raise WorkflowDefinitionError("A saved iterable must select a required, nonpartial node-output or Repeat-state input.") collection_kind(binding["source"]) - values = [collection_item_schema(schema) for schema in output_schemas( - binding["source"]["node_id"], binding["source"]["output"], - )] + values = [collection_item_schema(contract.get("schema") or {}) + for contract, _ in source_contracts(binding["source"]).values()] else: values = [WORKFLOW_DOCUMENT_ITEM_SCHEMA] loop_item_schemas[node["id"]] = [{ @@ -540,7 +653,7 @@ def check_current_document(node): if action.get("type") != "analyze" or action.get("target_mode") != "current_item" or action.get("analysis_mode") != "combined": raise WorkflowDefinitionError("Current-item Analyze requires combined analysis of one enclosing document item.") loop_id = _id(action.get("loop_id")) - if loop_id not in node_loop_ids[node["id"]]: + if loop_id not in node_loop_ids[node["id"]] or nodes[loop_id]["node"]["kind"] != "for_each": raise WorkflowDefinitionError("Current-item Analyze must select an enclosing document loop.") if nodes[loop_id]["node"]["iterable"]["kind"] not in {"documents", "workspace_query"}: raise WorkflowDefinitionError("Current-item Analyze requires a document iterable, not saved records or document results.") @@ -559,12 +672,14 @@ def check_input_processing(node): ) for binding in task["inputs"]: source = binding["source"] - if source["kind"] == "node_output" and all( - descriptor(producer_id, output) in {"records", "document_results"} - for producer_id, output in output_leaves(source["node_id"], source["output"]) + if source["kind"] in {"node_output", "repeat_state"} and all( + contract_kind(contract, selector) in {"records", "document_results"} + for contract, selector in source_contracts(source).values() ): return - raise WorkflowDefinitionError("saved_record_report requires at least one node-output records or document_results input.") + raise WorkflowDefinitionError( + "saved_record_report requires at least one node-output records or document_results input, or equivalent Repeat state." + ) def check_predicate(predicate, bindings): used = {binding["name"]: binding for binding in bindings} @@ -577,6 +692,17 @@ def operand_types(operand, *, scalar=True): source = used[operand["input"]]["source"] if source["kind"] == "loop_item": schemas = loop_item_schemas[source["loop_id"]] + elif source["kind"] == "repeat_state": + contract = state_slot(source)["output_contract"] + schema = contract.get("schema") or {} + root = schema.get("type") + root_types = {root} if isinstance(root, str) else set(root or []) + if ( + contract["kind"] not in {"json", "records", "document_results"} + or not root_types or not root_types <= {"object", "array"} + ): + raise WorkflowDefinitionError("Repeat conditions require explicitly schema-validated JSON or record fields.") + schemas = [schema] else: if not structured_output(source["node_id"], source["output"]): raise WorkflowDefinitionError("Conditions require JSON or record fields with an explicit schema on every possible producer.") @@ -675,6 +801,53 @@ def analyze(current, initial, *, branch=False): check_bindings(node["body"]["outputs"], *ends, node["body"]["id"]) after_possible.add((node["id"], "loop_complete")) after_definite.add((node["id"], "loop_complete")) + elif kind == "repeat_until": + initial = [{ + "name": slot["name"], "source": slot["initial"], "required": True, + "expected_kind": slot["output_contract"]["kind"], + "allow_partial": slot["output_contract"]["allow_partial"], + } for slot in node["state"]] + check_bindings(initial, definite, possible, node["id"]) + for slot in node["state"]: + if any( + contract_kind(contract, selector) not in {"any", slot["output_contract"]["kind"]} + for contract, selector in source_contracts(slot["initial"]).values() + ): + raise WorkflowDefinitionError("Initial Repeat state must preserve its exact declared output kind.") + ends = analyze(node["body"], (set(definite), set(possible))) + body_outputs = {binding["name"]: binding for binding in node["body"]["outputs"]} + for binding in body_outputs.values(): + source = binding["source"] + if source["kind"] == "node_output": + if node_loop_ids.get(source.get("node_id")) != [*node_loop_ids[node["id"]], node["id"]]: + raise WorkflowDefinitionError("A Repeat body export must select a producer in its own body scope.") + elif source["kind"] != "repeat_state": + raise WorkflowDefinitionError("Repeat body exports require saved outputs or explicitly retained state.") + check_bindings(node["body"]["outputs"], *ends, node["body"]["id"]) + for slot in node["state"]: + binding = body_outputs.get(slot["next"]) + if binding is None or not binding["required"]: + raise WorkflowDefinitionError("Every next-state slot must select a required declared Repeat body output.") + contract = slot["output_contract"] + if any( + contract_kind(source_contract, selector) not in {"any", contract["kind"]} + for source_contract, selector in source_contracts(binding["source"]).values() + ): + raise WorkflowDefinitionError("Next Repeat state must preserve its exact declared output kind.") + if binding["allow_partial"] and not contract["allow_partial"]: + raise WorkflowDefinitionError("Partial next-state outputs require explicit acceptance by the Repeat state slot.") + next_bindings = [{ + "name": slot["name"], "source": { + "kind": "repeat_state", "loop_id": node["id"], "state_name": slot["name"], "scope": "current", + }, + } for slot in node["state"]] + check_predicate(node["until"], next_bindings) + for export in node["exports"]: + binding = repeat_export_binding(node, export["name"]) + key = (node["id"], export["name"]) + after_possible.add(key) + if binding["required"]: + after_definite.add(key) elif kind == "collect": check_collect(node, definite, possible) keys = collection_keys(node) diff --git a/application/single_app/functions_workflow_flow_runner.py b/application/single_app/functions_workflow_flow_runner.py index 2e5143f6f..60f2f6e82 100644 --- a/application/single_app/functions_workflow_flow_runner.py +++ b/application/single_app/functions_workflow_flow_runner.py @@ -37,7 +37,7 @@ def __init__(self, workflow, run_id, execution, task_results, *, actor_user_id, self.finished = False self.control_receipts = [] self.loop_frames = [] - self.has_loops = any(entry["node"]["kind"] == "for_each" for entry in self.compiled["nodes"].values()) + self.has_loops = any(entry["node"]["kind"] in {"for_each", "repeat_until"} for entry in self.compiled["nodes"].values()) def _producer_path(self, node_id): ancestors = self.compiled.get("node_loop_ids", {}).get(node_id, []) @@ -66,6 +66,33 @@ def _remember(self, node_id, value): if not self.execution.iteration_path: self.completed[node_id] = value + def source_producer(self, source): + if source.get("kind", "node_output") != "repeat_state": + return self.producer(source["node_id"]) + from functions_workflow_node_results import load_node_result + from functions_workflow_repeat_state import current_repeat_state, repeat_node + + slot, receipt = current_repeat_state( + self.workflow, self.run_id, self.execution.iteration_path, source, + reader_user_id=self.actor_user_id, store=self.execution.store, load_result=self.execution.load_result, + ) + manifest = load_node_result( + self.workflow, self.run_id, receipt["producer"], receipt["result_ref"], load_result=self.execution.load_result, + ) + summary = workflow_result_summary(manifest, receipt["result_ref"]) + summary["workflow_validation"] = deepcopy(slot["workflow_validation"]) + declaration = next( + value for value in repeat_node(self.workflow, source["loop_id"])["state"] + if value["name"] == source["state_name"] + ) + return { + "state": "completed", "summary": summary, "state_receipt": receipt, + "structured_validated": bool(declaration["output_contract"].get("schema")), + "state_metadata": {name: deepcopy(slot[name]) for name in ( + "workflow_validation", "coverage", "prior_coverage", "limitations", + ) if name in slot}, + } + def current_item(self, loop_id): from functions_workflow_iterations import load_frozen_item_value @@ -114,7 +141,7 @@ def resolve(self, bindings, *, condition=None, stream_collections=False, metadat "kind": "json", "value": value, }}) continue - producer = self.producer(source["node_id"]) + producer = self.source_producer(source) if producer is None or producer.get("state") == "skipped": if binding["required"]: raise WorkflowInputError("A required producer was intentionally skipped or did not finish.") @@ -124,23 +151,33 @@ def resolve(self, bindings, *, condition=None, stream_collections=False, metadat if producer.get("state") not in {"succeeded", "completed", "completed_partial"}: raise WorkflowInputError("The selected producer is failed, invalid or pending, not optional absence.") summary = producer["summary"] + state_receipt = producer.get("state_receipt") + output_name = state_receipt["output_name"] if state_receipt else source["output"] + if ( + state_receipt and (summary.get("workflow_validation") or {}).get("status") == "accepted_partial" + and not binding["allow_partial"] + ): + raise WorkflowInputError("This input does not accept the retained partial Repeat state.") try: - name = summary.get("authoritative_output") if source["output"] == "authoritative" else source["output"] + name = summary.get("authoritative_output") if output_name == "authoritative" else output_name descriptor = summary.get("outputs", {}).get(name) or {} if stream_collections and descriptor.get("kind") in {"records", "document_results"}: reader = open_workflow_record_input( self.workflow, self.run_id, summary["producer"], summary["result_ref"], - output_name=source["output"], allow_partial=binding["allow_partial"], + output_name=output_name, allow_partial=binding["allow_partial"], reader_user_id=self.actor_user_id, load_result=self.execution.load_result, ) if not workflow_output_kind_matches(reader.kind, binding["expected_kind"]): raise WorkflowInputError("The selected collection does not match the declared input kind.") + if state_receipt: + reader.receipt["repeat_state"] = deepcopy(state_receipt["repeat_state"]) receipts.append({**reader.receipt, "input_name": binding["name"]}) values[binding["name"]] = reader record_inputs.append({"name": binding["name"], "reader": reader}) inputs.append({"name": binding["name"], "status": "available", "result": { "kind": reader.kind, "record_count": reader.record_count, "consumed_result": reader.receipt, "complete_records_supplied_separately": True, + **(producer.get("state_metadata") or {}), }}) self.partial |= (summary.get("workflow_validation") or {}).get("status") == "accepted_partial" continue @@ -165,7 +202,7 @@ def resolve(self, bindings, *, condition=None, stream_collections=False, metadat else: prompt, receipt = self.load_output( self.workflow, self.run_id, summary["producer"], summary["result_ref"], - output_name=source["output"], allow_partial=binding["allow_partial"], + output_name=output_name, allow_partial=binding["allow_partial"], reader_user_id=self.actor_user_id, required=binding["required"], ) except AnalysisResultUnavailable: @@ -176,7 +213,11 @@ def resolve(self, bindings, *, condition=None, stream_collections=False, metadat inputs.append({"name": binding["name"], "status": "unavailable"}) values[binding["name"]] = MISSING continue + if state_receipt: + receipt["repeat_state"] = deepcopy(state_receipt["repeat_state"]) payload = json.loads(prompt) + if state_receipt: + payload.update(producer.get("state_metadata") or {}) if not workflow_output_kind_matches(payload["kind"], binding["expected_kind"]): raise WorkflowInputError("The input does not match its declared output kind.") if condition is not None and binding["name"] in condition: @@ -304,7 +345,7 @@ def _join(self, node, region_id, branch, decision_summary): continue selected = resolved["bound_inputs"][0] receipts.append(selected) - producer = self.producer(source["node_id"]) + producer = self.source_producer(source) structured = structured and producer.get("structured_validated", False) descriptor = producer["summary"]["outputs"][selected["output_name"]] exports[export["name"]] = { @@ -521,6 +562,10 @@ def _region(self, region): self.execution.check() if node["kind"] == "for_each": yield from self._for_each(node, region["id"]) + elif node["kind"] == "repeat_until": + from functions_workflow_repeat_execution import run_repeat_until + + yield from run_repeat_until(self, node, region["id"]) elif node["kind"] == "collect": self._collect(node, region["id"]) elif node["kind"] == "task": diff --git a/application/single_app/functions_workflow_identity.py b/application/single_app/functions_workflow_identity.py index 21028cf21..d0963c9a9 100644 --- a/application/single_app/functions_workflow_identity.py +++ b/application/single_app/functions_workflow_identity.py @@ -6,6 +6,7 @@ import re from functions_workflow_definitions import workflow_definition_revision +from functions_workflow_limits import WORKFLOW_MAX_EXECUTION_ADMISSIONS, WORKFLOW_REPEAT_ITERATIONS_MAX from functions_workflow_loop_schema import WORKFLOW_LOOP_MAX_ITEMS @@ -19,24 +20,33 @@ def canonical_digest(value): def normalize_workflow_iteration_path(path): - """Validate shape only; readers must additionally prove sealed item membership.""" + """Validate shape only; readers must additionally prove sealed iteration admission.""" if path is None: return [] if not isinstance(path, list) or len(path) > 3: raise ValueError("An iteration path requires at most three enclosing loop frames.") normalized, loops = [], set() for frame in path: - if not isinstance(frame, dict) or frame.keys() != {"loop_id", "item_id", "index"}: - raise ValueError("An iteration frame requires exactly loop_id, item_id and index.") - loop_id, item_id, index = frame["loop_id"], frame["item_id"], frame["index"] + if not isinstance(frame, dict) or frame.keys() not in ( + {"loop_id", "item_id", "index"}, {"loop_id", "iteration"}, + ): + raise ValueError("An iteration frame requires an exact For-each item or Repeat round.") + loop_id = frame["loop_id"] if not isinstance(loop_id, str) or not _NODE_ID.fullmatch(loop_id) or loop_id in loops: raise ValueError("Iteration loop ids must be stable and unique within a path.") - if not isinstance(item_id, str) or not _ITEM_ID.fullmatch(item_id): - raise ValueError("Iteration item ids must be lowercase SHA256 digests.") - if type(index) is not int or not 0 <= index < WORKFLOW_LOOP_MAX_ITEMS: - raise ValueError("An iteration index must be a zero-based integer within the technical item limit.") loops.add(loop_id) - normalized.append({"loop_id": loop_id, "item_id": item_id, "index": index}) + if "iteration" in frame: + iteration = frame["iteration"] + if type(iteration) is not int or not 0 <= iteration < WORKFLOW_MAX_EXECUTION_ADMISSIONS: + raise ValueError("A Repeat iteration must be a zero-based lifetime index within the execution limit.") + normalized.append({"loop_id": loop_id, "iteration": iteration}) + else: + item_id, index = frame["item_id"], frame["index"] + if not isinstance(item_id, str) or not _ITEM_ID.fullmatch(item_id): + raise ValueError("Iteration item ids must be lowercase SHA256 digests.") + if type(index) is not int or not 0 <= index < WORKFLOW_LOOP_MAX_ITEMS: + raise ValueError("An iteration index must be a zero-based integer within the technical item limit.") + normalized.append({"loop_id": loop_id, "item_id": item_id, "index": index}) return normalized @@ -62,7 +72,7 @@ def visit(region, ancestors, depth): raise ValueError("A saved flow node must be an object.") register(node.get("id")) kind = node.get("kind") - if not isinstance(kind, str) or kind not in {"task", "if", "route", "for_each", "collect"}: + if not isinstance(kind, str) or kind not in {"task", "if", "route", "for_each", "collect", "repeat_until"}: raise ValueError("The saved flow node kind is unsupported.") if kind == "task": task_id = node.get("task_id") @@ -81,11 +91,12 @@ def visit(region, ancestors, depth): matched = ({"id": node_id, "kind": "join"}, ancestors) visit(node.get("then"), ancestors, depth + 1) visit(node.get("else"), ancestors, depth + 1) - elif kind == "for_each": - max_items = node.get("max_items") - if type(max_items) is not int or not 1 <= max_items <= WORKFLOW_LOOP_MAX_ITEMS: - raise ValueError("A saved loop requires a bounded max_items value.") - visit(node.get("body"), (*ancestors, (node["id"], max_items)), depth + 1) + elif kind in {"for_each", "repeat_until"}: + limit = node.get("max_items" if kind == "for_each" else "max_iterations") + maximum = WORKFLOW_LOOP_MAX_ITEMS if kind == "for_each" else WORKFLOW_REPEAT_ITERATIONS_MAX + if type(limit) is not int or not 1 <= limit <= maximum: + raise ValueError("A saved loop requires an explicit bounded iteration limit.") + visit(node.get("body"), (*ancestors, (node["id"], kind, limit)), depth + 1) visit(workflow.get("flow"), (), 1) if matched is None: @@ -107,10 +118,20 @@ def _execution_parts(workflow, run_id, node_id, iteration_path): raise ValueError("Iteration paths require a structured workflow.") return path, None node, ancestors = _flow_node(workflow, node_id) - if [frame["loop_id"] for frame in path] != [loop_id for loop_id, _ in ancestors]: + if [frame["loop_id"] for frame in path] != [loop_id for loop_id, _, _ in ancestors]: raise ValueError("The iteration path does not match the node's enclosing loop ancestors.") - if any(frame["index"] >= limit for frame, (_, limit) in zip(path, ancestors)): - raise ValueError("An iteration index exceeds its enclosing loop's authored item limit.") + for frame, (_, kind, limit) in zip(path, ancestors): + if kind == "for_each": + if "index" not in frame or frame["index"] >= limit: + raise ValueError("An iteration index exceeds its enclosing For-each loop's authored item limit.") + else: + run_limit = (workflow.get("limits") or {}).get("max_executions", WORKFLOW_MAX_EXECUTION_ADMISSIONS) + if ( + "iteration" not in frame or type(run_limit) is not int + or not 1 <= run_limit <= WORKFLOW_MAX_EXECUTION_ADMISSIONS + or frame["iteration"] >= run_limit + ): + raise ValueError("A Repeat lifetime iteration exceeds its enclosing run's execution limit.") return path, node diff --git a/application/single_app/functions_workflow_iterations.py b/application/single_app/functions_workflow_iterations.py index cb1ec148b..4ba3788e6 100644 --- a/application/single_app/functions_workflow_iterations.py +++ b/application/single_app/functions_workflow_iterations.py @@ -6,7 +6,9 @@ from copy import deepcopy from functions_analysis_access import AnalysisResultUnavailable -from functions_workflow_identity import canonical_digest, workflow_execution_id, workflow_node_identity +from functions_workflow_identity import ( + canonical_digest, normalize_workflow_iteration_path, workflow_execution_id, workflow_node_identity, +) from functions_workflow_result_store import load_workflow_node_result, _quota_bytes from functions_workflow_runtime_store import workflow_runtime_store @@ -171,7 +173,7 @@ def load_frozen_loop(workflow, run_id, identity, *, store=None, load_result=load if candidate["id"] == identity["node_id"] and candidate["kind"] == "for_each": node = candidate break - if candidate["kind"] == "for_each": + if candidate["kind"] in {"for_each", "repeat_until"}: pending.extend(candidate["body"]["nodes"]) elif candidate["kind"] == "if": pending.extend(candidate["then"]["nodes"]) @@ -191,12 +193,20 @@ def load_frozen_loop(workflow, run_id, identity, *, store=None, load_result=load binding = next((value for value in node["inputs"] if value["name"] == node["iterable"]["name"]), None) source = manifest.get("source_receipt") or {} producer = source.get("producer") or {} - if ( - binding is None or producer.get("node_id") != binding["source"]["node_id"] + if binding is None or manifest.get("source_allow_partial") != binding["allow_partial"]: + raise AnalysisResultUnavailable("workflow_loop_source_receipt_invalid") + if binding["source"]["kind"] == "repeat_state": + from functions_workflow_repeat_state import validate_repeat_source_receipt + + validate_repeat_source_receipt( + workflow, run_id, binding["source"], source, identity["iteration_path"], + store=store, load_result=load_result, + ) + elif ( + producer.get("node_id") != binding["source"]["node_id"] or producer.get("workflow_id") != workflow["id"] or producer.get("run_id") != run_id or len(producer.get("iteration_path") or []) > len(identity["iteration_path"]) or (producer.get("iteration_path") or []) != identity["iteration_path"][:len(producer.get("iteration_path") or [])] - or manifest.get("source_allow_partial") != binding["allow_partial"] ): raise AnalysisResultUnavailable("workflow_loop_source_receipt_invalid") elif manifest.get("source_receipt") is not None: @@ -257,6 +267,8 @@ def load_frozen_item_value(workflow, run_id, manifest, item, *, reader_user_id, allow_partial=manifest.get("source_allow_partial", False), load_result=load_result, source_resolver=source_resolver, ) + if receipt.get("repeat_state"): + reader.receipt["repeat_state"] = deepcopy(receipt["repeat_state"]) rows, _ = reader.read_records(offset=item["source_ordinal"], limit=1) if len(rows) != 1 or canonical_digest(rows[0]) != item["record_sha256"]: raise AnalysisResultUnavailable("workflow_loop_item_changed") @@ -269,66 +281,68 @@ def load_frozen_item_value(workflow, run_id, manifest, item, *, reader_user_id, def authorize_frozen_loop(workflow, run_id, binding, *, reader_user_id, load_result=load_workflow_node_result, source_resolver=None, store=None, source_callback=None): - from functions_workflow_node_results import authorize_workflow_node_result_read + from functions_workflow_node_results import WorkflowLineageAuthorization - if not isinstance(binding, dict) or not isinstance(binding.get("producer"), dict): - raise AnalysisResultUnavailable("workflow_loop_identity_invalid") - manifest, reference, _ = load_frozen_loop( - workflow, run_id, binding["producer"], store=store, load_result=load_result, + authorization = WorkflowLineageAuthorization( + workflow, run_id, reader_user_id=reader_user_id, load_result=load_result, source_resolver=source_resolver, + include_sources=source_callback is None, source_callback=source_callback, + store=store or workflow_runtime_store(workflow, run_id), ) - if reference != binding.get("manifest_ref"): - raise AnalysisResultUnavailable("workflow_loop_manifest_invalid") - sources, changed = {}, False - source_ids = set() - - def source_seen(source): - key = canonical_digest(source) - if key in source_ids: - return - source_ids.add(key) - if source_callback is not None: - source_callback(source) - else: - sources[key] = source - - for receipt in manifest.get("consumed_inputs") or []: - _, access = authorize_workflow_node_result_read( - workflow, run_id, receipt["producer"], receipt["result_ref"], - reader_user_id=reader_user_id, load_result=load_result, source_resolver=source_resolver, - include_sources=False, source_callback=source_seen, - ) - changed |= access["source_snapshot_changed"] - if manifest["selection"]["kind"] != "input": - for index in range(manifest["count"]): - item = read_frozen_item(workflow, run_id, manifest, index, load_result=load_result) - _authorize_frozen_document(workflow, item, reader_user_id) - source_seen(item["source"]) - return { - "sources": list(sources.values()) if source_callback is None else None, - "source_count": len(source_ids), "source_snapshot_changed": changed, - } + authorization.walk([("frozen", binding)]) + return authorization.access() -def authorize_iteration_path(workflow, run_id, identity, *, reader_user_id, receipts=None, - load_result=load_workflow_node_result, source_resolver=None, store=None): - path = identity.get("iteration_path") or [] +def iteration_path_proofs(workflow, run_id, identity, *, receipts=None, store=None, + load_result=load_workflow_node_result): + try: + path = normalize_workflow_iteration_path(identity.get("iteration_path") or []) + if identity.get("node_id") and identity.get("execution_id") != workflow_execution_id( + workflow, run_id, identity["node_id"], path, + ): + raise ValueError("Mismatched execution path.") + except ValueError as exc: + raise AnalysisResultUnavailable("workflow_iteration_receipt_invalid") from exc if receipts is not None and (not isinstance(receipts, list) or len(receipts) != len(path)): raise AnalysisResultUnavailable("workflow_iteration_receipt_invalid") - verified = [] + store = store or workflow_runtime_store(workflow, run_id) + verified, dependencies = [], [] for index, frame in enumerate(path): producer = loop_execution_identity(workflow, run_id, frame["loop_id"], path[:index]) - manifest, reference, _ = load_frozen_loop( - workflow, run_id, producer, store=store, load_result=load_result, - ) - item = read_frozen_item(workflow, run_id, manifest, frame["index"], load_result=load_result) - if frame["item_id"] != item["item_id"]: - raise AnalysisResultUnavailable("workflow_loop_item_invalid") - expected = frozen_item_receipt(manifest, reference, item) + if "iteration" in frame: + from functions_workflow_repeat_state import load_repeat_admission + + _, expected = load_repeat_admission(workflow, run_id, producer, frame["iteration"], store=store) + dependencies.append(("admission", producer, frame["iteration"])) + else: + manifest, reference, _ = load_frozen_loop( + workflow, run_id, producer, store=store, load_result=load_result, + ) + item = read_frozen_item(workflow, run_id, manifest, frame["index"], load_result=load_result) + if frame["item_id"] != item["item_id"]: + raise AnalysisResultUnavailable("workflow_loop_item_invalid") + expected = frozen_item_receipt(manifest, reference, item) + dependencies.append(( + "frozen_item", {"producer": producer, "manifest_ref": reference}, frame["index"], frame["item_id"], + )) if receipts is not None and receipts[index] != expected: raise AnalysisResultUnavailable("workflow_iteration_receipt_invalid") - load_frozen_item_value( - workflow, run_id, manifest, item, reader_user_id=reader_user_id, - load_result=load_result, source_resolver=source_resolver, - ) verified.append(expected) + return verified, dependencies + + +def authorize_iteration_path(workflow, run_id, identity, *, reader_user_id, receipts=None, + load_result=load_workflow_node_result, source_resolver=None, store=None): + from functions_workflow_node_results import WorkflowLineageAuthorization + + store = store or workflow_runtime_store(workflow, run_id) + verified, dependencies = iteration_path_proofs( + workflow, run_id, identity, receipts=receipts, store=store, load_result=load_result, + ) + authorization = WorkflowLineageAuthorization( + workflow, run_id, reader_user_id=reader_user_id, load_result=load_result, + source_resolver=source_resolver, store=store, + ) + authorization.walk(dependencies) + if authorization.access()["source_snapshot_changed"]: + raise AnalysisResultUnavailable("analysis_source_snapshot_changed") return verified diff --git a/application/single_app/functions_workflow_journal.py b/application/single_app/functions_workflow_journal.py index b3782a2d9..0d040cd3c 100644 --- a/application/single_app/functions_workflow_journal.py +++ b/application/single_app/functions_workflow_journal.py @@ -20,6 +20,7 @@ PUBLIC_DECISION_FIELDS = ( "sequence", "execution_id", "node_id", "iteration_path", "attempt", "gate_id", "gate_kind", "choice", "decision", "actor_user_id", "decided_at", "reason_code", "input_digest", + "decision_kind", "iteration", "batch_number", "condition_result", "repeat", "event_id", ) @@ -27,6 +28,20 @@ def journal_record_id(kind, key): return f"workflow-journal:v2:{kind}:{canonical_digest(key)}" +def _public_journal_value(value): + if isinstance(value, dict): + private = {"repeat_state"} if {"producer", "result_ref", "output_name", "output_ref"} <= value.keys() else set() + if {"loop_id", "loop_execution_id", "iteration", "state_ref"} <= value.keys(): + private.add("state_ref") + return { + name: _public_journal_value(item) for name, item in value.items() + if name not in private + } + if isinstance(value, list): + return [_public_journal_value(item) for item in value] + return deepcopy(value) + + class WorkflowJournalMixin: """Atomic decision/cursor/admission updates; control does not grow per execution.""" @@ -123,6 +138,102 @@ def journal_commit(self, token, kind, key, payload, *, updates=None, admission=F raise self._journal_conflict("etag_conflict") + def journal_commit_many(self, token, entries, *, updates=None, counters=None): + """Commit a bounded transition behind one immutable acknowledgement marker.""" + from functions_workflow_execution import WorkflowSuspended + from functions_workflow_runtime_store import _bounded_json_copy, _validate_gate + + if ( + not isinstance(entries, list) or not 1 <= len(entries) <= 12 + or not entries[0].get("immutable") + or any(entry.get("kind") not in JOURNAL_KINDS for entry in entries) + or len({journal_record_id(entry["kind"], entry["key"]) for entry in entries}) != len(entries) + ): + self._journal_conflict("invalid_payload") + updates = deepcopy(updates or {}) + if set(updates) - {"cursor", "progress", "phase", "state", "gate", "lease", "loop_progress", "repeat_progress"}: + self._journal_conflict("invalid_payload") + if updates.get("gate") is not None: + updates["gate"] = _validate_gate(updates["gate"], updates.get("state", "paused")) + counters = counters or {} + if set(counters) - {"exhaustion_count", "continuation_count"} or any( + type(value) is not int or value < 0 for value in counters.values() + ): + self._journal_conflict("invalid_payload") + marker = entries[0] + for _ in range(8): + control = self._read_control() + saved = self.journal_read(marker["kind"], marker["key"]) + if saved is not None: + if saved["payload"] != marker["payload"]: + self._journal_conflict("immutable_conflict") + return saved + self._assert_current_owned(control, token) + if control.get("schema_version") != 2: + self._journal_conflict("schema_mismatch") + if self._now().isoformat() >= control["deadline_at"]: + self.pause_execution_limit(token, "deadline_exceeded") + raise WorkflowSuspended("paused") + replacement = self._base_replacement(control) + sequence = int(control.get("journal_sequence") or 0) + counts = dict(control.get("journal_counts") or {}) + admitted = int(control.get("admitted_count") or 0) + operations = [] + for entry in entries: + kind, key, payload = entry["kind"], entry["key"], entry["payload"] + previous = self.journal_read(kind, key) + if "expected" in entry and (previous["payload"] if previous else None) != entry["expected"]: + self._journal_conflict("transition_changed") + if previous is not None and entry.get("immutable"): + self._journal_conflict("immutable_conflict") + if previous is None: + sequence += 1 + counts[kind] = int(counts.get(kind) or 0) + 1 + admitted += int(bool(entry.get("admission"))) + body = self._runtime_record({ + "id": journal_record_id(kind, key), "run_id": self.identity["run_id"], + "type": JOURNAL_TYPE, "item_type": JOURNAL_TYPE, "record_kind": kind, + "key": key, "sequence": previous["sequence"] if previous else sequence, + "execution_id": payload.get("execution_id"), "payload": deepcopy(payload), + }) + operations.append( + ("replace", (body["id"], body), {"if_match_etag": previous["_etag"]}) + if previous else ("create", (body,)) + ) + if admitted > control["max_executions"]: + self.pause_execution_limit(token, "execution_budget_exceeded") + raise WorkflowSuspended("paused") + metrics = dict(control.get("repeat_counts") or {}) + for name, value in counters.items(): + metrics[name] = int(metrics.get(name) or 0) + value + replacement.update( + **updates, version=control["version"] + 1, journal_sequence=sequence, + journal_counts=counts, admitted_count=admitted, + ) + if counters: + replacement["repeat_counts"] = metrics + _bounded_json_copy(replacement) + operations.insert(0, ("replace", (control["id"], replacement), {"if_match_etag": control["_etag"]})) + if len(json.dumps(operations, ensure_ascii=True).encode("ascii")) > 1500000: + self._journal_conflict("transition_too_large") + if self._now().isoformat() >= control["deadline_at"]: + self.pause_execution_limit(token, "deadline_exceeded") + raise WorkflowSuspended("paused") + self._assert_current_owned(control, token) + try: + self.container.execute_item_batch(batch_operations=operations, partition_key=self.identity["run_id"]) + return self.journal_read(marker["kind"], marker["key"]) + except (cosmos_exceptions.CosmosBatchOperationError, cosmos_exceptions.CosmosHttpResponseError) as exc: + acknowledged = self.journal_read(marker["kind"], marker["key"]) + if acknowledged is not None: + if acknowledged["payload"] != marker["payload"]: + self._journal_conflict("immutable_conflict") + return acknowledged + if getattr(exc, "status_code", None) in {409, 412}: + continue + raise + self._journal_conflict("etag_conflict") + def journal_page(self, kind, *, cursor=None, limit=50, execution_id=None): control = self._read_control() if kind not in {"execution", "attempt", "decision"} or type(limit) is not int or not 1 <= limit <= 100: @@ -170,7 +281,7 @@ def journal_page(self, kind, *, cursor=None, limit=50, execution_id=None): fields = PUBLIC_DECISION_FIELDS if kind == "decision" else PUBLIC_EXECUTION_FIELDS entries = [] for row in rows[:limit]: - entry = {name: deepcopy(value) for name, value in {**row["payload"], "sequence": row["sequence"]}.items() if name in fields} + entry = {name: _public_journal_value(value) for name, value in {**row["payload"], "sequence": row["sequence"]}.items() if name in fields} decision = entry.get("decision") if isinstance(decision, dict): decision = {name: value for name, value in decision.items() if name in {"choice", "target"}} @@ -219,9 +330,17 @@ def journal_decide(self, *, expected_version, gate_id, choice, actor_user_id, re pair = (gate.get("kind"), choice) if pair not in NEXT_STATE_BY_DECISION or choice not in gate.get("choices", []): self._journal_conflict("invalid_choice") - if choice in {"approve", "retry", "resume"} and self._now().isoformat() >= control["deadline_at"]: + if choice in {"approve", "retry", "resume", "continue_repeat"} and self._now().isoformat() >= control["deadline_at"]: self.expire_deadline() self._journal_conflict("deadline_exceeded") + repeat_row, repeat_head = None, None + if choice == "continue_repeat": + from functions_workflow_repeat_state import prepare_repeat_grant + + if int(control.get("admitted_count") or 0) >= control["max_executions"]: + self._replace(control, self._limit_pause(control, "execution_budget_exceeded")) + self._journal_conflict("execution_budget_exceeded") + repeat_row, repeat_head = prepare_repeat_grant(self, control, gate, actor_user_id=actor_user_id) decision = { **wanted, "gate_kind": gate["kind"], "decided_at": self._now().isoformat(), **{key: deepcopy(gate[key]) for key in ( @@ -229,6 +348,11 @@ def journal_decide(self, *, expected_version, gate_id, choice, actor_user_id, re "input_digest", "definition_revision", ) if key in gate}, } + if repeat_head is not None: + decision.update( + repeat=deepcopy(gate["repeat"]), reason_code="repeat_iteration_limit", request_id=request_id, + event_id=canonical_digest(["workflow_repeat_manually_continued", gate_id, request_id]), + ) active = self.journal_read("execution", gate.get("execution_id")) if gate.get("execution_id") else None if active: decision["consumed_inputs"] = deepcopy(active["payload"].get("consumed_inputs") or []) @@ -240,6 +364,30 @@ def journal_decide(self, *, expected_version, gate_id, choice, actor_user_id, re version=control["version"] + 1, journal_sequence=sequence + 2) counts = dict(control.get("journal_counts") or {}) operations = [("replace", (control["id"], replacement), {"if_match_etag": control["_etag"]})] + if repeat_head is not None: + from functions_workflow_repeat_state import repeat_summary + + body = {key: value for key, value in repeat_row.items() if not key.startswith("_")} + body["payload"] = repeat_head + operations.append(("replace", (body["id"], body), {"if_match_etag": repeat_row["_etag"]})) + replacement["repeat_progress"] = repeat_summary(repeat_head) + metrics = dict(control.get("repeat_counts") or {}) + metrics["continuation_count"] = int(metrics.get("continuation_count") or 0) + 1 + replacement["repeat_counts"] = metrics + elif choice in {"cancel", "reject"}: + repeat_cancellations = self._repeat_cancellation_operations(control, replacement) + operations.extend(repeat_cancellations) + if repeat_cancellations and active is not None: + attempt = self.journal_read("attempt", [active["payload"]["execution_id"], active["payload"]["attempt"]]) + for row in (active, attempt): + if row is None: + continue + body = {name: value for name, value in row.items() if not name.startswith("_")} + body["payload"] = { + **body["payload"], "state": "cancelled", "reason_code": "run_cancelled", + "completed_at": self._now().isoformat(), + } + operations.append(("replace", (row["id"], body), {"if_match_etag": row["_etag"]})) for offset, (kind, key, payload) in enumerate(( ("decision", ["gate", gate_id], decision), ("request", request_key, wanted), ), start=1): @@ -251,18 +399,63 @@ def journal_decide(self, *, expected_version, gate_id, choice, actor_user_id, re }) operations.append(("create", (body,))) replacement["journal_counts"] = counts + if choice in {"approve", "retry", "resume", "continue_repeat"} and self._now().isoformat() >= control["deadline_at"]: + self.expire_deadline() + self._journal_conflict("deadline_exceeded") try: self.container.execute_item_batch(batch_operations=operations, partition_key=self.identity["run_id"]) + if repeat_head is not None: + from functions_workflow_repeat_state import log_repeat_event + + log_repeat_event(self, decision) return self._read_control() except (cosmos_exceptions.CosmosBatchOperationError, cosmos_exceptions.CosmosHttpResponseError) as exc: if getattr(exc, "status_code", None) in {409, 412}: continue marker = self.journal_read("request", request_key) if marker and marker["payload"] == wanted: + if repeat_head is not None: + from functions_workflow_repeat_state import log_repeat_event + + log_repeat_event(self, decision) return self._read_control() raise self._journal_conflict("etag_conflict") + def _repeat_cancellation_operations(self, control, replacement): + from functions_workflow_repeat_state import repeat_summary + + cursor = control.get("cursor") or {} + path = cursor.get("iteration_path") or [] + identifiers = set() + if any("iteration" in frame for frame in path): + snapshot = self.run_definition() + identifiers.update( + workflow_execution_id(snapshot, self.identity["run_id"], frame["loop_id"], path[:index]) + for index, frame in enumerate(path) if "iteration" in frame + ) + if cursor.get("execution_id"): + identifiers.add(cursor["execution_id"]) + for summary in ((control.get("gate") or {}).get("repeat"), control.get("repeat_progress")): + if summary and summary.get("execution_id"): + identifiers.add(summary["execution_id"]) + operations = [] + for identifier in sorted(identifiers): + row = self.journal_read("loop", identifier) + if row is None or row["payload"].get("kind") != "repeat_until" or row["payload"].get("state") in {"completed", "cancelled"}: + continue + body = {name: value for name, value in row.items() if not name.startswith("_")} + body["payload"] = {**body["payload"], "state": "cancelled"} + operations.append(("replace", (row["id"], body), {"if_match_etag": row["_etag"]})) + if (control.get("repeat_progress") or {}).get("execution_id") == identifier: + replacement["repeat_progress"] = repeat_summary(body["payload"]) + iteration = self.journal_read("iteration", [identifier, row["payload"]["next_iteration"]]) + if iteration and iteration["payload"].get("state") == "running": + body = {name: value for name, value in iteration.items() if not name.startswith("_")} + body["payload"] = {**body["payload"], "state": "cancelled", "completed_at": self._now().isoformat()} + operations.append(("replace", (iteration["id"], body), {"if_match_etag": iteration["_etag"]})) + return operations + def journal_request(self, action, *, actor_user_id, request_id, expected_version=None): from functions_workflow_runtime_store import RESUMABLE_STATES, TERMINAL_STATES, _require_id @@ -308,6 +501,7 @@ def journal_request(self, action, *, actor_user_id, request_id, expected_version ("create", (body,)), ] if action == "cancel": + operations.extend(self._repeat_cancellation_operations(control, replacement)) node_id = (control.get("cursor") or {}).get("node_id") execution_id = (control.get("gate") or {}).get("execution_id") or (control.get("cursor") or {}).get("execution_id") if not execution_id and node_id: diff --git a/application/single_app/functions_workflow_limits.py b/application/single_app/functions_workflow_limits.py index 40297a537..573ff6abe 100644 --- a/application/single_app/functions_workflow_limits.py +++ b/application/single_app/functions_workflow_limits.py @@ -8,6 +8,11 @@ WORKFLOW_LOOP_ITEMS_MIN = 1 WORKFLOW_LOOP_ITEMS_MAX = 5000 WORKFLOW_LOOP_LIMIT_SETTING = "workflow_max_loop_items" +WORKFLOW_MAX_EXECUTION_ADMISSIONS = 5000 +WORKFLOW_REPEAT_ITERATIONS_DEFAULT = 25 +WORKFLOW_REPEAT_ITERATIONS_MIN = 1 +WORKFLOW_REPEAT_ITERATIONS_MAX = 1000 +WORKFLOW_REPEAT_LIMIT_SETTING = "workflow_max_repeat_iterations" class WorkflowLoopInputError(ValueError): @@ -72,6 +77,44 @@ def get_workflow_loop_item_limit(settings=None): return get_workflow_max_loop_items(settings) +def validate_workflow_max_repeat_iterations(value): + """Validate the separate ceiling for one automatically executed Repeat batch.""" + candidate = value + if isinstance(value, str): + text = value.strip() + candidate = ( + int(text) + if text.isascii() and text.isdecimal() and len(text) <= 10 + else None + ) + if ( + type(candidate) is not int + or not WORKFLOW_REPEAT_ITERATIONS_MIN <= candidate <= WORKFLOW_REPEAT_ITERATIONS_MAX + ): + raise WorkflowLoopLimitError( + "Workflow Repeat Iteration Limit must be a whole number from 1 to 1,000.", + code="workflow_repeat_limit_invalid", + ) + return candidate + + +def get_workflow_max_repeat_iterations(settings=None): + """Read new-run policy without changing an admitted run's frozen allowance.""" + if settings is None: + # Settings initialize application clients; load them only at a request boundary. + from functions_settings import get_settings + + settings = get_settings() + if not isinstance(settings, Mapping): + raise WorkflowLoopLimitError( + "The workflow Repeat limit is temporarily unavailable.", + code="workflow_repeat_limit_unavailable", + ) + return validate_workflow_max_repeat_iterations( + settings.get(WORKFLOW_REPEAT_LIMIT_SETTING, WORKFLOW_REPEAT_ITERATIONS_DEFAULT) + ) + + def effective_workflow_loop_limit(max_items=None, *, settings=None, policy=None): """Combine an authored ceiling with a run's admitted policy or current settings.""" if policy is None: diff --git a/application/single_app/functions_workflow_loop_runners.py b/application/single_app/functions_workflow_loop_runners.py index 441f28567..9a034a68c 100644 --- a/application/single_app/functions_workflow_loop_runners.py +++ b/application/single_app/functions_workflow_loop_runners.py @@ -14,7 +14,7 @@ def assert_workflow_loop_agent_type(agent_type): ) if execution is not None and (getattr(execution, "iteration_path", []) or reporting) and (agent_type or "local") != "local": raise WorkflowInputError( - "For each and saved-record reporting require locally metered models or local agents. Hosted agents are not supported for these steps.", + "For each, Repeat until, and saved-record reporting require locally metered models or local agents. Hosted agents are not supported for these steps.", ) @@ -29,7 +29,7 @@ def require_local_loop_runner(workflow, *, actor_user_id, settings, resolve_agen agent = resolve_agent(workflow.get("selected_agent") or {}, user_id=actor_user_id, settings=settings) if agent.get("agent_type", "local") != "local": raise WorkflowInputError( - "For each and saved-record reporting require locally metered models or local agents. Hosted agents are not supported for these steps.", + "For each, Repeat until, and saved-record reporting require locally metered models or local agents. Hosted agents are not supported for these steps.", ) @@ -40,7 +40,7 @@ def validate_workflow_loop_runners(workflow, *, actor_user_id, settings, resolve pending = [(node, False) for node in (workflow.get("flow") or {}).get("nodes") or []] while pending: node, inside = pending.pop() - if node["kind"] == "for_each": + if node["kind"] in {"for_each", "repeat_until"}: pending.extend((child, True) for child in node["body"]["nodes"]) elif node["kind"] == "if": pending.extend((child, inside) for name in ("then", "else") for child in node[name]["nodes"]) diff --git a/application/single_app/functions_workflow_node_results.py b/application/single_app/functions_workflow_node_results.py index ec8cd93b7..ee102b39c 100644 --- a/application/single_app/functions_workflow_node_results.py +++ b/application/single_app/functions_workflow_node_results.py @@ -7,7 +7,9 @@ from copy import deepcopy from functions_analysis_access import AnalysisResultUnavailable, analysis_source_snapshot, authorize_analysis_sources +from functions_workflow_bindings import WorkflowInputError from functions_workflow_identity import canonical_digest, workflow_node_identity +from functions_workflow_limits import WORKFLOW_MAX_EXECUTION_ADMISSIONS from functions_workflow_result_store import load_workflow_node_result @@ -56,139 +58,371 @@ def read_consumed_inputs(manifest, load_section): return list(iter_consumed_inputs(manifest, load_section)) -def authorize_workflow_node_result_read( - workflow, run_id, identity, reference, *, reader_user_id=None, manifest=None, - load_result=load_workflow_node_result, source_resolver=None, include_sources=True, source_callback=None, -): - root = manifest if manifest is not None else load_node_result(workflow, run_id, identity, reference, load_result=load_result) - active, visited, sources = set(), set(), {} - loaded = OrderedDict() - source_ids, source_batch = set(), [] - changed = False - - def flush_sources(): - nonlocal changed - if not source_batch: - return - checked = authorize_analysis_sources( - reader_user_id or workflow["user_id"], source_batch, resolver=source_resolver, - ) - changed |= checked["source_snapshot_changed"] - if source_callback is not None: - for source in source_batch: - source_callback(source) - source_batch.clear() +class WorkflowLineageAuthorization: + """One bounded iterative graph for results, mixed paths and temporal state proofs.""" + + def __init__(self, workflow, run_id, *, reader_user_id=None, load_result=load_workflow_node_result, + source_resolver=None, include_sources=True, source_callback=None, store=None): + self.workflow, self.run_id = workflow, run_id + self.reader_user_id = reader_user_id or workflow["user_id"] + self.load_result, self.source_resolver = load_result, source_resolver + self.include_sources, self.source_callback = include_sources, source_callback + self._store = store + self._compiled = None + self.loaded = OrderedDict() + self.active, self.visited = set(), set() + self.sources, self.source_ids, self.source_batch = {}, set(), [] + self.changed = False + self._recording = None + self._proof_cache = None + # Reuse immutable graph structure only within this active worker request; + # source, document and publication authority is never cached. + from functions_workflow_execution import current_workflow_execution + + execution = current_workflow_execution() + if execution and execution.run_id == run_id and execution.workflow == workflow: + self._proof_cache = getattr(execution, "lineage_proof_cache", None) + + @property + def store(self): + if self._store is None: + from functions_workflow_execution import current_workflow_execution + from functions_workflow_iterations import workflow_runtime_store + + current = current_workflow_execution() + self._store = ( + current.store if current and current.workflow["id"] == self.workflow["id"] and current.run_id == self.run_id + else workflow_runtime_store(self.workflow, self.run_id) + ) + return self._store + + @property + def compiled(self): + if self._compiled is None: + from functions_workflow_flow import compile_workflow_flow + + self._compiled = compile_workflow_flow(self.workflow) + return self._compiled + + def load(self, identity, reference): + key = (canonical_digest(identity), canonical_digest(reference)) + if key not in self.loaded: + self.loaded[key] = load_node_result( + self.workflow, self.run_id, identity, reference, load_result=self.load_result, + ) + if len(self.loaded) > 32: + self.loaded.popitem(last=False) + self.loaded.move_to_end(key) + return self.loaded[key] - def source_seen(source): + def _flush_sources(self): + if not self.source_batch: + return + checked = authorize_analysis_sources(self.reader_user_id, self.source_batch, resolver=self.source_resolver) + self.changed |= checked["source_snapshot_changed"] + if self.source_callback is not None: + for source in self.source_batch: + self.source_callback(source) + self.source_batch.clear() + + def source_seen(self, source): source = analysis_source_snapshot([source])[0] + if self._recording is not None: + self._recording["sources"].append(source) digest = canonical_digest(source) - if digest in source_ids: + if digest in self.source_ids: return - source_ids.add(digest) - if include_sources: - sources[digest] = source - source_batch.append(source) - if len(source_batch) >= 100: - flush_sources() - - def enter(producer, ref, current): - nonlocal changed - key = (canonical_digest(producer), canonical_digest(ref)) - if key in active or len(visited) + len(active) > 100000: - raise AnalysisResultUnavailable("analysis_lineage_invalid") - if key in visited: - return None - expected = workflow_node_identity( - workflow, run_id, producer.get("node_id"), producer.get("execution_id"), producer.get("attempt"), - task_id=producer.get("task_id"), iteration_path=producer.get("iteration_path"), - ) - if producer != expected or not isinstance(current, Mapping) or ( - current.get("contract_version") != "workflow-result-v2" or current.get("identity") != expected - ): - raise AnalysisResultUnavailable("analysis_lineage_invalid") - if current.get("publication"): - # Result readers own source lineage; publication owns the additional destination boundary. - from functions_artifact_publication import authorize_publication_status_read - from functions_workflow_runtime_store import workflow_runtime_store - - actor = workflow_runtime_store(workflow, run_id).read()["actor_user_id"] - authorize_publication_status_read( - reader_user_id or workflow["user_id"], current["publication"], actor_user_id=actor, - ) - if producer["iteration_path"]: - from functions_workflow_iterations import authorize_iteration_path + self.source_ids.add(digest) + if self.include_sources: + self.sources[digest] = source + self.source_batch.append(source) + if len(self.source_batch) >= 100: + self._flush_sources() - authorize_iteration_path( - workflow, run_id, producer, reader_user_id=reader_user_id or workflow["user_id"], - receipts=current.get("iteration_inputs") or [], - load_result=load_result, source_resolver=source_resolver, - ) - if current.get("frozen_loop"): - from functions_workflow_iterations import authorize_frozen_loop + def _document_access(self, item): + from functions_workflow_iterations import _authorize_frozen_document - frozen_access = authorize_frozen_loop( - workflow, run_id, current["frozen_loop"], - reader_user_id=reader_user_id or workflow["user_id"], - load_result=load_result, source_resolver=source_resolver, source_callback=source_seen, - ) - changed |= frozen_access["source_snapshot_changed"] - active.add(key) - access = current.get("analysis_access") - if access is not None: - if not isinstance(access, Mapping) or access.get("version") != "analysis-source-access-v1": + _authorize_frozen_document(self.workflow, item, self.reader_user_id) + if self._recording is not None: + self._recording["documents"].append(item) + + def _publication_access(self, publication): + from functions_artifact_publication import authorize_publication_status_read + + authorize_publication_status_read( + self.reader_user_id, publication, actor_user_id=self.store.read()["actor_user_id"], + ) + if self._recording is not None: + self._recording["publications"].append(publication) + + def _cached_children(self, key, proof): + cache = self._proof_cache + cached = cache["entries"].get(key) if cache is not None else None + if cached is not None: + cache["entries"].move_to_end(key) + entry, _ = cached + for source in entry["sources"]: + self.source_seen(source) + for item in entry["documents"]: + self._document_access(item) + for publication in entry["publications"]: + self._publication_access(publication) + yield from entry["dependencies"] + return + entry = {"dependencies": [], "sources": [], "documents": [], "publications": []} + children = iter(self._children(proof)) + while True: + self._recording = entry if cache is not None else None + try: + child = next(children, None) + finally: + self._recording = None + if child is None: + break + if cache is not None: + entry["dependencies"].append(child) + yield child + if cache is not None: + size = len(json.dumps(entry, ensure_ascii=True).encode("ascii")) + if size <= 1024 * 1024: + cache["entries"][key] = (entry, size) + cache["bytes"] += size + while cache["bytes"] > 32 * 1024 * 1024 or len(cache["entries"]) > 32768: + _, (_, removed_size) = cache["entries"].popitem(last=False) + cache["bytes"] -= removed_size + + def access(self): + self._flush_sources() + return { + "source_count": len(self.source_ids), "source_snapshot_changed": self.changed, + "sources": list(self.sources.values()) if self.include_sources else None, + } + + def walk(self, roots): + if self._proof_cache is not None: + self.store.read() + pending = [(None, iter(roots))] + while pending: + key, children = pending[-1] + proof = next(children, None) + if proof is None: + if key is not None: + self.active.remove(key) + self.visited.add(key) + pending.pop() + continue + child_key = canonical_digest(proof) + if child_key in self.active or len(self.visited) + len(self.active) >= 100000: raise AnalysisResultUnavailable("analysis_lineage_invalid") - direct = analysis_source_snapshot(access.get("sources")) - if not direct: - raise AnalysisResultUnavailable("analysis_source_manifest_missing") - for source in direct: - source_seen(source) - selected = set() - for descriptor in (current.get("outputs") or {}).values(): - if not isinstance(descriptor, Mapping): + if child_key in self.visited: + continue + self.active.add(child_key) + pending.append((child_key, iter(self._cached_children(child_key, proof)))) + self._flush_sources() + + def _children(self, proof): + kind, *arguments = proof + if kind == "result": + identity, reference = arguments + current = self.load(identity, reference) + expected = workflow_node_identity( + self.workflow, self.run_id, identity.get("node_id"), identity.get("execution_id"), identity.get("attempt"), + task_id=identity.get("task_id"), iteration_path=identity.get("iteration_path"), + ) + if ( + identity != expected or not isinstance(current, Mapping) + or current.get("contract_version") != "workflow-result-v2" or current.get("identity") != expected + ): raise AnalysisResultUnavailable("analysis_lineage_invalid") - if descriptor.get("selected_producer"): - receipt = descriptor["selected_producer"] - if not isinstance(receipt, Mapping) or receipt.get("output_ref") != descriptor.get("result_ref"): + if current.get("publication"): + self._publication_access(current["publication"]) + if identity["iteration_path"]: + yield ("path", identity, current.get("iteration_inputs") or []) + if current.get("frozen_loop"): + yield ("frozen", current["frozen_loop"]) + if current.get("repeat_state_proof"): + binding = current["repeat_state_proof"] + if binding.get("producer") != identity: + raise AnalysisResultUnavailable("workflow_repeat_identity_invalid") + yield ("state", identity, binding["state_ref"]) + access = current.get("analysis_access") + if access is not None: + if not isinstance(access, Mapping) or access.get("version") != "analysis-source-access-v1": raise AnalysisResultUnavailable("analysis_lineage_invalid") - selected.add(canonical_digest(receipt)) - consumed = iter_consumed_inputs( - current, lambda section: load_node_result(workflow, run_id, producer, section, load_result=load_result), - ) - return key, iter(consumed), selected - - pending = [enter(identity, reference, root)] - while pending: - key, children, selected = pending[-1] - item = next(children, None) - if item is None: + direct = analysis_source_snapshot(access.get("sources")) + if not direct: + raise AnalysisResultUnavailable("analysis_source_manifest_missing") + for source in direct: + self.source_seen(source) + selected = set() + for descriptor in (current.get("outputs") or {}).values(): + if not isinstance(descriptor, Mapping): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + if descriptor.get("selected_producer"): + receipt = descriptor["selected_producer"] + if not isinstance(receipt, Mapping) or receipt.get("output_ref") != descriptor.get("result_ref"): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + selected.add(canonical_digest(receipt)) + for receipt in iter_consumed_inputs(current, lambda section: self.load(identity, section)): + selected.discard(canonical_digest(receipt)) + yield ("receipt", receipt) if selected: raise AnalysisResultUnavailable("analysis_lineage_invalid") - active.remove(key) - visited.add(key) - pending.pop() - continue - selected.discard(canonical_digest(item)) - parent_identity, parent_ref = item.get("producer"), item.get("result_ref") - if not isinstance(parent_identity, dict) or not isinstance(parent_ref, dict): - raise AnalysisResultUnavailable("analysis_lineage_invalid") - parent_key = (canonical_digest(parent_identity), canonical_digest(parent_ref)) - parent = loaded.get(parent_key) - if parent is None: - parent = load_node_result(workflow, run_id, parent_identity, parent_ref, load_result=load_result) - loaded[parent_key] = parent - if len(loaded) > 8: - loaded.popitem(last=False) - output = (parent.get("outputs") or {}).get(item.get("output_name")) or {} - if output.get("result_ref") != item.get("output_ref"): + elif kind == "receipt": + receipt = arguments[0] + producer, reference = receipt.get("producer"), receipt.get("result_ref") + if not isinstance(producer, dict) or not isinstance(reference, dict): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + parent = self.load(producer, reference) + output = (parent.get("outputs") or {}).get(receipt.get("output_name")) or {} + if output.get("result_ref") != receipt.get("output_ref"): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + if receipt.get("repeat_state"): + from functions_workflow_repeat_state import load_repeat_admission, load_repeat_state + + binding = receipt["repeat_state"] + row = self.store.journal_read("loop", binding.get("loop_execution_id")) + identity = ((row or {}).get("payload") or {}).get("identity") + if not identity or identity["node_id"] != binding.get("loop_id"): + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + admission, expected = load_repeat_admission( + self.workflow, self.run_id, identity, binding.get("iteration"), store=self.store, + ) + if binding != {**expected, "state_name": binding.get("state_name")}: + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + state, _ = load_repeat_state( + self.workflow, self.run_id, identity, admission["before_state_ref"], + store=self.store, load_result=self.load_result, compiled=self.compiled, + ) + slot = (state["slots"].get(binding.get("state_name")) or {}).get("receipt") or {} + if any(receipt.get(name) != slot.get(name) for name in ("producer", "result_ref", "output_name", "output_ref")): + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + yield ("admission", identity, binding["iteration"]) + yield ("result", producer, reference) + elif kind == "state": + from functions_workflow_repeat_state import load_repeat_state + + identity, reference = arguments + state, _ = load_repeat_state( + self.workflow, self.run_id, identity, reference, + store=self.store, load_result=self.load_result, compiled=self.compiled, + ) + if identity["iteration_path"]: + yield ("path", identity, state.get("iteration_inputs") or []) + if state["state_index"]: + yield ("admission", identity, state["state_index"] - 1) + for slot in state["slots"].values(): + yield ("receipt", slot["receipt"]) + for receipt in state["body_outputs"].values(): + yield ("receipt", receipt) + for receipt in state.get("consumed_inputs") or []: + yield ("receipt", receipt) + elif kind == "admission": + from functions_workflow_repeat_state import load_repeat_admission + + identity, iteration = arguments + admission, _ = load_repeat_admission(self.workflow, self.run_id, identity, iteration, store=self.store) + yield ("state", identity, admission["before_state_ref"]) + elif kind == "path": + from functions_workflow_iterations import iteration_path_proofs + + identity, receipts = arguments + _, dependencies = iteration_path_proofs( + self.workflow, self.run_id, identity, receipts=receipts, store=self.store, load_result=self.load_result, + ) + yield from dependencies + elif kind in {"frozen", "frozen_item"}: + from functions_workflow_iterations import ( + load_frozen_loop, read_frozen_item, + ) + + binding = arguments[0] + if not isinstance(binding, dict) or not isinstance(binding.get("producer"), dict): + raise AnalysisResultUnavailable("workflow_loop_identity_invalid") + identity = binding["producer"] + manifest, reference, _ = load_frozen_loop( + self.workflow, self.run_id, identity, store=self.store, load_result=self.load_result, + ) + if reference != binding.get("manifest_ref"): + raise AnalysisResultUnavailable("workflow_loop_manifest_invalid") + if identity["iteration_path"]: + yield ("path", identity, manifest.get("iteration_inputs") or []) + for receipt in manifest.get("consumed_inputs") or []: + yield ("receipt", receipt) + if manifest.get("source_receipt"): + yield ("receipt", manifest["source_receipt"]) + indices = [arguments[1]] if kind == "frozen_item" else ( + range(manifest["count"]) if manifest["selection"]["kind"] != "input" else [] + ) + for index in indices: + item = read_frozen_item(self.workflow, self.run_id, manifest, index, load_result=self.load_result) + if kind == "frozen_item" and item["item_id"] != arguments[2]: + raise AnalysisResultUnavailable("workflow_loop_item_invalid") + if item["kind"] == "document": + self._document_access(item) + self.source_seen(item["source"]) + elif kind == "frozen_item": + receipt = manifest["source_receipt"] + producer, current, name = self._selected_output(receipt) + from functions_workflow_results import read_result_records + + rows, _ = read_result_records( + current, name, lambda ref: self.load(producer, ref), offset=item["source_ordinal"], limit=1, + ) + if len(rows) != 1 or canonical_digest(rows[0]) != item["record_sha256"]: + raise AnalysisResultUnavailable("workflow_loop_item_changed") + else: raise AnalysisResultUnavailable("analysis_lineage_invalid") - child = enter(parent_identity, parent_ref, parent) - if child is not None: - pending.append(child) - flush_sources() - return root, { - "source_count": len(source_ids), "source_snapshot_changed": changed, - "sources": list(sources.values()) if include_sources else None, - } + + def _selected_output(self, receipt): + seen = set() + while True: + key = canonical_digest(receipt) + if key in seen or len(seen) >= WORKFLOW_MAX_EXECUTION_ADMISSIONS: + raise AnalysisResultUnavailable("analysis_lineage_invalid") + seen.add(key) + identity = receipt["producer"] + manifest = self.load(identity, receipt["result_ref"]) + descriptor = manifest["outputs"].get(receipt["output_name"]) or {} + if descriptor.get("result_ref") != receipt.get("output_ref"): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + selected = descriptor.get("selected_producer") + if not selected: + return identity, manifest, receipt["output_name"] + receipt = selected + + def authorize_result(self, identity, reference, *, manifest=None): + if manifest is not None: + if self._proof_cache is not None: + key = canonical_digest(("result", identity, reference)) + removed = self._proof_cache["entries"].pop(key, None) + if removed is not None: + self._proof_cache["bytes"] -= removed[1] + self.loaded[(canonical_digest(identity), canonical_digest(reference))] = manifest + self.walk([("result", identity, reference)]) + return self.load(identity, reference) + + def authorize_repeat(self, identity, reference): + from functions_workflow_repeat_state import load_repeat_state + + self.walk([("state", identity, reference)]) + return load_repeat_state( + self.workflow, self.run_id, identity, reference, + store=self.store, load_result=self.load_result, compiled=self.compiled, + )[0] + + +def authorize_workflow_node_result_read( + workflow, run_id, identity, reference, *, reader_user_id=None, manifest=None, + load_result=load_workflow_node_result, source_resolver=None, include_sources=True, source_callback=None, + authorization=None, +): + authorization = authorization or WorkflowLineageAuthorization( + workflow, run_id, reader_user_id=reader_user_id, load_result=load_result, + source_resolver=source_resolver, include_sources=include_sources, source_callback=source_callback, + ) + root = authorization.authorize_result(identity, reference, manifest=manifest) + return root, authorization.access() class AuthorizedWorkflowRecordInput: @@ -229,32 +463,49 @@ def __init__(self, workflow, run_id, identity, reference, *, output_name="author } if self.access["source_count"]: self.receipt["analysis_result"] = True - self._selected = None - if descriptor.get("selected_producer"): + self._data_identity, self._data_manifest, self._data_name = self.identity, self.manifest, self.name + seen = set() + while descriptor.get("selected_producer"): selected = descriptor["selected_producer"] - self._selected = AuthorizedWorkflowRecordInput( - workflow, run_id, selected["producer"], selected["result_ref"], - output_name=selected["output_name"], reader_user_id=self.reader_user_id, - allow_partial=allow_partial, load_result=load_result, - source_resolver=source_resolver, inspection=inspection, + key = canonical_digest(selected) + if key in seen or len(seen) >= WORKFLOW_MAX_EXECUTION_ADMISSIONS: + raise AnalysisResultUnavailable("analysis_lineage_invalid") + seen.add(key) + self._data_identity = selected["producer"] + self._data_manifest = load_node_result( + workflow, run_id, self._data_identity, selected["result_ref"], load_result=load_result, ) - self.kind = self._selected.kind - self.record_count = self._selected.record_count + if not inspection: + _require_completed_result(self._data_manifest, allow_partial=allow_partial) + if (self._data_manifest.get("workflow_validation") or {}).get("eligible") is not True: + raise ValueError("The selected producer did not satisfy its output requirements.") + self._data_name = selected["output_name"] + descriptor = (self._data_manifest.get("outputs") or {}).get(self._data_name) or {} + if descriptor.get("result_ref") != selected.get("output_ref"): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + self.kind = descriptor.get("kind") + if self.kind not in {"records", "document_results"}: + raise ValueError("The selected input must be a complete record collection.") + if type(descriptor.get("record_count")) is int: + self.record_count = descriptor["record_count"] else: - self.kind = descriptor.get("kind") - if self.kind not in {"records", "document_results"}: - raise ValueError("The selected input must be a complete record collection.") - if type(descriptor.get("record_count")) is int: - self.record_count = descriptor["record_count"] - else: - _, self.record_count = read_result_records(self.manifest, self.name, self._load, offset=0, limit=1) + _, self.record_count = read_result_records( + self._data_manifest, self._data_name, self._load, offset=0, limit=1, + ) def _authorize(self): + authorization = WorkflowLineageAuthorization( + self.workflow, self.run_id, reader_user_id=self.reader_user_id, + load_result=self.load_result, source_resolver=self.source_resolver, include_sources=False, + ) manifest, access = authorize_workflow_node_result_read( self.workflow, self.run_id, self.identity, self.reference, reader_user_id=self.reader_user_id, load_result=self.load_result, - source_resolver=self.source_resolver, include_sources=False, + source_resolver=self.source_resolver, include_sources=False, authorization=authorization, ) + if getattr(self, "receipt", {}).get("repeat_state"): + authorization.walk([("receipt", self.receipt)]) + access = authorization.access() if access["source_snapshot_changed"] and not self.inspection: raise AnalysisResultUnavailable("analysis_source_snapshot_changed") return manifest, access @@ -263,7 +514,7 @@ def _load(self, reference): key = canonical_digest(reference) if key not in self._sections: self._sections[key] = load_node_result( - self.workflow, self.run_id, self.identity, reference, load_result=self.load_result, + self.workflow, self.run_id, self._data_identity, reference, load_result=self.load_result, ) if len(self._sections) > 4: self._sections.popitem(last=False) @@ -280,9 +531,7 @@ def read_records(self, *, offset=0, limit=100): if type(limit) is not int or not 1 <= limit <= 100: raise ValueError("Complete record pages support between 1 and 100 records.") self.recheck() - if self._selected is not None: - return self._selected.read_records(offset=offset, limit=limit) - rows, total = read_result_records(self.manifest, self.name, self._load, offset=offset, limit=limit) + rows, total = read_result_records(self._data_manifest, self._data_name, self._load, offset=offset, limit=limit) if total != self.record_count: raise ValueError("The complete record count changed.") return rows, total @@ -304,11 +553,9 @@ def record_page(self, *, offset=0, limit=100, max_bytes=240 * 1024): if type(offset) is not int or offset < 0 or type(limit) is not int or not 1 <= limit <= 100: raise ValueError("Invalid complete-record page.") self.recheck() - if self._selected is not None: - return self._selected.record_page(offset=offset, limit=limit, max_bytes=max_bytes) records, used = [], 2 for index in range(offset, min(self.record_count, offset + limit)): - rows, total = read_result_records(self.manifest, self.name, self._load, offset=index, limit=1) + rows, total = read_result_records(self._data_manifest, self._data_name, self._load, offset=index, limit=1) if len(rows) != 1 or total != self.record_count: raise ValueError("The complete record page is invalid.") size = len(json.dumps(rows[0], ensure_ascii=True, allow_nan=False).encode("ascii")) + 1 @@ -330,6 +577,7 @@ def open_workflow_record_input(workflow, run_id, identity, reference, **options) def load_workflow_node_input( workflow, run_id, identity, reference, *, output_name="authoritative", allow_partial=False, reader_user_id=None, load_result=load_workflow_node_result, source_resolver=None, required=True, + max_bytes=None, ): from functions_workflow_results import _require_completed_result, read_result_records @@ -355,15 +603,36 @@ def load_workflow_node_input( "output_ref": descriptor["result_ref"]} if access["source_count"]: receipt["analysis_result"] = True - loader = lambda ref: load_node_result(workflow, run_id, identity, ref, load_result=load_result) - if descriptor.get("selected_producer"): + value_receipt = receipt + seen = set() + while descriptor.get("selected_producer"): selected = descriptor["selected_producer"] - payload, _ = load_workflow_node_input( - workflow, run_id, selected["producer"], selected["result_ref"], - output_name=selected["output_name"], allow_partial=allow_partial, - reader_user_id=reader_user_id, load_result=load_result, source_resolver=source_resolver, + key = canonical_digest(selected) + if key in seen or len(seen) >= WORKFLOW_MAX_EXECUTION_ADMISSIONS: + raise AnalysisResultUnavailable("analysis_lineage_invalid") + seen.add(key) + identity, reference, name = selected["producer"], selected["result_ref"], selected["output_name"] + manifest = load_node_result(workflow, run_id, identity, reference, load_result=load_result) + _require_completed_result(manifest, allow_partial=allow_partial) + if (manifest.get("workflow_validation") or {}).get("eligible") is not True: + raise ValueError("The producer's output contract is not eligible.") + descriptor = (manifest.get("outputs") or {}).get(name) or {} + if descriptor.get("result_ref") != selected.get("output_ref"): + raise AnalysisResultUnavailable("analysis_lineage_invalid") + value_receipt = { + "producer": identity, "output_name": name, "result_ref": reference, "output_ref": descriptor["result_ref"], + } + if seen and receipt.get("analysis_result"): + _, leaf_access = authorize_workflow_node_result_read( + workflow, run_id, identity, reference, reader_user_id=reader_user_id, + load_result=load_result, source_resolver=source_resolver, include_sources=False, ) - return payload, receipt + if leaf_access["source_count"]: + value_receipt["analysis_result"] = True + loader = lambda ref: load_node_result(workflow, run_id, identity, ref, load_result=load_result) + if max_bytes is not None and descriptor.get("storage_kind") not in {"record_pages", "record_tree"}: + if type(descriptor["result_ref"].get("size_bytes")) is not int or descriptor["result_ref"]["size_bytes"] > max_bytes: + raise WorkflowInputError("This saved state is too large for a complete bounded read; its original data was retained.") if descriptor.get("storage_kind") in {"record_pages", "record_tree"}: value, _ = read_result_records(manifest, name, loader) output = {"producer": identity, "contract_version": "workflow-result-v2", @@ -376,7 +645,7 @@ def load_workflow_node_input( ): raise ValueError("Saved output section does not match its exact manifest.") return json.dumps({ - "consumed_result": receipt, "provenance": manifest.get("provenance") or {}, + "consumed_result": value_receipt, "provenance": manifest.get("provenance") or {}, "coverage": manifest.get("coverage") or {}, "validation": manifest.get("validation") or {}, "kind": output["kind"], "value": output["value"], "source_snapshot_changed": access["source_snapshot_changed"], diff --git a/application/single_app/functions_workflow_repeat_execution.py b/application/single_app/functions_workflow_repeat_execution.py new file mode 100644 index 000000000..6939e67b2 --- /dev/null +++ b/application/single_app/functions_workflow_repeat_execution.py @@ -0,0 +1,265 @@ +# functions_workflow_repeat_execution.py +"""Serial post-body Repeat traversal using the existing operation units and journal.""" + +import json +from copy import deepcopy + +from functions_analysis_access import AnalysisResultUnavailable +from functions_workflow_bindings import WorkflowInputError +from functions_workflow_collections import COLLECTION_MATERIALIZATION_BYTES +from functions_workflow_execution import WorkflowSuspended +from functions_workflow_flow import evaluate_predicate +from functions_workflow_identity import canonical_digest +from functions_workflow_node_results import ( + WorkflowLineageAuthorization, authorize_workflow_node_result_read, load_node_result, + load_workflow_node_input, result_selectors, +) +from functions_workflow_repeat_state import ( + _plain_receipt, load_repeat_admission, load_repeat_head, log_repeat_event, + prepare_repeat_state, repeat_identity, repeat_iteration_receipt, repeat_summary, +) +from functions_workflow_results import WorkflowResultNotReadyError, workflow_result_summary + + +def _boundary_payload(execution, node, region_id, identity, **fields): + previous = execution.store.journal_read("execution", identity["execution_id"]) + return { + **((previous or {}).get("payload") or {}), + "execution_id": identity["execution_id"], "node_id": node["id"], "node_kind": "repeat_until", + "region_id": region_id, "iteration_path": deepcopy(identity["iteration_path"]), + "iteration_inputs": deepcopy(execution.iteration_inputs), "attempt": 1, **fields, + } + + +def repeat_limit_gate(workflow, head, transition): + return { + "id": transition["gate_id"], "kind": "pause", "reason_code": "repeat_iteration_limit", + "unit_id": head["node_id"], "input_digest": canonical_digest(transition), + **result_selectors(head["identity"]), "definition_revision": workflow["definition_revision"], + "reason": "The stop condition is still unmet. Saved state is retained. Explicit continuation grants one more batch without resetting the run's admission or elapsed-time limits.", + "choices": ["continue_repeat", "cancel"], "repeat": repeat_summary(head), + } + + +def _final_result(flow, node, identity, state, reference, outputs, controls): + exports = {} + receipts = list(controls) + partial = state["partial"] + for export in node["exports"]: + receipt = outputs.get(export["output"]) + if receipt is None: + continue + manifest = load_node_result( + flow.workflow, flow.run_id, receipt["producer"], receipt["result_ref"], + load_result=flow.execution.load_result, + ) + descriptor = manifest["outputs"][receipt["output_name"]] + exports[export["name"]] = { + "kind": descriptor["kind"], "result_ref": deepcopy(receipt["output_ref"]), + "selected_producer": deepcopy(receipt), + } + receipts.append(receipt) + partial |= (manifest.get("workflow_validation") or {}).get("status") == "accepted_partial" + validation = { + "version": 1, "status": "accepted_partial" if partial else "valid", "eligible": True, + "reason_codes": ["repeat_partial_state_retained"] if partial else [], + } + manifest = { + "contract_version": "workflow-result-v2", "identity": deepcopy(identity), + "execution": {"status": "succeeded"}, "analysis_origin": False, + "authoritative_output": next(iter(exports), None), "outputs": exports, + "consumed_inputs": flow._receipts(receipts), + "iteration_inputs": deepcopy(flow.execution.iteration_inputs), + "repeat_state_proof": {"producer": deepcopy(identity), "state_ref": deepcopy(reference)}, + "workflow_validation": validation, "validation": {"status": "partial" if partial else "valid"}, + "coverage": {"status": "incomplete" if partial else "completed", "partial_coverage": partial}, + } + result_ref = flow.execution.save_result( + flow.workflow, flow.run_id, None, manifest, settings=flow.execution.settings, **result_selectors(identity), + ) + return workflow_result_summary(manifest, result_ref) + + +def run_repeat_until(flow, node, region_id): + execution, store = flow.execution, flow.execution.store + parent_path = deepcopy(execution.iteration_path) + parent_receipts = deepcopy(execution.iteration_inputs) + controls = list(flow.control_receipts) + identity = repeat_identity(flow.workflow, flow.run_id, node["id"], parent_path) + flow._admit_control(node) + existing = store.journal_read("loop", identity["execution_id"]) + if existing is None: + try: + initial, initial_ref = prepare_repeat_state(flow, node, identity) + except AnalysisResultUnavailable: + execution.pause_input("The Repeat state's original sources are no longer available.", code="workflow_repeat_source_unavailable") + except (WorkflowInputError, WorkflowResultNotReadyError, ValueError): + execution.pause_input( + "The required initial Repeat state did not satisfy its contract. Saved originals are retained.", + code="workflow_repeat_state_invalid", + ) + head = { + "kind": "repeat_until", "execution_id": identity["execution_id"], "node_id": node["id"], + "identity": identity, "initial_state_ref": initial_ref, "current_state_ref": initial_ref, + "next_iteration": 0, "completed_count": 0, "batch_number": 0, "batch_start_iteration": 0, + "batch_size": node["max_iterations"], "batch_usage": 0, "exhaustion_count": 0, + "continuation_count": 0, "grant_gate_id": None, "partial": initial["partial"], "state": "running", + } + store.journal_commit(execution.lease.token, "loop", identity["execution_id"], head, immutable=True) + _, head, _ = load_repeat_head(flow.workflow, flow.run_id, identity, store=store) + if head["state"] == "completed": + row = store.journal_read("execution", identity["execution_id"]) + summary = row["payload"]["workflow_result"] + authorize_workflow_node_result_read( + flow.workflow, flow.run_id, identity, summary["result_ref"], + reader_user_id=flow.actor_user_id, load_result=execution.load_result, + ) + flow.partial |= head["partial"] + flow._remember(node["id"], {"state": "completed", "summary": summary, "structured_validated": True}) + return + if head["state"] == "waiting_manual_continue": + transition = store.journal_read( + "decision", ["repeat-transition", identity["execution_id"], head["next_iteration"] - 1], + )["payload"] + store.wait(execution.lease.token, state="paused", gate=repeat_limit_gate(flow.workflow, head, transition)) + raise WorkflowSuspended("paused") + execution.record_execution(state="running", attempt=1, reason_code="") + execution._attempt(1, state="running") + while head["state"] == "running": + execution.set_node(node, region_id, iteration_path=parent_path, iteration_inputs=parent_receipts) + execution.check() + iteration = head["next_iteration"] + path = parent_path + [{"loop_id": node["id"], "iteration": iteration}] + admission_key = ["repeat-iteration", identity["execution_id"], iteration] + admitted = store.journal_read("admission", admission_key) + if admitted is None: + if head["batch_usage"] >= head["batch_size"]: + raise WorkflowInputError("This Repeat batch requires an explicit continuation grant.") + admission = { + "execution_id": identity["execution_id"], "node_id": node["id"], "iteration": iteration, + "iteration_path": path, "definition_revision": store.read()["definition_revision"], + "before_state_ref": deepcopy(head["current_state_ref"]), + "batch_number": head["batch_number"], "batch_start_iteration": head["batch_start_iteration"], + "batch_size": head["batch_size"], "batch_usage": head["batch_usage"] + 1, + "grant_gate_id": head["grant_gate_id"], + } + next_head = {**head, "batch_usage": head["batch_usage"] + 1} + store.journal_commit_many(execution.lease.token, [ + {"kind": "admission", "key": admission_key, "payload": admission, "immutable": True, "admission": True}, + {"kind": "iteration", "key": [identity["execution_id"], iteration], + "payload": {**admission, "state": "running"}, "expected": None}, + {"kind": "loop", "key": identity["execution_id"], "payload": next_head, "expected": head}, + ], updates={"cursor": execution.cursor(), "phase": node["id"], "repeat_progress": repeat_summary(next_head)}) + head = next_head + admission, iteration_receipt = load_repeat_admission(flow.workflow, flow.run_id, identity, iteration, store=store) + try: + authorization = WorkflowLineageAuthorization( + flow.workflow, flow.run_id, reader_user_id=flow.actor_user_id, store=store, load_result=execution.load_result, + ) + before = authorization.authorize_repeat(identity, admission["before_state_ref"]) + if authorization.access()["source_snapshot_changed"]: + raise AnalysisResultUnavailable("analysis_source_snapshot_changed") + execution.set_node( + None, node["body"]["id"], iteration_path=path, iteration_inputs=parent_receipts + [iteration_receipt], + ) + yield from flow._region(node["body"]) + resolved = flow.resolve(node["body"]["outputs"], metadata_only=True) + outputs = {receipt["input_name"]: _plain_receipt(receipt) for receipt in resolved["bound_inputs"]} + after, after_ref = prepare_repeat_state( + flow, node, identity, previous=before, previous_ref=admission["before_state_ref"], outputs=outputs, + consumed_inputs=resolved["consumed_inputs"], + ) + values = {} + for name in flow._predicate_names(node["until"]): + receipt = after["slots"][name]["receipt"] + payload, _ = load_workflow_node_input( + flow.workflow, flow.run_id, receipt["producer"], receipt["result_ref"], + output_name=receipt["output_name"], reader_user_id=flow.actor_user_id, + allow_partial=True, load_result=execution.load_result, max_bytes=COLLECTION_MATERIALIZATION_BYTES, + ) + values[name] = json.loads(payload)["value"] + condition = evaluate_predicate(node["until"], values) + except AnalysisResultUnavailable: + execution.set_node(node, region_id, iteration_path=parent_path, iteration_inputs=parent_receipts) + execution.pause_input( + "The Repeat state's original sources are no longer available. Saved originals are retained.", + code="workflow_repeat_source_unavailable", + ) + except (WorkflowInputError, WorkflowResultNotReadyError, ValueError): + execution.set_node(node, region_id, iteration_path=parent_path, iteration_inputs=parent_receipts) + execution.pause_input( + "The required Repeat state or stop condition could not be validated. Saved originals are retained.", + code="workflow_repeat_state_invalid", + ) + finally: + flow.control_receipts = list(controls) + execution.set_node(node, region_id, iteration_path=parent_path, iteration_inputs=parent_receipts) + exhausted = not condition and head["batch_usage"] == head["batch_size"] + outcome = "completed" if condition else "exhausted" if exhausted else "continue" + next_head = { + **head, "current_state_ref": after_ref, "next_iteration": iteration + 1, "completed_count": iteration + 1, + "partial": after["partial"], "state": "completed" if condition else "waiting_manual_continue" if exhausted else "running", + "exhaustion_count": head["exhaustion_count"] + int(exhausted), + } + transition = { + "decision_kind": "repeat_transition", "execution_id": identity["execution_id"], "node_id": node["id"], + "iteration_path": parent_path, "iteration_inputs": parent_receipts, "body_path": path, + "definition_revision": store.read()["definition_revision"], "attempt": 1, + "iteration": iteration, "batch_number": head["batch_number"], "batch_size": head["batch_size"], + "batch_usage": head["batch_usage"], "before_state_ref": admission["before_state_ref"], "after_state_ref": after_ref, + "body_outputs_sha256": canonical_digest(outputs), "predicate_sha256": canonical_digest(node["until"]), + "condition_result": condition, "outcome": outcome, "next_iteration": iteration + 1, + "decided_at": store._now().isoformat(), + } + if exhausted: + transition["gate_id"] = canonical_digest(["repeat-limit", identity, iteration, after_ref]) + transition["event_id"] = canonical_digest(["workflow_repeat_batch_exhausted", transition["gate_id"]]) + transition["reason_code"] = "repeat_iteration_limit" + transition["repeat"] = repeat_summary(next_head) + iteration_row = store.journal_read("iteration", [identity["execution_id"], iteration]) + rows = [ + {"kind": "decision", "key": ["repeat-transition", identity["execution_id"], iteration], + "payload": transition, "immutable": True}, + {"kind": "iteration", "key": [identity["execution_id"], iteration], "expected": iteration_row["payload"], + "payload": {**admission, "state": "completed_partial" if after["partial"] else "completed", + "after_state_ref": after_ref, "condition_result": condition, "completed_at": transition["decided_at"]}}, + {"kind": "loop", "key": identity["execution_id"], "payload": next_head, "expected": head}, + ] + updates = {"cursor": execution.cursor(), "repeat_progress": repeat_summary(next_head), "phase": node["id"]} + summary = None + if condition: + summary = _final_result(flow, node, identity, after, after_ref, outputs, controls) + payload = _boundary_payload( + execution, node, region_id, identity, state="completed", workflow_result=summary, + workflow_validation=summary["workflow_validation"], consumed_inputs=summary["consumed_inputs"], + structured_validated=True, reason_code="", completed_at=transition["decided_at"], + ) + rows.extend([ + {"kind": "execution", "key": identity["execution_id"], "payload": payload}, + {"kind": "attempt", "key": [identity["execution_id"], 1], "payload": payload}, + ]) + elif exhausted: + updates.update(state="paused", lease=None, gate=repeat_limit_gate(flow.workflow, next_head, transition)) + payload = _boundary_payload( + execution, node, region_id, identity, state="paused", reason_code="repeat_iteration_limit", + ) + rows.extend([ + {"kind": "execution", "key": identity["execution_id"], "payload": payload}, + {"kind": "attempt", "key": [identity["execution_id"], 1], "payload": payload}, + ]) + store.journal_commit_many( + execution.lease.token, rows, updates=updates, counters={"exhaustion_count": 1} if exhausted else None, + ) + head = next_head + flow.partial |= after["partial"] + flow.task_results[:] = [ + result for result in flow.task_results + if not ((result.get("result") or {}).get("workflow_result") or {}).get("producer", {}).get("iteration_path") + and not result.get("iteration_path") + ] + if exhausted: + log_repeat_event(store, transition) + raise WorkflowSuspended("paused") + if condition: + flow._remember(node["id"], {"state": "completed", "summary": summary, "structured_validated": True}) + return diff --git a/application/single_app/functions_workflow_repeat_history.py b/application/single_app/functions_workflow_repeat_history.py new file mode 100644 index 000000000..66ed1f833 --- /dev/null +++ b/application/single_app/functions_workflow_repeat_history.py @@ -0,0 +1,163 @@ +# functions_workflow_repeat_history.py +"""Authorized, bounded Repeat round and state-slot metadata inspection.""" + +import base64 +import binascii +import json + +from functions_workflow_execution_history import authorize_execution_payload +from functions_workflow_flow import compile_workflow_flow +from functions_workflow_identity import canonical_digest, workflow_execution_id +from functions_workflow_loop_history import _next_cursor, _page_position +from functions_workflow_node_results import WorkflowLineageAuthorization +from functions_workflow_repeat_state import load_repeat_admission, load_repeat_head, repeat_summary +from functions_workflow_runtime_store import workflow_runtime_store + + +def _repeat(workflow, run_id, execution_id, reader_user_id): + store = workflow_runtime_store(workflow, run_id) + workflow = store.run_definition() + row = store.journal_read("loop", execution_id) + if row is None or row["payload"].get("kind") != "repeat_until": + raise LookupError("Repeat execution not found.") + identity = row["payload"]["identity"] + node, head, _ = load_repeat_head(workflow, run_id, identity, store=store) + authorization = WorkflowLineageAuthorization(workflow, run_id, reader_user_id=reader_user_id, store=store) + return workflow, store, node, head, identity, authorization + + +def workflow_repeat_iterations_page(workflow, run_id, execution_id, *, reader_user_id, cursor=None, limit=50): + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Repeat pages require between 1 and 100 rounds.") + workflow, store, node, head, identity, authorization = _repeat(workflow, run_id, execution_id, reader_user_id) + scope = canonical_digest({"producer": identity, "initial_state_ref": head["initial_state_ref"], "kind": "repeat_iterations"}) + running = store.journal_read("admission", ["repeat-iteration", execution_id, head["next_iteration"]]) + total = head["next_iteration"] + int(running is not None) + offset, through = 0, total + anchor = None + if cursor: + try: + if not isinstance(cursor, str) or len(cursor) > 1024: + raise ValueError + value = json.loads(base64.urlsafe_b64decode(cursor.encode("ascii"))) + if set(value) != {"scope", "offset", "through", "anchor"} or value["scope"] != scope: + raise ValueError + offset, through, anchor = value["offset"], value["through"], value["anchor"] + if type(offset) is not int or type(through) is not int or not 0 <= offset < through <= total: + raise ValueError + except (ValueError, TypeError, UnicodeError, binascii.Error) as exc: + raise ValueError("Invalid Repeat iteration cursor.") from exc + if through: + last, _ = load_repeat_admission(workflow, run_id, identity, through - 1, store=store) + expected_anchor = canonical_digest(last) + if anchor is not None and anchor != expected_anchor: + raise ValueError("The Repeat iteration snapshot changed.") + anchor = expected_anchor + authorization.authorize_repeat(identity, head["initial_state_ref"]) + compiled = compile_workflow_flow(workflow) + ancestors = [frame["loop_id"] for frame in identity["iteration_path"]] + [node["id"]] + body_nodes = [ + node_id for node_id, enclosing in compiled["node_loop_ids"].items() + if enclosing == ancestors and node_id in compiled["nodes"] + ] + items, used = [], 2 + for iteration in range(offset, min(through, offset + limit)): + admission, _ = load_repeat_admission(workflow, run_id, identity, iteration, store=store) + authorization.walk([("admission", identity, iteration)]) + row = store.journal_read("iteration", [execution_id, iteration]) + payload = row["payload"] + if payload.get("after_state_ref"): + authorization.authorize_repeat(identity, payload["after_state_ref"]) + executions = [] + for node_id in body_nodes: + child_id = workflow_execution_id(workflow, run_id, node_id, admission["iteration_path"]) + child = store.journal_read("execution", child_id) + if child is not None: + authorize_execution_payload( + workflow, run_id, child["payload"], reader_user_id=reader_user_id, authorization=authorization, + ) + executions.append(child_id) + item = { + "iteration": iteration, "iteration_path": admission["iteration_path"], + "batch_number": admission["batch_number"], "batch_size": admission["batch_size"], + "batch_usage": admission["batch_usage"], "state": payload["state"], + "condition_result": payload.get("condition_result"), "execution_ids": executions, + "before_available": True, "after_available": bool(payload.get("after_state_ref")), + "partial": payload["state"] == "completed_partial", + } + size = len(json.dumps(item, ensure_ascii=True).encode("ascii")) + 1 + if used + size > 240 * 1024: + if not items: + raise ValueError("This round's complete inspection metadata is too large.") + break + items.append(item) + used += size + following = offset + len(items) + next_cursor = None + if following < through: + next_cursor = base64.urlsafe_b64encode(json.dumps({ + "scope": scope, "offset": following, "through": through, "anchor": anchor, + }, separators=(",", ":")).encode("ascii")).decode("ascii") + return { + "iterations": items, "total_count": through, "next_cursor": next_cursor, + "repeat_execution_id": execution_id, "repeat": repeat_summary(head), + "source_snapshot_changed": authorization.access()["source_snapshot_changed"], + } + + +def workflow_repeat_state_page(workflow, run_id, execution_id, iteration, *, reader_user_id, + phase="before", cursor=None, limit=50): + if phase not in {"before", "after"} or type(iteration) is not int or iteration < 0: + raise ValueError("Select an exact Repeat round and before or after state.") + workflow, store, node, _, identity, authorization = _repeat(workflow, run_id, execution_id, reader_user_id) + if store.journal_read("admission", ["repeat-iteration", execution_id, iteration]) is None: + raise LookupError("This Repeat round was not admitted.") + admission, _ = load_repeat_admission(workflow, run_id, identity, iteration, store=store) + payload = store.journal_read("iteration", [execution_id, iteration])["payload"] + reference = admission["before_state_ref"] if phase == "before" else payload.get("after_state_ref") + result = {"iteration": iteration, "phase": phase, "repeat_execution_id": execution_id} + if reference is None: + _page_position({"producer": identity, "iteration": iteration, "phase": phase}, cursor, limit) + if cursor: + raise ValueError("An uncommitted state has no page cursor.") + authorization.authorize_repeat(identity, admission["before_state_ref"]) + return {**result, "states": [], "available": False, "total_count": 0, "next_cursor": None} + state = authorization.authorize_repeat(identity, reference) + scope = {"producer": identity, "iteration": iteration, "phase": phase, "state_ref": reference, "kind": "repeat_state"} + offset = _page_position(scope, cursor, limit) + declarations = node["state"] + if offset > len(declarations): + raise ValueError("The state cursor exceeds its immutable snapshot.") + states, used = [], 2 + for declaration in declarations[offset:offset + limit]: + slot = state["slots"][declaration["name"]] + receipt = slot["receipt"] + source = { + **{name: value for name, value in receipt["producer"].items() + if name in {"node_id", "execution_id", "task_id", "iteration_path", "attempt"}}, + "output_name": receipt["output_name"], + } + item = { + "name": declaration["name"], "kind": slot["kind"], "source": source, + "workflow_validation": slot["workflow_validation"], + "coverage": {name: value for name, value in slot.get("coverage", {}).items() + if type(value) in {str, int, bool} or value is None}, + "limitations": slot.get("limitations") or [], + } + if slot.get("prior_coverage"): + item["prior_coverage"] = { + name: value for name, value in slot["prior_coverage"].items() + if type(value) in {str, int, bool} or value is None + } + size = len(json.dumps(item, ensure_ascii=True).encode("ascii")) + 1 + if used + size > 240 * 1024: + if not states: + raise ValueError("This state's complete inspection metadata is too large.") + break + states.append(item) + used += size + return { + **result, "states": states, "available": True, "total_count": len(declarations), + "next_cursor": _next_cursor(scope, offset + len(states), len(declarations)), + "partial": state["partial"], "source_snapshot_changed": authorization.access()["source_snapshot_changed"], + } diff --git a/application/single_app/functions_workflow_repeat_state.py b/application/single_app/functions_workflow_repeat_state.py new file mode 100644 index 000000000..ac57d7e58 --- /dev/null +++ b/application/single_app/functions_workflow_repeat_state.py @@ -0,0 +1,521 @@ +# functions_workflow_repeat_state.py +"""Exact, reference-only state and sealed lifetime admissions for Repeat until. + +Compiler, validator, reader and logger imports are deferred at bound operations +to avoid the existing definition/runtime/result import cycles and client startup. +""" + +import json +from copy import deepcopy + +from functions_analysis_access import AnalysisResultUnavailable +from functions_workflow_bindings import WorkflowInputError +from functions_workflow_identity import canonical_digest, workflow_execution_id, workflow_node_identity +from functions_workflow_limits import WORKFLOW_MAX_EXECUTION_ADMISSIONS, WORKFLOW_REPEAT_ITERATIONS_MAX +from functions_workflow_result_store import load_workflow_node_result, _quota_bytes +from functions_workflow_runtime_store import WorkflowRuntimeConflict, workflow_runtime_store + + +REPEAT_STATE_VERSION = "workflow-repeat-state-v1" +REPEAT_SUMMARY_FIELDS = frozenset({ + "execution_id", "node_id", "completed_iteration", "next_iteration", "batch_number", "batch_size", + "batch_usage", "completed_count", "exhaustion_count", "continuation_count", "state", "partial", +}) + + +def repeat_summary(head): + return { + **{name: deepcopy(head[name]) for name in REPEAT_SUMMARY_FIELDS if name in head}, + "completed_iteration": head["completed_count"] - 1, + } + + +def validate_repeat_gate_summary(value): + if ( + not isinstance(value, dict) or set(value) != REPEAT_SUMMARY_FIELDS + or any(not isinstance(value.get(name), str) or not value[name] for name in ("execution_id", "node_id")) + or any(type(value.get(name)) is not int or value[name] < 0 for name in ( + "completed_iteration", "next_iteration", "batch_number", "batch_size", "batch_usage", + "completed_count", "exhaustion_count", "continuation_count", + )) + or not 1 <= value["batch_size"] <= WORKFLOW_REPEAT_ITERATIONS_MAX + or value["batch_usage"] != value["batch_size"] + or value["next_iteration"] != value["completed_count"] + or value["completed_iteration"] + 1 != value["next_iteration"] + or value["state"] != "waiting_manual_continue" or type(value["partial"]) is not bool + ): + raise WorkflowRuntimeConflict("invalid_repeat_gate") + return value + + +def _selectors(identity): + return {name: identity[name] for name in ("node_id", "execution_id", "iteration_path", "attempt")} + + +def repeat_identity(workflow, run_id, node_id, parent_path): + return workflow_node_identity( + workflow, run_id, node_id, workflow_execution_id(workflow, run_id, node_id, parent_path), + 1, iteration_path=parent_path, + ) + + +def repeat_node(workflow, node_id): + pending = list(workflow["flow"]["nodes"]) + while pending: + node = pending.pop() + if node["id"] == node_id and node["kind"] == "repeat_until": + return node + if node["kind"] in {"for_each", "repeat_until"}: + pending.extend(node["body"]["nodes"]) + elif node["kind"] == "if": + pending.extend(node["then"]["nodes"]) + pending.extend(node["else"]["nodes"]) + raise AnalysisResultUnavailable("workflow_repeat_identity_invalid") + + +def load_repeat_head(workflow, run_id, identity, *, store=None): + store = store or workflow_runtime_store(workflow, run_id) + if identity != repeat_identity(workflow, run_id, identity["node_id"], identity["iteration_path"]): + raise AnalysisResultUnavailable("workflow_repeat_identity_invalid") + node = repeat_node(workflow, identity["node_id"]) + row = store.journal_read("loop", identity["execution_id"]) + head = (row or {}).get("payload") or {} + policy = store.read().get("repeat_policy") or {} + if ( + head.get("kind") != "repeat_until" or head.get("identity") != identity + or head.get("batch_size") != node["max_iterations"] or policy.get("version") != 1 + or type(policy.get("max_iterations")) is not int + or not node["max_iterations"] <= policy["max_iterations"] <= WORKFLOW_REPEAT_ITERATIONS_MAX + or any(type(head.get(name)) is not int or not 0 <= head[name] <= WORKFLOW_MAX_EXECUTION_ADMISSIONS for name in ( + "next_iteration", "completed_count", "batch_number", "batch_start_iteration", + "batch_usage", "exhaustion_count", "continuation_count", + )) + or head["completed_count"] != head["next_iteration"] + or head["batch_usage"] > head["batch_size"] + or head["batch_number"] != head["continuation_count"] + or head["batch_start_iteration"] != head["batch_number"] * head["batch_size"] + or not head["batch_start_iteration"] <= head["next_iteration"] <= head["batch_start_iteration"] + head["batch_usage"] + or head["batch_start_iteration"] + head["batch_usage"] - head["next_iteration"] not in {0, 1} + or head.get("state") not in {"running", "waiting_manual_continue", "completed", "cancelled"} + or type(head.get("partial")) is not bool + or not isinstance(head.get("initial_state_ref"), dict) or not isinstance(head.get("current_state_ref"), dict) + ): + raise AnalysisResultUnavailable("workflow_repeat_head_invalid") + return node, head, row + + +def repeat_iteration_receipt(identity, iteration, state_ref): + return { + "loop_id": identity["node_id"], "loop_execution_id": identity["execution_id"], + "iteration": iteration, "state_ref": deepcopy(state_ref), + } + + +def _transition(workflow, run_id, identity, iteration, *, store): + row = store.journal_read("decision", ["repeat-transition", identity["execution_id"], iteration]) + value = (row or {}).get("payload") or {} + node = repeat_node(workflow, identity["node_id"]) + expected_path = identity["iteration_path"] + [{"loop_id": identity["node_id"], "iteration": iteration}] + batch, used = divmod(iteration, node["max_iterations"]) + if ( + value.get("decision_kind") != "repeat_transition" + or value.get("execution_id") != identity["execution_id"] or value.get("node_id") != identity["node_id"] + or value.get("definition_revision") != store.read()["definition_revision"] + or value.get("iteration") != iteration or value.get("body_path") != expected_path + or value.get("iteration_path") != identity["iteration_path"] + or value.get("predicate_sha256") != canonical_digest(node["until"]) + or value.get("batch_number") != batch or value.get("batch_usage") != used + 1 + or value.get("batch_size") != node["max_iterations"] + or type(value.get("condition_result")) is not bool + or value.get("next_iteration") != iteration + 1 + or not isinstance(value.get("before_state_ref"), dict) or not isinstance(value.get("after_state_ref"), dict) + or value.get("outcome") not in {"continue", "exhausted", "completed"} + or (value["outcome"] == "completed") != value["condition_result"] + or not value["condition_result"] and (value["outcome"] == "exhausted") != (used + 1 == node["max_iterations"]) + ): + raise AnalysisResultUnavailable("workflow_repeat_transition_invalid") + if value["outcome"] == "exhausted" and ( + value.get("gate_id") != canonical_digest(["repeat-limit", identity, iteration, value["after_state_ref"]]) + or value.get("reason_code") != "repeat_iteration_limit" + ): + raise AnalysisResultUnavailable("workflow_repeat_transition_invalid") + return value + + +def load_repeat_admission(workflow, run_id, identity, iteration, *, store=None): + store = store or workflow_runtime_store(workflow, run_id) + node, head, _ = load_repeat_head(workflow, run_id, identity, store=store) + if type(iteration) is not int or not 0 <= iteration < store.read()["max_executions"]: + raise AnalysisResultUnavailable("workflow_repeat_iteration_invalid") + row = store.journal_read("admission", ["repeat-iteration", identity["execution_id"], iteration]) + admission = (row or {}).get("payload") or {} + batch, usage = divmod(iteration, node["max_iterations"]) + path = identity["iteration_path"] + [{"loop_id": identity["node_id"], "iteration": iteration}] + if ( + admission.get("execution_id") != identity["execution_id"] or admission.get("node_id") != node["id"] + or admission.get("iteration") != iteration or admission.get("iteration_path") != path + or admission.get("definition_revision") != store.read()["definition_revision"] + or admission.get("batch_number") != batch or admission.get("batch_usage") != usage + 1 + or admission.get("batch_size") != node["max_iterations"] + or admission.get("batch_start_iteration") != batch * node["max_iterations"] + or batch > head["batch_number"] or iteration > head["next_iteration"] + or not isinstance(admission.get("before_state_ref"), dict) + ): + raise AnalysisResultUnavailable("workflow_repeat_admission_invalid") + if iteration == 0: + expected_ref = head["initial_state_ref"] + else: + previous = _transition(workflow, run_id, identity, iteration - 1, store=store) + if previous["condition_result"] or previous["outcome"] == "exhausted" and usage != 0: + raise AnalysisResultUnavailable("workflow_repeat_admission_invalid") + expected_ref = previous["after_state_ref"] + if admission["before_state_ref"] != expected_ref: + raise AnalysisResultUnavailable("workflow_repeat_state_changed") + if batch: + gate_id = admission.get("grant_gate_id") + decision = store.journal_read("decision", ["gate", gate_id]) if gate_id else None + grant = (decision or {}).get("payload") or {} + exhausted = _transition(workflow, run_id, identity, batch * node["max_iterations"] - 1, store=store) + if ( + grant.get("choice") != "continue_repeat" or grant.get("gate_id") != gate_id + or grant.get("execution_id") != identity["execution_id"] + or grant.get("definition_revision") != admission["definition_revision"] + or grant.get("input_digest") != canonical_digest(exhausted) + or (grant.get("repeat") or {}).get("batch_number") != batch - 1 + or exhausted["outcome"] != "exhausted" + or exhausted.get("gate_id") != gate_id + ): + raise AnalysisResultUnavailable("workflow_repeat_grant_invalid") + elif admission.get("grant_gate_id") is not None: + raise AnalysisResultUnavailable("workflow_repeat_grant_invalid") + outcome = store.journal_read("iteration", [identity["execution_id"], iteration]) + if outcome is None or any(outcome["payload"].get(name) != value for name, value in admission.items()): + raise AnalysisResultUnavailable("workflow_repeat_iteration_invalid") + return admission, repeat_iteration_receipt(identity, iteration, expected_ref) + + +def _plain_receipt(receipt): + return {name: deepcopy(value) for name, value in receipt.items() if name not in {"input_name", "control"}} + + +def _check_source(workflow, run_id, source, receipt, path, *, compiled, store, load_result): + producer = receipt.get("producer") or {} + if source["kind"] == "repeat_state": + binding = receipt.get("repeat_state") or {} + frame_index = next((index for index, frame in enumerate(path) if frame["loop_id"] == source["loop_id"]), None) + if ( + frame_index is None or "iteration" not in path[frame_index] + or binding.get("loop_id") != source["loop_id"] or binding.get("state_name") != source["state_name"] + or binding.get("iteration") != path[frame_index]["iteration"] + ): + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + enclosing = repeat_identity(workflow, run_id, source["loop_id"], path[:frame_index]) + if binding.get("loop_execution_id") != enclosing["execution_id"]: + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + admission, _ = load_repeat_admission(workflow, run_id, enclosing, binding["iteration"], store=store) + if binding.get("state_ref") != admission["before_state_ref"]: + raise AnalysisResultUnavailable("workflow_repeat_state_changed") + return + ancestors = compiled["node_loop_ids"][source["node_id"]] + expected_path = path[:len(ancestors)] + if ( + [frame["loop_id"] for frame in expected_path] != ancestors + or producer.get("node_id") != source["node_id"] or producer.get("iteration_path") != expected_path + or producer.get("execution_id") != workflow_execution_id(workflow, run_id, source["node_id"], expected_path) + ): + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + committed = store.journal_read("attempt", [producer["execution_id"], producer.get("attempt")]) + summary = ((committed or {}).get("payload") or {}).get("workflow_result") or {} + name = summary.get("authoritative_output") if source["output"] == "authoritative" else source["output"] + if ( + summary.get("producer") != producer or summary.get("result_ref") != receipt.get("result_ref") + or receipt.get("output_name") != name + or (summary.get("outputs", {}).get(name) or {}).get("result_ref") != receipt.get("output_ref") + ): + raise AnalysisResultUnavailable("workflow_repeat_source_uncommitted") + + +def load_repeat_state(workflow, run_id, identity, reference, *, store=None, + load_result=load_workflow_node_result, compiled=None): + # Compiler imports definition normalization; defer it until a bound read. + from functions_workflow_flow import compile_workflow_flow + + store = store or workflow_runtime_store(workflow, run_id) + compiled = compiled or compile_workflow_flow(workflow) + node, head, _ = load_repeat_head(workflow, run_id, identity, store=store) + state = load_result(workflow, run_id, None, reference, **_selectors(identity)) + index = state.get("state_index") + slots = state.get("slots") + if ( + state.get("contract_version") != REPEAT_STATE_VERSION or state.get("identity") != identity + or type(index) is not int or not 0 <= index <= head["next_iteration"] + or not isinstance(slots, dict) or set(slots) != {slot["name"] for slot in node["state"]} + or type(state.get("partial")) is not bool + ): + raise AnalysisResultUnavailable("workflow_repeat_state_invalid") + transition = None + if index == 0: + if reference != head["initial_state_ref"] or state.get("previous_state_ref") is not None: + raise AnalysisResultUnavailable("workflow_repeat_state_uncommitted") + path = identity["iteration_path"] + else: + transition = _transition(workflow, run_id, identity, index - 1, store=store) + if reference != transition["after_state_ref"] or state.get("previous_state_ref") != transition["before_state_ref"]: + raise AnalysisResultUnavailable("workflow_repeat_state_uncommitted") + admission, _ = load_repeat_admission(workflow, run_id, identity, index - 1, store=store) + if transition["before_state_ref"] != admission["before_state_ref"]: + raise AnalysisResultUnavailable("workflow_repeat_transition_invalid") + path = admission["iteration_path"] + outputs = state.get("body_outputs") + declared = {binding["name"]: binding for binding in node["body"]["outputs"]} + if ( + not isinstance(outputs, dict) or set(outputs) - set(declared) + or transition.get("body_outputs_sha256") != canonical_digest(outputs) + ): + raise AnalysisResultUnavailable("workflow_repeat_state_invalid") + for name, binding in declared.items(): + if name not in outputs: + if binding["required"]: + raise AnalysisResultUnavailable("workflow_repeat_state_invalid") + continue + _check_source(workflow, run_id, binding["source"], outputs[name], path, + compiled=compiled, store=store, load_result=load_result) + for declaration in node["state"]: + saved = slots[declaration["name"]] + validation = saved.get("workflow_validation") or {} + if ( + saved.get("kind") != declaration["output_contract"]["kind"] + or saved.get("contract_sha256") != canonical_digest(declaration["output_contract"]) + or not isinstance(saved.get("receipt"), dict) + or validation.get("version") != 1 or validation.get("eligible") is not True + or validation.get("status") not in {"valid", "accepted_partial"} + or validation["status"] == "accepted_partial" and not declaration["output_contract"]["allow_partial"] + ): + raise AnalysisResultUnavailable("workflow_repeat_state_invalid") + if index == 0: + _check_source(workflow, run_id, declaration["initial"], saved["receipt"], path, + compiled=compiled, store=store, load_result=load_result) + elif saved["receipt"] != state["body_outputs"].get(declaration["next"]): + raise AnalysisResultUnavailable("workflow_repeat_state_invalid") + if any(slot["workflow_validation"]["status"] == "accepted_partial" for slot in slots.values()) and not state["partial"]: + raise AnalysisResultUnavailable("workflow_repeat_state_invalid") + return state, transition + + +def current_repeat_state(workflow, run_id, path, source, *, reader_user_id, store, + load_result=load_workflow_node_result): + from functions_workflow_node_results import WorkflowLineageAuthorization + + position = next((index for index, frame in enumerate(path) if frame["loop_id"] == source["loop_id"]), None) + if position is None or "iteration" not in path[position]: + raise WorkflowInputError("Current Repeat state is outside its admitted body.") + identity = repeat_identity(workflow, run_id, source["loop_id"], path[:position]) + iteration = path[position]["iteration"] + admission, _ = load_repeat_admission(workflow, run_id, identity, iteration, store=store) + authorization = WorkflowLineageAuthorization( + workflow, run_id, reader_user_id=reader_user_id, load_result=load_result, store=store, + ) + state = authorization.authorize_repeat(identity, admission["before_state_ref"]) + if authorization.access()["source_snapshot_changed"]: + raise AnalysisResultUnavailable("analysis_source_snapshot_changed") + slot = state["slots"].get(source["state_name"]) + if slot is None: + raise WorkflowInputError("The requested Repeat state slot is not declared.") + receipt = { + **_plain_receipt(slot["receipt"]), + "repeat_state": { + **repeat_iteration_receipt(identity, iteration, admission["before_state_ref"]), + "state_name": source["state_name"], + }, + } + return slot, receipt + + +def validate_repeat_source_receipt(workflow, run_id, source, receipt, path, *, store, + load_result=load_workflow_node_result): + position = next((index for index, frame in enumerate(path) if frame["loop_id"] == source["loop_id"]), None) + if position is None or "iteration" not in path[position]: + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + identity = repeat_identity(workflow, run_id, source["loop_id"], path[:position]) + admission, expected = load_repeat_admission(workflow, run_id, identity, path[position]["iteration"], store=store) + state, _ = load_repeat_state( + workflow, run_id, identity, admission["before_state_ref"], store=store, load_result=load_result, + ) + selected = (state["slots"].get(source["state_name"]) or {}).get("receipt") or {} + if ( + receipt.get("repeat_state") != {**expected, "state_name": source["state_name"]} + or any(receipt.get(name) != selected.get(name) for name in ("producer", "result_ref", "output_name", "output_ref")) + ): + raise AnalysisResultUnavailable("workflow_repeat_source_invalid") + + +def prepare_repeat_state(flow, node, identity, *, previous=None, previous_ref=None, outputs=None, consumed_inputs=None): + from functions_workflow_collect import _CollectionContract + from functions_workflow_collections import ( + COLLECTION_MATERIALIZATION_BYTES, CollectionWriteBudget, RecordIdentityValidator, + ) + from functions_workflow_node_results import ( + authorize_workflow_node_result_read, load_node_result, load_workflow_node_input, open_workflow_record_input, + ) + from functions_workflow_results import _encoded_result_size, _require_completed_result + from functions_workflow_validation import validate_workflow_task_output, workflow_coverage_status + + execution = flow.execution + slots = {} + outputs = outputs or {} + maximum = _quota_bytes(execution.settings) + budget = CollectionWriteBudget(maximum) + + def save(value): + execution.check() + return execution.save_result( + flow.workflow, flow.run_id, None, value, settings=execution.settings, **_selectors(identity), + ) + + for declaration in node["state"]: + contract = declaration["output_contract"] + if previous is None: + resolved = flow.resolve([{ + "name": declaration["name"], "source": declaration["initial"], "required": True, + "expected_kind": contract["kind"], "allow_partial": contract["allow_partial"], + }], metadata_only=True) + receipt = _plain_receipt(resolved["bound_inputs"][0]) + else: + receipt = deepcopy(outputs.get(declaration["next"])) + if receipt is None: + raise WorkflowInputError("A required next-state output did not finish.") + manifest, access = authorize_workflow_node_result_read( + flow.workflow, flow.run_id, receipt["producer"], receipt["result_ref"], + reader_user_id=flow.actor_user_id, load_result=execution.load_result, + ) + _require_completed_result(manifest, allow_partial=contract["allow_partial"]) + if access["source_snapshot_changed"]: + raise AnalysisResultUnavailable("analysis_source_snapshot_changed") + descriptor = manifest["outputs"].get(receipt["output_name"]) or {} + if descriptor.get("result_ref") != receipt["output_ref"] or descriptor.get("kind") != contract["kind"]: + raise WorkflowInputError("Repeat state must preserve its exact declared output kind.") + coverage = deepcopy(manifest.get("coverage") or {}) + inherited = (previous or {}).get("slots", {}).get(declaration["name"], {}) + producer_partial = (manifest.get("workflow_validation") or {}).get("status") == "accepted_partial" + inherited_partial = (inherited.get("workflow_validation") or {}).get("status") == "accepted_partial" + if contract["kind"] in {"records", "document_results"}: + reader = open_workflow_record_input( + flow.workflow, flow.run_id, receipt["producer"], receipt["result_ref"], + output_name=receipt["output_name"], reader_user_id=flow.actor_user_id, + allow_partial=contract["allow_partial"], load_result=execution.load_result, + ) + validator = _CollectionContract(contract) + identities = None + if contract.get("identity_field"): + identities = RecordIdentityValidator( + contract["identity_field"], identity, save, + lambda ref: execution.load_result(flow.workflow, flow.run_id, None, ref, **_selectors(identity)), + max_result_bytes=maximum, budget=budget, contract_version=REPEAT_STATE_VERSION, + output_name=f"state:{declaration['name']}:identity", + ) + for record in reader.iter_records(): + execution.check() + validator.add(record) + if identities is not None: + identities.add(record) + incomplete = ["producer_coverage_incomplete"] if producer_partial or inherited_partial else [] + if contract["require_complete_coverage"] and workflow_coverage_status(coverage) != "complete": + incomplete.append(f"coverage_{workflow_coverage_status(coverage)}") + validation = validator.finish( + invalid=[], incomplete=incomplete, identities=identities.finish() if identities else None, + ) + else: + payload, _ = load_workflow_node_input( + flow.workflow, flow.run_id, receipt["producer"], receipt["result_ref"], + output_name=receipt["output_name"], reader_user_id=flow.actor_user_id, + allow_partial=contract["allow_partial"], load_result=execution.load_result, + max_bytes=COLLECTION_MATERIALIZATION_BYTES, + ) + output = json.loads(payload) + validation = validate_workflow_task_output({ + **manifest, "authoritative_output": "state", + "outputs": {"state": {"kind": output["kind"], "value": output["value"]}}, + "validation": {"status": "accepted_partial"} if producer_partial or inherited_partial else manifest.get("validation", {}), + }, contract) + if validation["eligible"] is not True: + raise WorkflowInputError("Repeat state did not satisfy its declared output contract; the original output was retained.") + limitations = list(dict.fromkeys([ + *(inherited.get("limitations") or []), *(validation.get("reason_codes") or []), + *((manifest.get("workflow_validation") or {}).get("reason_codes") or []), + ])) + slots[declaration["name"]] = { + "kind": contract["kind"], "contract_sha256": canonical_digest(contract), "receipt": receipt, + "workflow_validation": validation, "coverage": coverage, "limitations": limitations, + **({"prior_coverage": inherited.get("prior_coverage") or inherited.get("coverage") or {}} + if inherited_partial else {}), + } + outputs_partial = any( + (load_node_result( + flow.workflow, flow.run_id, receipt["producer"], receipt["result_ref"], load_result=execution.load_result, + ).get("workflow_validation") or {}).get("status") == "accepted_partial" + for receipt in outputs.values() + ) + state = { + "contract_version": REPEAT_STATE_VERSION, "identity": deepcopy(identity), + "state_index": 0 if previous is None else previous["state_index"] + 1, + "slots": slots, "previous_state_ref": deepcopy(previous_ref), "body_outputs": deepcopy(outputs), + "consumed_inputs": deepcopy(flow.control_receipts if consumed_inputs is None else consumed_inputs), + "iteration_inputs": deepcopy(execution.iteration_inputs[:len(identity["iteration_path"])]), + "partial": bool((previous or {}).get("partial")) or outputs_partial or any( + slot["workflow_validation"]["status"] == "accepted_partial" for slot in slots.values() + ), + } + budget.consume(_encoded_result_size(state)) + return state, save(state) + + +def prepare_repeat_grant(store, control, gate, *, actor_user_id): + from functions_workflow_node_results import WorkflowLineageAuthorization + + if gate.get("reason_code") != "repeat_iteration_limit" or gate.get("choices") != ["continue_repeat", "cancel"]: + raise WorkflowRuntimeConflict("invalid_repeat_gate") + validate_repeat_gate_summary(gate.get("repeat")) + workflow = store.run_definition() + identity = repeat_identity(workflow, store.identity["run_id"], gate["node_id"], gate["iteration_path"]) + _, head, row = load_repeat_head(workflow, store.identity["run_id"], identity, store=store) + transition = _transition(workflow, store.identity["run_id"], identity, head["next_iteration"] - 1, store=store) + if ( + control["state"] != "paused" or head["state"] != "waiting_manual_continue" + or gate.get("execution_id") != identity["execution_id"] or gate.get("attempt") != 1 + or gate.get("definition_revision") != control["definition_revision"] + or gate["repeat"] != repeat_summary(head) or gate.get("input_digest") != canonical_digest(transition) + or transition.get("gate_id") != gate["id"] or transition["outcome"] != "exhausted" + or head["current_state_ref"] != transition["after_state_ref"] + or head["batch_usage"] != head["batch_size"] + ): + raise WorkflowRuntimeConflict("repeat_gate_changed") + for reader in dict.fromkeys([actor_user_id, control["actor_user_id"]]): + authorization = WorkflowLineageAuthorization( + workflow, store.identity["run_id"], reader_user_id=reader, store=store, + ) + authorization.authorize_repeat(identity, head["current_state_ref"]) + if authorization.access()["source_snapshot_changed"]: + raise AnalysisResultUnavailable("analysis_source_snapshot_changed") + return row, { + **head, "state": "running", "batch_number": head["batch_number"] + 1, + "batch_start_iteration": head["next_iteration"], "batch_usage": 0, + "continuation_count": head["continuation_count"] + 1, "grant_gate_id": gate["id"], + } + + +def log_repeat_event(store, decision): + # Logging initialization depends on application configuration; use the existing + # logger only after an authoritative journal decision has committed. + from functions_appinsights import log_event + + event = "workflow_repeat_manually_continued" if decision.get("choice") == "continue_repeat" else "workflow_repeat_batch_exhausted" + log_event( + f"[WORKFLOW_SCHEDULER] {event}", + extra={ + "event_name": event, "event_id": decision["event_id"], + **{name: value for name, value in store.identity.items() if name in {"workflow_id", "run_id", "scope_type", "scope_id"}}, + **{name: decision[name] for name in ("execution_id", "node_id", "gate_id", "request_id", "actor_user_id", "decided_at") if name in decision}, + **{name: value for name, value in (decision.get("repeat") or {}).items() + if name in {"batch_number", "batch_size", "completed_count", "exhaustion_count", "continuation_count"}}, + }, + ) diff --git a/application/single_app/functions_workflow_results.py b/application/single_app/functions_workflow_results.py index 5c364f8d7..a350bc54e 100644 --- a/application/single_app/functions_workflow_results.py +++ b/application/single_app/functions_workflow_results.py @@ -522,10 +522,14 @@ def cached_load(bound_workflow, bound_run_id, task_id, reference, **selectors): ) if structured_run: from functions_workflow_execution_history import workflow_execution_history + from functions_workflow_node_results import WorkflowLineageAuthorization cursor = None + authorization = WorkflowLineageAuthorization(workflow, run_id, reader_user_id=reader_user_id, store=store) while True: - page = workflow_execution_history(workflow, run_id, reader_user_id=reader_user_id, cursor=cursor, limit=100) + page = workflow_execution_history( + workflow, run_id, reader_user_id=reader_user_id, cursor=cursor, limit=100, authorization=authorization, + ) cursor = page["next_cursor"] if cursor is None: break diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 4be576747..95a9066e4 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -6046,7 +6046,9 @@ def _add_workflow_activity_thought( activity_key = f"{activity_key}:{identity['execution_id']}:{identity['attempt']}" lane_key = identity['execution_id'] lane_label = ' / '.join( - f"{frame['loop_id']} item {frame['index'] + 1}" for frame in identity['iteration_path'] + f"{frame['loop_id']} round {frame['iteration'] + 1}" if 'iteration' in frame + else f"{frame['loop_id']} item {frame['index'] + 1}" + for frame in identity['iteration_path'] ) return thought_tracker.add_thought( step_type, @@ -10350,6 +10352,9 @@ def raise_if_cancelled(): if callable(cancel_check): cancel_check(workflow, run_id) + has_repeat = flow_runner is not None and any( + entry['node']['kind'] == 'repeat_until' for entry in flow_runner.compiled['nodes'].values() + ) for task_index, raw_task in enumerate(flow_runner.tasks() if flow_runner else tasks): raise_if_cancelled() task = dict(raw_task or {}) @@ -10357,6 +10362,10 @@ def raise_if_cancelled(): task_id = str(task.get('id') or f'task-{task_index + 1}').strip() task['id'] = task_id task_unit_key = f'task:{task_id}' + if has_repeat: + # Resume skips sealed rounds; retain the original ordinal rather than + # weakening native/task checkpoint fingerprints when enumeration changes. + task['order'] = durable.cache(f'task-order:{task_id}', {'order': task['order']})['order'] if durable is not None: checkpoint = durable.snapshot(f'task-result:{task_id}') if checkpoint is not None: @@ -10400,11 +10409,11 @@ def raise_if_cancelled(): workflow, run_id, step_type='task', - content=f"Starting task {task_index + 1}: {task.get('name') or task_id}", + content=f"Starting task {task['order']}: {task.get('name') or task_id}", detail=None, activity_key=f'task:{run_id}:{task_id}', kind='workflow_task', - title=str(task.get('name') or f'Task {task_index + 1}'), + title=str(task.get('name') or f"Task {task['order']}"), status='running', ) task_result = None @@ -10675,11 +10684,11 @@ def dispatch_task(): workflow, run_id, step_type='task', - content=f"Retrying task {task_index + 1}: {task.get('name') or task_id}", + content=f"Retrying task {task['order']}: {task.get('name') or task_id}", detail=f'attempt={attempt_count + 1}', activity_key=f'task:{run_id}:{task_id}', kind='workflow_task', - title=str(task.get('name') or f'Task {task_index + 1}'), + title=str(task.get('name') or f"Task {task['order']}"), status='running', ) @@ -10892,11 +10901,11 @@ def save_analysis_section(bound_workflow, bound_run, bound_task, section, **_kwa workflow, run_id, step_type='task', - content=f"Finished task {task_index + 1}: {task.get('name') or task_id}", + content=f"Finished task {task['order']}: {task.get('name') or task_id}", detail=task_error or f'attempts={attempt_count}; validation={validation["status"]}', activity_key=f'task:{run_id}:{task_id}', kind='workflow_task', - title=str(task.get('name') or f'Task {task_index + 1}'), + title=str(task.get('name') or f"Task {task['order']}"), status='completed' if validation['eligible'] else 'failed', ) completed_results[task_id] = { @@ -10956,11 +10965,11 @@ def save_analysis_section(bound_workflow, bound_run, bound_task, section, **_kwa workflow, run_id, step_type='task', - content=f"Failed task {task_index + 1}: {task.get('name') or task_id}", + content=f"Failed task {task['order']}: {task.get('name') or task_id}", detail=task_error, activity_key=f'task:{run_id}:{task_id}', kind='workflow_task', - title=str(task.get('name') or f'Task {task_index + 1}'), + title=str(task.get('name') or f"Task {task['order']}"), status='failed', ) if error_strategy != 'continue': diff --git a/application/single_app/functions_workflow_runtime.py b/application/single_app/functions_workflow_runtime.py index bed0e2274..f5c5aa1b1 100644 --- a/application/single_app/functions_workflow_runtime.py +++ b/application/single_app/functions_workflow_runtime.py @@ -30,6 +30,7 @@ RuntimeUnavailable, workflow_runtime_projection, workflow_runtime_store, + validate_repeat_admission_policy, ) @@ -133,8 +134,9 @@ def queue_durable_workflow_run(workflow, *, actor_user_id, trigger_source="manua _authorize_execution(current, actor_user_id, settings) if type(current.get("definition_version", 1)) is not int or current.get("definition_version", 1) not in {1, 2, 3}: raise ValueError("This workflow definition requires a newer execution engine.") + compiled = None if current.get("definition_version") == 3: - compile_workflow_flow(current) + compiled = compile_workflow_flow(current) from functions_workflow_loop_runners import validate_workflow_loop_runners validate_workflow_loop_runners(current, actor_user_id=actor_user_id, settings=settings) @@ -157,21 +159,30 @@ def queue_durable_workflow_run(workflow, *, actor_user_id, trigger_source="manua raise WorkflowRuntimeConflict("tombstoned", "This workflow run was deleted.") snapshot_ref = existing_control["snapshot_ref"] snapshot = store.run_definition() if existing_control.get("schema_version") == 2 else load_workflow_task_result(current, run_id, "runtime:definition", snapshot_ref) - else: + loop_policy, repeat_policy = None, None + if snapshot.get("definition_version") == 3: + from functions_workflow_limits import get_workflow_loop_item_limit, get_workflow_max_repeat_iterations + + loop_policy = existing_control.get("loop_policy") if existing_control is not None else {"max_items": get_workflow_loop_item_limit(settings)} + repeat_policy = ( + existing_control.get("repeat_policy") if existing_control is not None + else {"max_iterations": get_workflow_max_repeat_iterations(settings)} + if any(entry["node"]["kind"] == "repeat_until" for entry in compiled["nodes"].values()) + else None + ) + if existing_control is None: + repeat_policy = validate_repeat_admission_policy(compiled, repeat_policy) + if existing_control is None: snapshot_ref = ( save_workflow_runtime_result(snapshot, run_id, snapshot, settings=settings) if snapshot.get("definition_version") == 3 else save_workflow_task_result(current, run_id, "runtime:definition", snapshot, settings=settings) ) - loop_policy = None - if snapshot.get("definition_version") == 3: - from functions_workflow_limits import get_workflow_loop_item_limit - - loop_policy = {"max_items": get_workflow_loop_item_limit(settings)} control = store.initialize( snapshot_ref=snapshot_ref, definition_revision=snapshot["definition_revision"], actor_user_id=actor_user_id, request_id=request_id, **({"loop_policy": loop_policy} if loop_policy is not None else {}), + **({"repeat_policy": repeat_policy} if repeat_policy is not None else {}), ) if control["state"] in RUNTIME_TERMINAL_STATES: run = services["runs"].read_item(item=run_id, partition_key=services["partition"]) diff --git a/application/single_app/functions_workflow_runtime_store.py b/application/single_app/functions_workflow_runtime_store.py index 3a1de7d52..d4948c14a 100644 --- a/application/single_app/functions_workflow_runtime_store.py +++ b/application/single_app/functions_workflow_runtime_store.py @@ -22,6 +22,7 @@ from functions_workflow_journal import WorkflowJournalMixin from functions_workflow_identity import workflow_execution_id from functions_artifact_publication_readiness import public_publication_status +from functions_workflow_limits import WORKFLOW_REPEAT_ITERATIONS_DEFAULT, WORKFLOW_REPEAT_ITERATIONS_MAX CONTROL_ID = "workflow-runtime:v1" @@ -54,6 +55,7 @@ "reference_snapshot_ref", "metadata", "loop_progress", + "repeat_progress", }) IDENTITY_KEYS = frozenset({"workflow_id", "user_id", "group_id", "scope_type", "scope_id", "run_id"}) FORBIDDEN_PAYLOAD_KEY_PARTS = ("token", "secret", "password", "connection") @@ -79,6 +81,8 @@ "retryable", "metadata", "publication", + "reason_code", + "repeat", }) GATE_KIND_BY_STATE = { "waiting_approval": "approval", @@ -99,6 +103,7 @@ ("recovery", "cancel"): "cancelled", ("pause", "resume"): "queued", ("pause", "cancel"): "cancelled", + ("pause", "continue_repeat"): "queued", } @@ -114,6 +119,24 @@ def __init__(self, code, public_message="Workflow runtime changed. Reload and tr RuntimeConflict = WorkflowRuntimeConflict +def validate_repeat_admission_policy(compiled, policy=None): + repeats = [entry["node"] for entry in compiled["nodes"].values() if entry["node"]["kind"] == "repeat_until"] + if not repeats: + return None + if policy is not None and not isinstance(policy, dict): + raise RuntimeConflict("invalid_repeat_policy") + maximum = (policy or {}).get("max_iterations", WORKFLOW_REPEAT_ITERATIONS_DEFAULT) + if type(maximum) is not int or not 1 <= maximum <= WORKFLOW_REPEAT_ITERATIONS_MAX: + raise RuntimeConflict("invalid_repeat_policy") + if any(node["max_iterations"] > maximum for node in repeats): + raise RuntimeConflict( + "repeat_policy_exceeded", + "A Repeat maximum exceeds the administrator's current iteration limit. " + "Change the authored maximum or ask an administrator to change the policy before starting a new run.", + ) + return {"version": 1, "max_iterations": maximum} + + class RuntimeUnavailable(RuntimeError): """A safe storage availability wrapper which does not expose provider text.""" @@ -283,7 +306,16 @@ def _validate_gate(gate, state): if kind != expected_kind: raise RuntimeConflict("invalid_gate", "Workflow runtime gate kind does not match state.") choices = gate.get("choices") - allowed_choices = CHOICES_BY_GATE_KIND[kind] + repeat_limit = gate.get("reason_code") == "repeat_iteration_limit" + allowed_choices = frozenset({"continue_repeat", "cancel"}) if repeat_limit else CHOICES_BY_GATE_KIND[kind] + if repeat_limit: + from functions_workflow_repeat_state import validate_repeat_gate_summary + + if kind != "pause" or choices != ["continue_repeat", "cancel"]: + raise RuntimeConflict("invalid_gate", "Repeat continuation requires its exact exhaustion gate.") + validate_repeat_gate_summary(gate.get("repeat")) + elif "repeat" in gate: + raise RuntimeConflict("invalid_gate", "Repeat state requires an exhaustion gate.") if choices is None: choices = sorted(allowed_choices) if not isinstance(choices, list) or any(not _valid_id(choice, max_length=64) for choice in choices): @@ -447,6 +479,8 @@ def public_projection(control): "phase": control.get("phase"), "progress": control.get("progress"), **({"loop_progress": deepcopy(control["loop_progress"])} if control.get("loop_progress") else {}), + **({"repeat_progress": deepcopy(control["repeat_progress"])} if control.get("repeat_progress") else {}), + **({"repeat_counts": deepcopy(control["repeat_counts"])} if control.get("repeat_counts") else {}), **({ "limits": { "max_executions": control["max_executions"], @@ -455,6 +489,7 @@ def public_projection(control): "deadline_seconds": control["deadline_seconds"], "waits_count": True, **({"max_loop_items": control["loop_policy"]["max_items"]} if control.get("loop_policy") else {}), + **({"max_repeat_iterations": control["repeat_policy"]["max_iterations"]} if control.get("repeat_policy") else {}), }, } if control.get("schema_version") == 2 else {}), "deleted": bool(control.get("deleted")), @@ -594,7 +629,11 @@ def read(self, *, allow_deleted=False): def expire_deadline(self): def mutator(current): deadline = _parse_timestamp(current.get("deadline_at")) - if current.get("schema_version") != 2 or deadline is None or self._now() < deadline or current["state"] in TERMINAL_STATES | {"paused"}: + if ( + current.get("schema_version") != 2 or deadline is None or self._now() < deadline + or current["state"] in TERMINAL_STATES + or current["state"] == "paused" and (current.get("gate") or {}).get("reason_code") != "repeat_iteration_limit" + ): return NO_WRITE return self._limit_pause(current, "deadline_exceeded") @@ -606,6 +645,7 @@ def _limit_pause(self, current, code): replacement.update(state="paused", phase=code, lease=None, version=current["version"] + 1, gate={ "id": uuid.uuid4().hex, "kind": "pause", "unit_id": node_id or "run-limits", "input_digest": current["definition_revision"], "choices": ["cancel"], + "reason_code": code, "reason": ( "The elapsed workflow deadline was reached, including time spent waiting. Cancel and start a new run." if code == "deadline_exceeded" else @@ -678,9 +718,24 @@ def write_record(self, token, record, *, immutable=False): raise raise RuntimeConflict("etag_conflict", "Workflow runtime changed concurrently. Retry the operation.") - def initialize(self, *, snapshot_ref, definition_revision, actor_user_id, request_id, loop_policy=None): + def initialize(self, *, snapshot_ref, definition_revision, actor_user_id, request_id, loop_policy=None, repeat_policy=None): actor_user_id = _require_id(actor_user_id, "actor_user_id") request_id = _require_id(request_id, "request_id") + try: + existing = self._read_control(allow_deleted=True) + except RuntimeConflict as exc: + if exc.code != "not_found": + raise + existing = None + if existing is not None: + if existing.get("deleted"): + raise RuntimeConflict("tombstoned", "Workflow runtime control was deleted.") + if ( + existing.get("request_id") != request_id or existing.get("snapshot_ref") != snapshot_ref + or existing.get("definition_revision") != definition_revision or existing.get("actor_user_id") != actor_user_id + ): + raise RuntimeConflict("initialize_conflict", "Workflow runtime was already initialized.") + return existing timestamp = _iso(self._now()) control = { "id": CONTROL_ID, @@ -722,6 +777,10 @@ def initialize(self, *, snapshot_ref, definition_revision, actor_user_id, reques if type(maximum) is not int or not 1 <= maximum <= 5000: raise RuntimeConflict("invalid_loop_policy") control["loop_policy"] = {"version": 1, "max_items": maximum} + admitted_repeat_policy = validate_repeat_admission_policy(compiled, repeat_policy) + if admitted_repeat_policy is not None: + control["repeat_policy"] = admitted_repeat_policy + control["repeat_counts"] = {"exhaustion_count": 0, "continuation_count": 0} _bounded_json_copy(control) try: saved = self.container.create_item(body=control) diff --git a/application/single_app/functions_workflow_structured_execution.py b/application/single_app/functions_workflow_structured_execution.py index ca353af9c..c39186872 100644 --- a/application/single_app/functions_workflow_structured_execution.py +++ b/application/single_app/functions_workflow_structured_execution.py @@ -1,8 +1,10 @@ # functions_workflow_structured_execution.py """Schema-2 operation boundaries using paged units rather than a growing control map.""" +from collections import OrderedDict from copy import deepcopy +from functions_analysis_access import AnalysisResultUnavailable from functions_workflow_execution import DurableWorkflowExecution, WorkflowSuspended, execution_fingerprint from functions_workflow_identity import workflow_execution_id from functions_workflow_runtime_store import WorkflowRuntimeConflict @@ -18,6 +20,7 @@ def __init__(self, *args, **kwargs): self.region_id = self.workflow["flow"]["id"] self.iteration_path = [] self.iteration_inputs = [] + self.lineage_proof_cache = {"entries": OrderedDict(), "bytes": 0} def set_node(self, node, region_id, *, iteration_path=None, iteration_inputs=None): self.node = node @@ -192,6 +195,7 @@ def record_execution(self, **fields): def run_unit(self, key, operation, *, inputs, replay_safe=False, approval=None): self.check() + self.authorize_iteration() digest = execution_fingerprint(inputs) unit = self.unit(key) if unit and unit.get("input_digest") != digest: @@ -239,10 +243,12 @@ def run_unit(self, key, operation, *, inputs, replay_safe=False, approval=None): consumed_inputs=inputs.get("consumed_inputs") or []) self._attempt(attempt, state="running") self.store.journal_commit(self.lease.token, "unit", self._key(key), unit) + self.authorize_iteration() try: result = operation() except Exception: self.check() + self.authorize_iteration() self.store.journal_commit(self.lease.token, "unit", self._key(key), {**unit, "state": "failed"}) if task_operation: self.record_execution(state="failed", attempt=attempt) @@ -259,6 +265,22 @@ def run_unit(self, key, operation, *, inputs, replay_safe=False, approval=None): }) return result + def authorize_iteration(self): + if any("iteration" in frame for frame in self.iteration_path): + from functions_workflow_iterations import authorize_iteration_path + + try: + authorize_iteration_path( + self.workflow, self.run_id, self.selectors(), + reader_user_id=self.store.read()["actor_user_id"], receipts=self.iteration_inputs, + store=self.store, load_result=self.load_result, + ) + except AnalysisResultUnavailable: + self.pause_input( + "The Repeat state's original sources are no longer available. Saved originals are retained.", + code="workflow_repeat_source_unavailable", + ) + def _attempt(self, attempt, **fields): row = self.store.journal_read("execution", self.execution_id()) payload = {**(row["payload"] if row else {}), "attempt": attempt, **fields} diff --git a/application/single_app/route_backend_workflows.py b/application/single_app/route_backend_workflows.py index 2192f215d..edc71df31 100644 --- a/application/single_app/route_backend_workflows.py +++ b/application/single_app/route_backend_workflows.py @@ -104,6 +104,7 @@ from functions_workflow_loop_history import ( workflow_execution_records_page, workflow_execution_provenance_page, workflow_loop_items_page, ) +from functions_workflow_repeat_history import workflow_repeat_iterations_page, workflow_repeat_state_page from route_backend_agents import ( _build_agent_instruction_api_params, _create_agent_instruction_client, @@ -319,7 +320,7 @@ def _workflow_runtime_response(workflow_id, run_id, *, group=False, action=None) def _workflow_execution_history_response(workflow_id, run_id, *, group=False, kind='execution', - execution_id=None, attempt=None, representation=None): + execution_id=None, attempt=None, representation=None, iteration=None): user_id = get_current_user_id() try: if group: @@ -331,7 +332,18 @@ def _workflow_execution_history_response(workflow_id, run_id, *, group=False, ki run = get_personal_workflow_run(user_id, run_id) if not workflow or not run or run.get('workflow_id') != workflow_id: return jsonify({'error': 'Workflow run not found.'}), 404 - if kind == 'items': + if kind == 'iterations': + response = workflow_repeat_iterations_page( + workflow, run_id, execution_id, reader_user_id=user_id, + cursor=request.args.get('cursor'), limit=int(request.args.get('limit', '50')), + ) + elif kind == 'states': + response = workflow_repeat_state_page( + workflow, run_id, execution_id, iteration, reader_user_id=user_id, + phase=request.args.get('phase', 'before'), + cursor=request.args.get('cursor'), limit=int(request.args.get('limit', '50')), + ) + elif kind == 'items': response = workflow_loop_items_page( workflow, run_id, execution_id, reader_user_id=user_id, cursor=request.args.get('cursor'), limit=int(request.args.get('limit', '50')), @@ -1040,6 +1052,48 @@ def _stream_group_workflow_activity(user_id, group_id, conversation_id='', workf def register_route_backend_workflows(bp): + @bp.route('/api/user/workflows//runs//executions//iterations', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('allow_user_workflows') + @workflow_user_required + def get_user_workflow_repeat_iterations(workflow_id, run_id, execution_id): + return _workflow_execution_history_response(workflow_id, run_id, execution_id=execution_id, kind='iterations') + + @bp.route('/api/group/workflows//runs//executions//iterations', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('enable_group_workspaces') + @enabled_required('allow_group_workflows') + def get_group_workflow_repeat_iterations(workflow_id, run_id, execution_id): + return _workflow_execution_history_response( + workflow_id, run_id, group=True, execution_id=execution_id, kind='iterations', + ) + + @bp.route('/api/user/workflows//runs//executions//iterations//state', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('allow_user_workflows') + @workflow_user_required + def get_user_workflow_repeat_state(workflow_id, run_id, execution_id, iteration): + return _workflow_execution_history_response( + workflow_id, run_id, execution_id=execution_id, iteration=iteration, kind='states', + ) + + @bp.route('/api/group/workflows//runs//executions//iterations//state', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + @enabled_required('enable_group_workspaces') + @enabled_required('allow_group_workflows') + def get_group_workflow_repeat_state(workflow_id, run_id, execution_id, iteration): + return _workflow_execution_history_response( + workflow_id, run_id, group=True, execution_id=execution_id, iteration=iteration, kind='states', + ) + @bp.route('/api/user/workflows/loop-inputs/preview', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 7e9028cd6..241fbeb1b 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -95,7 +95,9 @@ from functions_workflow_limits import ( WorkflowLoopLimitError, get_workflow_max_loop_items, + get_workflow_max_repeat_iterations, validate_workflow_max_loop_items, + validate_workflow_max_repeat_iterations, ) from support_menu_config import ( get_admin_latest_feature_release_groups_for_settings, @@ -1154,6 +1156,16 @@ def admin_settings(): flash(error.public_message, 'danger') return redirect(url_for('frontend_admin_settings.admin_settings')) + try: + workflow_max_repeat_iterations = ( + validate_workflow_max_repeat_iterations(form_data['workflow_max_repeat_iterations']) + if 'workflow_max_repeat_iterations' in form_data + else get_workflow_max_repeat_iterations(settings) + ) + except WorkflowLoopLimitError as error: + flash(error.public_message, 'danger') + return redirect(url_for('frontend_admin_settings.admin_settings')) + def admin_secret(field_name, form_field_name=None): submitted_value = form_data.get(form_field_name or field_name, '').strip() return resolve_admin_settings_secret_value(field_name, submitted_value, settings) @@ -2556,6 +2568,7 @@ def is_valid_url(url): 'workflow_max_auto_invoke_attempts': workflow_max_auto_invoke_attempts, 'workflow_max_tasks': workflow_max_tasks, 'workflow_max_loop_items': workflow_max_loop_items, + 'workflow_max_repeat_iterations': workflow_max_repeat_iterations, **chat_orchestration_settings, 'allow_personal_workspace_file_downloads': form_data.get('allow_personal_workspace_file_downloads') == 'on', 'allow_group_workspace_file_downloads': form_data.get('allow_group_workspace_file_downloads') == 'on', diff --git a/application/single_app/templates/admin/_panes/workflow.html b/application/single_app/templates/admin/_panes/workflow.html index cf382e206..17d5797b9 100644 --- a/application/single_app/templates/admin/_panes/workflow.html +++ b/application/single_app/templates/admin/_panes/workflow.html @@ -92,6 +92,29 @@
+
+ + +
+ Maximum rounds allowed in one automatic Repeat until batch in a new personal + or group workflow run. Default is 25; supported range is 1-1,000. Authors must + choose a per-block maximum; new runs above this ceiling are rejected, never + shortened. Active runs and manual continuation keep their admitted limit. + Another batch does not reset the run's execution-admission budget or elapsed deadline. +
+
+
void; +}) { + const producer = source.kind === 'node_output' ? producers.find((item) => item.id === source.node_id) : undefined; + const repeat = source.kind === 'repeat_state' ? repeats.find((item) => item.id === source.loop_id) : undefined; + const setRepeat = (loop: WorkflowRepeatUntilNode | undefined, stateName?: string) => { + const slot = stateName === undefined ? loop?.state[0] : loop?.state.find((item) => item.name === stateName); + onChange({ kind: 'repeat_state', loop_id: loop?.id ?? '', state_name: stateName ?? slot?.name ?? '', scope: 'current' }, + slot?.output_contract.kind ?? 'any'); + }; + return <> + {loops.length || repeats.length || source.kind !== 'node_output' ? : null} + {source.kind === 'node_output' ? <> + + + : source.kind === 'loop_item' ? : <> + + + } + ; +} + export function WorkflowFlowInputs({ workflow, nodeId, @@ -37,6 +129,7 @@ export function WorkflowFlowInputs({ label = 'Named inputs', availableIds, allowLoopItems = true, + allowRepeatState = true, recordsOnly = false, }: { workflow: WorkflowDefinition; @@ -46,6 +139,7 @@ export function WorkflowFlowInputs({ label?: string; availableIds?: Set; allowLoopItems?: boolean; + allowRepeatState?: boolean; recordsOnly?: boolean; }) { const available = availableIds ?? analyzeWorkflowFlow(workflow).available.get(nodeId) ?? new Set(); @@ -53,28 +147,31 @@ export function WorkflowFlowInputs({ .map((producer) => recordsOnly ? { ...producer, outputs: producer.outputs.filter(isRecordsFlowOutput) } : producer) .filter((producer) => !recordsOnly || producer.outputs.length > 0); const loops = allowLoopItems && !recordsOnly ? enclosingFlowLoops(workflow, nodeId) : []; + const repeats = allowRepeatState && !recordsOnly ? enclosingFlowRepeats(workflow, nodeId).filter((node) => node.state.length) : []; const update = (index: number, binding: WorkflowFlowBinding) => onChange(bindings.map((current, position) => position === index ? binding : current)); const add = () => { const producer = producers[0]; - if (!producer && !loops.length) return; + const repeat = repeats.at(-1); + if (!producer && !loops.length && !repeat) return; let number = 1; while (bindings.some((binding) => binding.name === `input${number}`)) number++; - onChange([...bindings, producer ? { + const next = producer ? { ...flowBinding(`input${number}`, producer.id, producer.outputs[0]?.name ?? 'authoritative'), expected_kind: producer.outputs[0]?.kind ?? 'any', - } : loopItemBinding(`input${number}`, loops[loops.length - 1].id)]); + } : loops.length ? loopItemBinding(`input${number}`, loops[loops.length - 1].id) + : repeat ? repeatStateBinding(`input${number}`, repeat.id, repeat.state[0]) : undefined; + if (next) onChange([...bindings, next]); }; return (
{label}

- Only these named final outputs are consumed. A skipped producer never falls back to another task. + Only these named saved values are consumed. A skipped producer never falls back to another task. {recordsOnly ? ' File export requires exactly one required records output. Partial output still needs this binding’s explicit acceptance and an eligible producer.' : ''}

{bindings.map((binding, index) => { const source = binding.source; - const producer = source.kind === 'node_output' ? producers.find((item) => item.id === source.node_id) : undefined; return (
- {loops.length || source.kind === 'loop_item' ? : null} - {source.kind === 'node_output' ? <> - - - : ( - - )} + update(index, { + ...binding, source: nextSource, expected_kind: kind, + allow_partial: nextSource.kind === 'loop_item' ? false : binding.allow_partial, + })} />

Reads the complete immutable records or document results in saved order, not a byte excerpt. Equal-looking records remain distinct. A document ID in model-generated JSON does not grant document access.

+ {binding?.source.kind === 'repeat_state' ?

Uses the named state saved at the start of this Repeat round. This For each instance freezes its membership from that state and requires complete eligible output.

: null}
: null} {iterable.kind === 'workspace_query' ? onChange({ ...node, iterable: next })} /> : null} @@ -305,9 +320,9 @@ export function WorkflowCollectFields({ node, workflow, onChange }: { onChange: (node: WorkflowCollectNode) => void; }) { const available = analyzeWorkflowFlow(workflow).available.get(node.id) ?? new Set(); - const parentIds = JSON.stringify(enclosingFlowLoops(workflow, node.id).map((loop) => loop.id)); + const parentIds = JSON.stringify(enclosingFlowLoopControls(workflow, node.id).map((loop) => loop.id)); const loops = flowLoops(workflow).filter((loop) => available.has(loop.node.id) && - JSON.stringify(enclosingFlowLoops(workflow, loop.node.id).map((parent) => parent.id)) === parentIds); + JSON.stringify(enclosingFlowLoopControls(workflow, loop.node.id).map((parent) => parent.id)) === parentIds); const selectedLoop = loops.find((loop) => loop.node.id === node.source.loop_id)?.node; const producers = flowProducers(workflow); const outputs = selectedLoop?.body.outputs.flatMap((item) => { diff --git a/application/v2_ui/src/components/workflows/WorkflowRepeatFields.tsx b/application/v2_ui/src/components/workflows/WorkflowRepeatFields.tsx new file mode 100644 index 000000000..fb98d8b62 --- /dev/null +++ b/application/v2_ui/src/components/workflows/WorkflowRepeatFields.tsx @@ -0,0 +1,236 @@ +// WorkflowRepeatFields.tsx +// Explicit typed state and per-batch limits for post-body Repeat until. + +import { useEffect, useRef } from 'react'; +import { Plus, Trash2 } from 'lucide-react'; +import { GlassButton } from '../ui/primitives'; +import { WorkflowDecisionFields, WorkflowFlowSourcePicker } from './WorkflowConditionEditor'; +import { + analyzeWorkflowFlow, + enclosingFlowRepeats, + flowProducers, + flowSourceOutput, + MAX_REPEAT_ITERATIONS, + REPEAT_STATE_KINDS, + repeatIterationErrors, + workflowRepeatLimit, + type WorkflowRepeatState, + type WorkflowRepeatUntilNode, +} from '../../lib/workflowFlow'; +import type { WorkflowDefinition, WorkflowEditorOptions } from '../../lib/workflowEditor'; + +const inputClass = 'mt-1 w-full min-w-0 rounded-lg border border-edge bg-surface-1 px-3 py-2 text-sm text-text-1 focus:border-accent focus:outline-none'; + +export function WorkflowRepeatFields({ node, workflow, options, onChange }: { + node: WorkflowRepeatUntilNode; + workflow: WorkflowDefinition; + options: WorkflowEditorOptions; + onChange: (node: WorkflowRepeatUntilNode) => void; +}) { + const maximumRef = useRef(null); + const initialMaximumUnset = useRef(!Number.isFinite(node.max_iterations)); + const lastStateRef = useRef(null); + const stateCount = useRef(node.state.length); + const ceiling = workflowRepeatLimit(options); + const errors = repeatIterationErrors(node, ceiling); + const available = analyzeWorkflowFlow(workflow).available.get(node.id) ?? new Set(); + const producers = flowProducers(workflow).filter((producer) => available.has(producer.id)); + const repeats = enclosingFlowRepeats(workflow, node.id); + const update = (index: number, slot: WorkflowRepeatState) => + onChange({ ...node, state: node.state.map((item, position) => position === index ? slot : item) }); + + useEffect(() => { + if (initialMaximumUnset.current) maximumRef.current?.focus(); + }, []); + useEffect(() => { + if (node.state.length > stateCount.current) lastStateRef.current?.focus(); + stateCount.current = node.state.length; + }, [node.state.length]); + + return <> + +

+ Administrator ceiling: {ceiling ?? 'unavailable'} rounds per automatic batch. Technical ceiling: 1,000. + The body runs at least once. If the condition is still false at the maximum, the run pauses for an authorized person's explicit continuation. +

+

+ Another batch never resets lifetime rounds, execution admissions, or the elapsed deadline, including time spent waiting. + The run's separate limits (at most 5,000 admissions and 86,400 seconds) may stop it sooner. +

+ {errors.length ?
+ {errors.map((error) =>

{error}

)} +
: null} +
+ Repeat state +

+ Initialize each named slot from an earlier saved output or an enclosing Repeat's current state, never a literal or latest-result lookup. + All next slots are validated and saved together after the body. To leave a slot unchanged, explicitly pass its current state through a body output. +

+ {node.state.map((slot, index) => { + const label = `State ${index + 1}`; + const contract = slot.output_contract; + const initialOutput = flowSourceOutput(workflow, slot.initial); + const typedProducers = producers.map((producer) => ({ + ...producer, outputs: producer.outputs.filter((output) => + (output.kinds ?? [output.kind]).every((kind) => kind === contract.kind)), + })).filter((producer) => producer.outputs.length); + const schemaType = typeof contract.schema?.type === 'string' ? contract.schema.type : ''; + return
+ {label} +
+ + + { + if (source.kind !== 'loop_item') update(index, { ...slot, initial: source }); + }} /> + +
+ {contract.kind === 'json' ? : null} + {initialOutput?.schema && initialOutput.kind === contract.kind ? update(index, { + ...slot, output_contract: { ...contract, schema: structuredClone(initialOutput.schema) }, + })}>Use initial output's declared schema : null} + update(index, { ...slot, output_contract: { ...next, kind: contract.kind } })} /> + {contract.kind !== 'text' ?
+ + {contract.kind === 'records' ? : null} +
: null} + + +

+ {contract.allow_partial + ? 'Partial coverage and limitations stay attached in later rounds and final output. Producers and consuming bindings must also explicitly accept partial data. Invalid or unauthorized data is never eligible.' + : 'Partial data is rejected by default. Manual continuation cannot bypass validation or authorize missing data.'} +

+ onChange({ ...node, state: node.state.filter((_, position) => position !== index) })}> + Remove state + +
; + })} + = 100} onClick={() => { + let name = 'state'; + let number = 2; + while (node.state.some((slot) => slot.name === name)) name = `state${number++}`; + onChange({ ...node, state: [...node.state, { + name, initial: { kind: 'node_output', node_id: '', output: '', scope: 'current' }, next: '', + output_contract: { kind: 'text', allow_partial: false, require_complete_coverage: false }, + }] }); + }}> Add state slot +
+ ; +} + +export function WorkflowRepeatExports({ node, onChange }: { + node: WorkflowRepeatUntilNode; + onChange: (node: WorkflowRepeatUntilNode) => void; +}) { + return
+ Repeat final exports +

These named outputs become available only after the stop condition is true. A batch-limit pause does not publish final outputs.

+ {node.exports.map((item, index) =>
+ + + onChange({ ...node, exports: node.exports.filter((_, position) => position !== index) })}> + Remove export + +
)} + = 100} onClick={() => { + let name = 'report'; + let number = 2; + while (node.exports.some((item) => item.name === name)) name = `report${number++}`; + onChange({ ...node, exports: [...node.exports, { name, output: '' }] }); + }}> Add Repeat export +
; +} diff --git a/application/v2_ui/src/components/workflows/WorkflowRepeatProgress.tsx b/application/v2_ui/src/components/workflows/WorkflowRepeatProgress.tsx new file mode 100644 index 000000000..601c0eb48 --- /dev/null +++ b/application/v2_ui/src/components/workflows/WorkflowRepeatProgress.tsx @@ -0,0 +1,24 @@ +// WorkflowRepeatProgress.tsx +// Lifetime identity and automatic-batch counters are intentionally separate. + +import type { WorkflowRepeatProgress as RepeatProgress } from '../../lib/workflowEditor'; + +export function WorkflowRepeatProgress({ summary, label = 'Repeat progress' }: { + summary?: RepeatProgress; + label?: string; +}) { + if (!summary) return null; + return
+

Repeat {summary.node_id}

+

Repeat execution: {summary.execution_id}

+

Lifetime completed rounds: {summary.completed_count}. + {summary.completed_iteration >= 0 ? ` Last completed round: ${summary.completed_iteration + 1}.` : ' No round has completed yet.'}

+

Automatic batch {summary.batch_number + 1}: {summary.batch_usage} of {summary.batch_size} rounds admitted.

+ {summary.state !== 'completed' && summary.state !== 'cancelled' ?

Next lifetime round: {summary.next_iteration + 1}.

: null} +

Batch-limit pauses: {summary.exhaustion_count}. Manual continuations: {summary.continuation_count}.

+

State: {summary.state.replaceAll('_', ' ')}. The batch size is frozen for this run; later administrator changes do not reset or shorten it.

+ {summary.partial ?

+ Accepted partial state. Coverage and limitations remain attached to subsequent rounds and final output. +

: null} +
; +} diff --git a/application/v2_ui/src/components/workflows/WorkflowRuntimePanel.tsx b/application/v2_ui/src/components/workflows/WorkflowRuntimePanel.tsx index ebc91ce42..6d65267e6 100644 --- a/application/v2_ui/src/components/workflows/WorkflowRuntimePanel.tsx +++ b/application/v2_ui/src/components/workflows/WorkflowRuntimePanel.tsx @@ -8,6 +8,7 @@ import { cancelScopedWorkflow, decideWorkflowRuntime, fetchWorkflowRuntime, + formatWorkflowIterationPath, resumeWorkflowRuntime, workflowErrorMessage, workflowScopeKey, @@ -22,6 +23,7 @@ import { ConfirmDialog } from '../ui/ConfirmDialog'; import { GlassButton, GlassPanel } from '../ui/primitives'; import { Pill } from '../workspace/primitives'; import { WorkflowPublicationDetails } from './WorkflowPublicationDetails'; +import { WorkflowRepeatProgress } from './WorkflowRepeatProgress'; function runtimeTone(state: string): 'ok' | 'warn' | 'danger' | 'neutral' | 'accent' { if (state === 'completed') { @@ -73,17 +75,7 @@ function safeJson(value: unknown): string { } function formatIterationPath(path: WorkflowRuntimeGate['iteration_path']): string { - if (!Array.isArray(path) || !path.length) { - return ''; - } - return path.map((frame, index) => { - const loop = String(frame.loop_id || `region ${index + 1}`); - const labels = [ - frame.item_id ? `item ${frame.item_id}` : '', - frame.index !== undefined ? `index ${frame.index}` : '', - ].filter(Boolean); - return labels.length ? `${loop} (${labels.join(', ')})` : loop; - }).join(' / '); + return formatWorkflowIterationPath(path); } function gateReference(gate: WorkflowRuntimeGate | undefined): string { @@ -100,6 +92,14 @@ function gateReference(gate: WorkflowRuntimeGate | undefined): string { return parts.join(' · '); } +function repeatBudgetBlocker(runtime: WorkflowRuntimeProjection | null): string { + const limits = runtime?.limits; + if (!limits) return 'The frozen run budgets are unavailable. Reload before continuing Repeat.'; + if (limits.admitted_count >= limits.max_executions) return 'The global execution-admission budget is exhausted. Another Repeat batch cannot extend it.'; + if (Date.parse(limits.deadline_at) <= Date.now()) return 'The elapsed run deadline has expired, including time spent waiting. Another Repeat batch cannot reset it.'; + return ''; +} + function RuntimeMemoryDetails({ memory, schemaVersion, @@ -232,6 +232,7 @@ export function WorkflowRuntimePanel({ const [action, setAction] = useState(null); const [confirmRetry, setConfirmRetry] = useState(false); const [retryTarget, setRetryTarget] = useState<{ key: string; gate: WorkflowRuntimeGate } | null>(null); + const [repeatTarget, setRepeatTarget] = useState<{ key: string; gate: WorkflowRuntimeGate } | null>(null); const [pollReadToken, setPollReadToken] = useState(0); const abortRef = useRef(null); const requestToken = useRef(0); @@ -244,6 +245,7 @@ export function WorkflowRuntimePanel({ setCanDecide(false); setConfirmRetry(false); setRetryTarget(null); + setRepeatTarget(null); retryRequest.current = null; }, [durable, runId, scopeKey, workflowId]); @@ -264,16 +266,19 @@ export function WorkflowRuntimePanel({ } setRuntime(response.runtime); setCanDecide(response.can_decide === true); + if (response.can_decide !== true) setRepeatTarget(null); } catch (cause: unknown) { if (controller.signal.aborted || token !== requestToken.current) { return; } setCanDecide(false); - setRuntime((current) => current?.gate?.publication ? null : current); + setRepeatTarget(null); + setRuntime((current) => current?.gate?.publication || current?.repeat_progress || current?.gate?.repeat ? null : current); if (cause instanceof ApiError && (cause.status === 403 || cause.status === 404)) { setRuntime(null); setConfirmRetry(false); setRetryTarget(null); + setRepeatTarget(null); retryRequest.current = null; onAccessLost?.(cause.status); } @@ -322,6 +327,7 @@ export function WorkflowRuntimePanel({ record.gate?.execution_id, record.gate?.node_id, record.gate?.attempt, record.gate?.input_digest, record.gate?.iteration_path, + record.gate?.repeat, ]); const applyRuntimeResponse = (nextRuntime: WorkflowRuntimeProjection, nextCanDecide: boolean) => { @@ -337,6 +343,7 @@ export function WorkflowRuntimePanel({ setCanDecide(false); setConfirmRetry(false); setRetryTarget(null); + setRepeatTarget(null); setError(status === 404 ? 'No durable runtime record is available for this run.' : 'You no longer have access to this workflow runtime. Reload or ask an owner to restore access.'); onAccessLost?.(status); @@ -344,9 +351,24 @@ export function WorkflowRuntimePanel({ const decide = async (choice: WorkflowRuntimeDecisionChoice) => { if (action) return; + if (choice === 'continue_repeat') { + if (!runtime?.gate?.repeat || !repeatTarget || repeatTarget.key !== recoveryKey(runtime) || + runtime.gate.kind !== 'pause' || runtime.gate.reason_code !== 'repeat_iteration_limit' || !canDecide) { + setRepeatTarget(null); + setError('The Repeat gate changed while you were reviewing it. Review the current batch and saved state before continuing.'); + return; + } + const blocker = repeatBudgetBlocker(runtime); + if (blocker) { + setRepeatTarget(null); + setError(blocker); + return; + } + } if (choice === 'retry' && (!runtime?.gate || !retryTarget || retryTarget.key !== recoveryKey(runtime) || runtime.gate.kind !== 'recovery' || !canDecide)) { setConfirmRetry(false); setRetryTarget(null); + setRepeatTarget(null); setError('The recovery gate changed while you were reviewing it. Review the current execution and attempt before retrying.'); return; } @@ -397,11 +419,12 @@ export function WorkflowRuntimePanel({ setAction(null); setConfirmRetry(false); setRetryTarget(null); + setRepeatTarget(null); } }; const resume = async () => { - if (!runtime || action) { + if (!runtime || action || runtime.gate?.reason_code === 'repeat_iteration_limit') { return; } abortRef.current?.abort(); @@ -454,9 +477,12 @@ export function WorkflowRuntimePanel({ }; const gate = runtime?.gate; + const repeatGate = gate?.reason_code === 'repeat_iteration_limit'; + const repeatBlocker = repeatGate ? repeatBudgetBlocker(runtime) : ''; const unsupportedRuntimeSchema = Boolean(runtime && runtime.schema_version !== undefined && ![1, 2].includes(runtime.schema_version)); const gateAllows = (choice: WorkflowRuntimeDecisionChoice) => Boolean(gate?.choices.includes(choice) && + (!repeatGate || choice === 'continue_repeat' || choice === 'cancel') && (!gate.publication || !['approve', 'reject', 'retry'].includes(choice))); const progressLabel = useMemo(() => { if (!runtime?.progress) { @@ -489,6 +515,11 @@ export function WorkflowRuntimePanel({ {loading ? : null}
{progressLabel ?

{progressLabel}

: null} + + {runtime?.repeat_counts ?

+ All Repeat blocks in this run: {runtime.repeat_counts.exhaustion_count ?? 0} batch-limit pauses; + {' '}{runtime.repeat_counts.continuation_count ?? 0} manual continuations. +

: null} {runtime?.loop_progress ? (

For each {runtime.loop_progress.loop_id}

@@ -505,12 +536,17 @@ export function WorkflowRuntimePanel({

{runtime.limits.admitted_count} of {runtime.limits.max_executions} execution admissions used. {' '}Deadline: {formatTimestamp(runtime.limits.deadline_at)} (including waits). + {runtime.repeat_progress || repeatGate ? <> + {' '}Remaining execution admissions: {Math.max(0, runtime.limits.max_executions - runtime.limits.admitted_count)}. + {' '}Elapsed time remaining: {Math.max(0, Math.floor((Date.parse(runtime.limits.deadline_at) - Date.now()) / 1000))} seconds. + {' '}Frozen Repeat policy ceiling: {runtime.limits.max_repeat_iterations} rounds per batch. + : null}

) : null} {error ?

{error}

: null} {runtime && !canDecide ? (

- You can view this runtime, but you do not have permission to approve, reject, retry, resume or cancel it. + You can view this runtime, but you do not have permission to approve, reject, retry, continue Repeat, resume or cancel it.

) : null} {unsupportedRuntimeSchema ? ( @@ -526,6 +562,14 @@ export function WorkflowRuntimePanel({ {gateReference(gate) ?

{gateReference(gate)}

: null} {gate.reason ?

{gate.reason}

: null} + {repeatGate ?
+

+ The stop condition is still unmet. The latest validated state is retained, but final Repeat outputs are not available. + Continue Repeat explicitly grants one more same-sized batch; ordinary Resume is not a continuation grant. +

+ {!runtime?.repeat_progress ? : null} + {repeatBlocker ?

{repeatBlocker}

: null} +
: null} {gate.input_digest && !gate.publication ?

Input digest: {gate.input_digest}

: null} {gate.kind === 'output' ? ( @@ -573,6 +617,15 @@ export function WorkflowRuntimePanel({ ) : null} {canMutate && gate.kind === 'pause' ? (
+ {repeatGate && gate.repeat && gateAllows('continue_repeat') ? ( + { + if (runtime) setRepeatTarget({ key: recoveryKey(runtime), gate: structuredClone(gate) }); + }}> + {action === 'continue_repeat' ? : } + Continue Repeat + + ) : null} {gateAllows('resume') ? ( void decide('resume')}> {action === 'resume' ? : } @@ -632,6 +685,27 @@ export function WorkflowRuntimePanel({ ) : null} ) : null} + {repeatTarget?.gate.repeat ? ( + } + cancelLabel="Keep paused" + busy={action === 'continue_repeat'} + tone="primary" + onClose={() => setRepeatTarget(null)} + onConfirm={() => void decide('continue_repeat')} + > + +

+ Lifetime round numbering, the admission budget and the original deadline are unchanged. + Remaining global budgets may stop the run before the whole batch finishes. + This does not approve body tasks, publication destinations, partial data, or invalid output. +

+

{gateReference(repeatTarget.gate)}

+ + ) : null}
); } diff --git a/application/v2_ui/src/components/workflows/WorkflowStructuredList.tsx b/application/v2_ui/src/components/workflows/WorkflowStructuredList.tsx index f029e6f30..e09c14ad8 100644 --- a/application/v2_ui/src/components/workflows/WorkflowStructuredList.tsx +++ b/application/v2_ui/src/components/workflows/WorkflowStructuredList.tsx @@ -7,17 +7,20 @@ import { ConfirmDialog } from '../ui/ConfirmDialog'; import { GlassButton } from '../ui/primitives'; import { WorkflowConditionEditor, WorkflowFlowInputs } from './WorkflowConditionEditor'; import { WorkflowCollectFields, WorkflowForEachFields } from './WorkflowLoopFields'; +import { WorkflowRepeatFields, WorkflowRepeatExports } from './WorkflowRepeatFields'; import { analyzeWorkflowFlow, DEFAULT_FLOW_LIMITS, defaultFlowPredicate, - enclosingFlowLoops, + enclosingFlowLoopControls, flowProducers, flowRegions, flowTaskIds, FLOW_MAX_DEPTH, FLOW_OUTPUT_KINDS, isFlowRegion, + repeatUntilBindings, + supportsWorkflowRepeat, updateFlowRegion, workflowLoopLimit, type FlowProducer, @@ -166,6 +169,11 @@ export function WorkflowStructuredList({ id, kind, source: { loop_id: '', output: '' }, output_contract: { kind: 'records', require_complete_coverage: true, allow_partial: false }, }; + else if (kind === 'repeat_until') node = { + id, kind, max_iterations: Number.NaN, state: [], + body: { id: `body-${task.id}`, nodes: [], outputs: [] }, + until: defaultFlowPredicate(), exports: [], + }; else node = { id, kind, inputs: [], condition: defaultFlowPredicate(), then: { id: `then-${task.id}`, nodes: [] }, else: { id: `else-${task.id}`, nodes: [] }, @@ -190,13 +198,13 @@ export function WorkflowStructuredList({ {label} {region.nodes.map((node, index) => { const childRegions = node.kind === 'if' ? [...flowRegions(node.then), ...flowRegions(node.else)] - : node.kind === 'for_each' ? flowRegions(node.body) : []; + : node.kind === 'for_each' || node.kind === 'repeat_until' ? flowRegions(node.body) : []; const descendants = new Set(childRegions.map((item) => item.id)); const subtreeDepth = childRegions.length ? Math.max(...childRegions.map((item) => item.depth)) + 1 : 0; const destinations = regions.filter((item) => item.id !== region.id && !descendants.has(item.id) && item.depth + subtreeDepth < FLOW_MAX_DEPTH); const task = node.kind === 'task' ? workflow.tasks.find((item) => item.id === node.task_id) : undefined; const title = node.kind === 'task' ? task?.name || 'Task' : node.kind === 'if' ? 'If / else' - : node.kind === 'for_each' ? 'For each' : node.kind === 'collect' ? 'Collect' : 'Forward route'; + : node.kind === 'for_each' ? 'For each' : node.kind === 'repeat_until' ? 'Repeat until' : node.kind === 'collect' ? 'Collect' : 'Forward route'; const routeTargetId = node.kind === 'route' && 'node_id' in node.target ? node.target.node_id : undefined; return (
@@ -230,12 +238,27 @@ export function WorkflowStructuredList({ onChange={(next) => setNode(region.id, next)} /> {renderRegion(node.body, 'Body', depth + 1)} - enclosingFlowLoops(workflow, id).at(-1)?.id === node.id))} + enclosingFlowLoopControls(workflow, id).at(-1)?.id === node.id))} onChange={(outputs) => setNode(region.id, { ...node, body: { ...node.body, outputs } })} />

Body outputs are per-item receipts. Add a following Collect to expose a complete collection outside this loop.

+ ) : node.kind === 'repeat_until' ? ( + <> + setNode(region.id, next)} /> + {renderRegion(node.body, 'Repeat body', depth + 1)} + + enclosingFlowLoopControls(workflow, id).at(-1)?.id === node.id))} + onChange={(outputs) => setNode(region.id, { ...node, body: { ...node.body, outputs } })} /> +

The stop condition reads the validated NEXT state, after every named slot is saved atomically. Body bindings always read CURRENT state.

+ setNode(region.id, { ...node, until })} /> + setNode(region.id, next)} /> + ) : node.kind === 'collect' ? ( setNode(region.id, next)} /> ) : ( @@ -285,6 +308,8 @@ export function WorkflowStructuredList({ aria-label={`Add forward route to ${label}`}> Add forward route {options.supported_node_kinds?.includes('for_each') ? = FLOW_MAX_DEPTH} onClick={() => add(region.id, 'for_each')} aria-label={`Add For each to ${label}`}> Add For each : null} + {supportsWorkflowRepeat(options) ? = FLOW_MAX_DEPTH} + onClick={() => add(region.id, 'repeat_until')} aria-label={`Add Repeat until to ${label}`}> Add Repeat until : null} {options.supported_node_kinds?.includes('collect') ? add(region.id, 'collect')} aria-label={`Add Collect to ${label}`}> Add Collect : null} diff --git a/application/v2_ui/src/lib/workflowEditor.ts b/application/v2_ui/src/lib/workflowEditor.ts index 0cb7b316d..74b61e603 100644 --- a/application/v2_ui/src/lib/workflowEditor.ts +++ b/application/v2_ui/src/lib/workflowEditor.ts @@ -10,19 +10,24 @@ import type { DocumentListResponse, DocumentQuery, WorkspaceDocument } from './t import { isRecord, sameEditorValue } from './workspaceAuthoring'; import { analyzeWorkflowFlow, - enclosingFlowLoops, + DEFAULT_FLOW_LIMITS, + enclosingFlowLoopControls, flowLoops, - flowProducers, + flowRepeats, + flowSourceOutput, flowTaskNodeId, flowUnsupportedReason, FLOW_ALIAS_PATTERN, FLOW_MAX_DEPTH, MAX_LOOP_ITEMS, + MAX_REPEAT_ITERATIONS, isFlowBinding, isFlowRegion, isLegacyWorkflowBinding, loopSelectionErrors, + repeatIterationErrors, workflowLoopLimit, + workflowRepeatLimit, type WorkflowFlowBinding, type WorkflowLoopIterable, } from './workflowFlow'; @@ -41,7 +46,7 @@ export type WorkflowRuntimeState = 'paused' | 'cancelling' | 'cancelled' | 'failed' | 'invalid' | 'incomplete' | 'completed' | 'completed_partial' | 'skipped'; export type WorkflowRuntimeGateKind = 'approval' | 'output' | 'recovery' | 'pause'; -export type WorkflowRuntimeDecisionChoice = 'approve' | 'reject' | 'retry' | 'cancel' | 'resume'; +export type WorkflowRuntimeDecisionChoice = 'approve' | 'reject' | 'retry' | 'cancel' | 'resume' | 'continue_repeat'; export interface WorkflowAgentOption { id: string; @@ -81,6 +86,8 @@ export interface WorkflowEditorOptions { max_executions: number; deadline_seconds: number; max_loop_items?: number; + max_repeat_iterations?: number; + hard_repeat_iterations?: number; }; can_manage: boolean; max_tasks: number; @@ -309,30 +316,87 @@ export interface WorkflowRuntimeGate { unit_id?: string; input_digest?: string; reason?: string; + reason_code?: string; choices: string[]; execution_id?: string; node_id?: string; attempt?: number; iteration_path?: WorkflowIterationFrame[]; publication?: WorkflowPublicationStatus; + repeat?: WorkflowRepeatProgress; } -export interface WorkflowIterationFrame { +export interface WorkflowForEachFrame { loop_id: string; item_id: string; index: number; } +export interface WorkflowRepeatFrame { + loop_id: string; + iteration: number; +} + +export type WorkflowIterationFrame = WorkflowForEachFrame | WorkflowRepeatFrame; + export function validWorkflowIterationPath(value: unknown): value is WorkflowIterationFrame[] { return Array.isArray(value) && value.length < FLOW_MAX_DEPTH && value.every((frame) => - isRecord(frame) && Object.keys(frame).every((key) => ['loop_id', 'item_id', 'index'].includes(key)) && + isRecord(frame) && typeof frame.loop_id === 'string' && frame.loop_id === frame.loop_id.trim() && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(frame.loop_id) && - typeof frame.item_id === 'string' && frame.item_id.length === 64 && /^[a-f0-9]{64}$/.test(frame.item_id) && - typeof frame.index === 'number' && Number.isSafeInteger(frame.index) && frame.index >= 0 && frame.index < MAX_LOOP_ITEMS) && + ('iteration' in frame + ? Object.keys(frame).length === 2 && typeof frame.iteration === 'number' && + Number.isSafeInteger(frame.iteration) && frame.iteration >= 0 && frame.iteration < DEFAULT_FLOW_LIMITS.max_executions + : Object.keys(frame).every((key) => ['loop_id', 'item_id', 'index'].includes(key)) && + typeof frame.item_id === 'string' && /^[a-f0-9]{64}$/.test(frame.item_id) && + typeof frame.index === 'number' && Number.isSafeInteger(frame.index) && frame.index >= 0 && frame.index < MAX_LOOP_ITEMS)) && new Set(value.map((frame) => frame.loop_id)).size === value.length; } +export function formatWorkflowIterationPath(path?: WorkflowIterationFrame[]): string { + return (path ?? []).map((frame) => 'iteration' in frame + ? `${frame.loop_id} (round ${frame.iteration + 1})` + : `${frame.loop_id} (item ${frame.item_id}, index ${frame.index})`).join(' / '); +} + +export interface WorkflowRepeatProgress { + execution_id: string; + node_id: string; + completed_iteration: number; + next_iteration: number; + batch_number: number; + batch_size: number; + batch_usage: number; + completed_count: number; + exhaustion_count: number; + continuation_count: number; + state: 'running' | 'waiting_manual_continue' | 'completed' | 'cancelled'; + partial: boolean; +} + +export function isWorkflowRepeatProgress(value: unknown): value is WorkflowRepeatProgress { + const counts = ['next_iteration', 'batch_number', 'batch_size', 'batch_usage', 'completed_count', + 'exhaustion_count', 'continuation_count']; + if (!isRecord(value) || Object.keys(value).some((key) => ![ + 'execution_id', 'node_id', 'completed_iteration', 'state', 'partial', ...counts, + ].includes(key)) || typeof value.execution_id !== 'string' || !value.execution_id.trim() || value.execution_id.length > 256 || + typeof value.node_id !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value.node_id) || + !['running', 'waiting_manual_continue', 'completed', 'cancelled'].includes(String(value.state)) || + typeof value.partial !== 'boolean' || counts.some((key) => + typeof value[key] !== 'number' || !Number.isSafeInteger(value[key]) || Number(value[key]) < 0 || + Number(value[key]) > DEFAULT_FLOW_LIMITS.max_executions) || + typeof value.completed_iteration !== 'number' || !Number.isSafeInteger(value.completed_iteration)) return false; + const admittedRounds = Number(value.batch_number) * Number(value.batch_size) + Number(value.batch_usage); + const completedRounds = Number(value.completed_count); + return Number(value.batch_size) >= 1 && Number(value.batch_size) <= MAX_REPEAT_ITERATIONS && + Number(value.batch_usage) <= Number(value.batch_size) && + admittedRounds >= completedRounds && admittedRounds <= completedRounds + 1 && + (!['completed', 'waiting_manual_continue'].includes(String(value.state)) || admittedRounds === completedRounds) && + value.next_iteration === value.completed_count && value.completed_iteration === Number(value.completed_count) - 1 && + value.batch_number === value.continuation_count && + (value.state !== 'waiting_manual_continue' || value.batch_usage === value.batch_size && Number(value.completed_count) > 0); +} + export interface WorkflowLoopProgress { loop_id: string; loop_execution_id: string; @@ -385,12 +449,16 @@ export interface WorkflowRuntimeProjection { memory?: WorkflowRuntimeMemory; can_resume?: boolean; loop_progress?: WorkflowLoopProgress; + repeat_progress?: WorkflowRepeatProgress; + repeat_counts?: { exhaustion_count?: number; continuation_count?: number }; limits?: { max_executions: number; admitted_count: number; deadline_at: string; deadline_seconds: number; waits_count: boolean; + max_loop_items?: number; + max_repeat_iterations?: number; }; } @@ -1077,14 +1145,12 @@ export function workflowInputProcessingErrors( if (task.document_action?.type !== 'none') errors.push(`${label}: saved-record reports require No document action; they explain saved data rather than reanalyzing sources.`); if (task.publication) errors.push(`${label}: saved-record reports cannot publish artifacts. Use a separate publication task.`); if (task.output_contract?.kind !== 'text') errors.push(`${label}: saved-record reports require a text output contract.`); - const producers = flowProducers(workflow); const hasCollection = (task.inputs ?? []).some((binding) => { - if (!isFlowBinding(binding) || binding.source.kind !== 'node_output') return false; - const source = binding.source; - return producers.find((producer) => producer.id === source.node_id)?.outputs.some((output) => - output.name === source.output && (output.kinds ?? [output.kind]).every((kind) => ['records', 'document_results'].includes(kind))); + if (!isFlowBinding(binding)) return false; + const output = flowSourceOutput(workflow, binding.source); + return output && (output.kinds ?? [output.kind]).every((kind) => ['records', 'document_results'].includes(kind)); }); - if (!hasCollection) errors.push(`${label}: saved-record reports require at least one saved records or document-results node-output input.`); + if (!hasCollection) errors.push(`${label}: saved-record reports require at least one saved records or document-results node-output input or current Repeat state.`); if (!workflowTaskHasLocalRunner(workflow, task, options)) { errors.push(`${label}: saved-record reports require a locally metered model or local agent; hosted runners are not supported.`); } @@ -1116,6 +1182,21 @@ export function workflowValidationErrors( errors.push('Group workflow loops can select only documents in this explicit group workspace.'); } }); + flowRepeats(draft).forEach(({ node }) => { + errors.push(...repeatIterationErrors(node, workflowRepeatLimit(options))); + node.state.forEach((slot) => { + const contract = slot.output_contract; + if (contract.schema) errors.push(...workflowSchemaErrors(contract.schema).map((error) => + `Repeat ${node.id} state ${slot.name}: ${error}`)); + if (contract.expected_count !== undefined && + (!Number.isSafeInteger(contract.expected_count) || contract.expected_count < 0 || contract.kind === 'text')) { + errors.push(`Repeat ${node.id} state ${slot.name}: expected count needs a nonnegative whole number for structured data.`); + } + if (contract.identity_field && contract.kind !== 'records') { + errors.push(`Repeat ${node.id} state ${slot.name}: identity fields apply only to records.`); + } + }); + }); const checkCollects = (region: typeof draft.flow) => { if (!isFlowRegion(region)) return; region.nodes.forEach((node) => { @@ -1126,7 +1207,7 @@ export function workflowValidationErrors( errors.push('Collect expected count must be a nonnegative whole number.'); } if (contract.identity_field && contract.kind !== 'records') errors.push('Collect business-key uniqueness is available only for records.'); - } else if (node.kind === 'for_each') checkCollects(node.body); + } else if (node.kind === 'for_each' || node.kind === 'repeat_until') checkCollects(node.body); else if (node.kind === 'if') { checkCollects(node.then); checkCollects(node.else); @@ -1199,9 +1280,9 @@ export function workflowValidationErrors( if (action?.target_mode === 'current_item' && draft.definition_version !== 3) { errors.push('Current-document Analyze requires structured control flow.'); } - if (draft.definition_version === 3 && enclosingFlowLoops(draft, flowTaskNodeId(draft, task.id)).length && !task.publication) { + if (draft.definition_version === 3 && enclosingFlowLoopControls(draft, flowTaskNodeId(draft, task.id)).length && !task.publication) { if (!workflowTaskHasLocalRunner(draft, task, options)) { - errors.push(`${task.name}: choose a loop-eligible local agent or model. Hosted runners are not supported inside For each.`); + errors.push(`${task.name}: choose a loop-eligible local agent or model. Hosted runners are not supported inside For each or Repeat.`); } } if (action?.type === 'search' && action.doc_scope !== 'all' && @@ -1329,7 +1410,11 @@ export async function fetchWorkflowEditorOptions( throw new Error('The workflow editor options returned an invalid response.'); } const ceiling = response.flow_limits?.max_loop_items; + const repeatCeiling = response.flow_limits?.max_repeat_iterations; + const repeatHardCeiling = response.flow_limits?.hard_repeat_iterations; if (ceiling !== undefined && (!Number.isInteger(ceiling) || ceiling < 1 || ceiling > 5000) || + repeatCeiling !== undefined && (!Number.isInteger(repeatCeiling) || repeatCeiling < 1 || repeatCeiling > MAX_REPEAT_ITERATIONS) || + repeatHardCeiling !== undefined && repeatHardCeiling !== MAX_REPEAT_ITERATIONS || [response.supported_node_kinds, response.supported_iterable_kinds, response.supported_query_modes, response.supported_binding_sources, response.supported_input_processing_modes, response.supported_publication_completion_policies] .some((values) => values !== undefined && (!Array.isArray(values) || values.some((value) => typeof value !== 'string'))) || @@ -1534,6 +1619,43 @@ function checkedRuntimeResponse(response: WorkflowRuntimeResponse): WorkflowRunt if (publication !== undefined && !isWorkflowPublicationStatus(publication)) { throw new Error('The workflow runtime returned an unsupported publication status. Reload before making a decision.'); } + const gate = response.runtime.gate; + const repeat = response.runtime.repeat_progress; + const repeatGate = gate?.reason_code === 'repeat_iteration_limit'; + if (repeat !== undefined && !isWorkflowRepeatProgress(repeat) || + gate?.repeat !== undefined && (!repeatGate || !isWorkflowRepeatProgress(gate.repeat)) || + repeatGate && (response.runtime.state !== 'paused' || gate?.kind !== 'pause' || + typeof gate.id !== 'string' || !gate.id.trim() || gate.id.length > 256 || + !Number.isSafeInteger(gate.attempt) || Number(gate.attempt) < 1 || + !validWorkflowIterationPath(gate.iteration_path) || + !gate.repeat || gate.repeat.state !== 'waiting_manual_continue' || + gate.execution_id !== gate.repeat.execution_id || gate.node_id !== gate.repeat.node_id || + !sameEditorValue(gate.choices, ['continue_repeat', 'cancel'])) || + !repeatGate && gate?.choices.includes('continue_repeat')) { + throw new Error('The workflow runtime returned an unsupported Repeat continuation gate or progress. Reload before making a decision.'); + } + if (repeat !== undefined || repeatGate) { + const limits = response.runtime.limits; + if (!Number.isSafeInteger(response.runtime.version) || response.runtime.version < 0 || + !limits || !Number.isSafeInteger(limits.max_executions) || limits.max_executions < 1 || + limits.max_executions > DEFAULT_FLOW_LIMITS.max_executions || + !Number.isSafeInteger(limits.admitted_count) || limits.admitted_count < 0 || + !Number.isSafeInteger(limits.deadline_seconds) || limits.deadline_seconds < 1 || + limits.deadline_seconds > DEFAULT_FLOW_LIMITS.deadline_seconds || limits.waits_count !== true || + typeof limits.deadline_at !== 'string' || !Number.isFinite(Date.parse(limits.deadline_at)) || + typeof limits.max_repeat_iterations !== 'number' || !Number.isInteger(limits.max_repeat_iterations) || + limits.max_repeat_iterations < 1 || limits.max_repeat_iterations > MAX_REPEAT_ITERATIONS || + repeat && repeat.batch_size > limits.max_repeat_iterations || + gate?.repeat && gate.repeat.batch_size > limits.max_repeat_iterations) { + throw new Error('The workflow runtime returned invalid frozen Repeat limits. Reload before making a decision.'); + } + } + const repeatCounts = response.runtime.repeat_counts; + if (repeatCounts !== undefined && (!isRecord(repeatCounts) || Object.entries(repeatCounts).some(([key, value]) => + !['exhaustion_count', 'continuation_count'].includes(key) || typeof value !== 'number' || + !Number.isSafeInteger(value) || value < 0 || value > DEFAULT_FLOW_LIMITS.max_executions))) { + throw new Error('The workflow runtime returned invalid Repeat audit counters.'); + } const progress = response.runtime.loop_progress; if (progress && ( typeof progress.loop_id !== 'string' || !progress.loop_id.trim() || diff --git a/application/v2_ui/src/lib/workflowExecutionHistory.ts b/application/v2_ui/src/lib/workflowExecutionHistory.ts index 394e290e7..6e3fedfbd 100644 --- a/application/v2_ui/src/lib/workflowExecutionHistory.ts +++ b/application/v2_ui/src/lib/workflowExecutionHistory.ts @@ -8,15 +8,18 @@ import { workflowLoopSelection, validWorkflowIterationPath, isWorkflowPublicationStatus, + isWorkflowRepeatProgress, type WorkflowConsumedInput, type WorkflowIterationFrame, type WorkflowLoopSelection, type WorkflowResultReference, type WorkflowPublicationStatus, + type WorkflowRepeatProgress, type WorkflowRunResultPage, type WorkflowScope, type WorkflowValidationResult, } from './workflowEditor'; +import { DEFAULT_FLOW_LIMITS, FLOW_ALIAS_PATTERN, MAX_REPEAT_ITERATIONS, REPEAT_STATE_KINDS, type WorkflowRepeatStateKind } from './workflowFlow'; export interface WorkflowExecutionDecisionPreview { choice?: string; @@ -31,6 +34,16 @@ export interface WorkflowExecutionDecisionPreview { reason_code?: string; timestamp?: string; decided_at?: string; + actor_user_id?: string; + request_id?: string; + event_id?: string; + repeat?: WorkflowRepeatProgress; + iteration?: number; + batch_number?: number; + batch_size?: number; + batch_usage?: number; + condition_result?: boolean; + outcome?: string; [key: string]: unknown; } @@ -64,6 +77,7 @@ export interface WorkflowExecutionRecord { export interface WorkflowExecutionAttemptRecord { execution_id: string; node_id: string; + node_kind?: string; task_id?: string; attempt: number; state: string; @@ -83,7 +97,7 @@ export interface WorkflowExecutionAttemptRecord { [key: string]: unknown; } -export interface WorkflowRuntimeDecisionRecord { +export interface WorkflowRuntimeDecisionRecord extends WorkflowExecutionDecisionPreview { execution_id?: string; node_id?: string; attempt?: number; @@ -116,9 +130,44 @@ export interface WorkflowExecutionPage { coverage?: Record; admittedLimit?: number; selection?: WorkflowLoopSelection; + repeat?: WorkflowRepeatProgress; + stateAvailable?: boolean; + partial?: boolean; + sourceSnapshotChanged?: boolean; }; } +export interface WorkflowRepeatIterationRecord { + iteration: number; + iteration_path: WorkflowIterationFrame[]; + batch_number: number; + batch_size: number; + batch_usage: number; + state: 'running' | 'completed' | 'completed_partial' | 'cancelled'; + condition_result: boolean | null; + execution_ids: string[]; + before_available: true; + after_available: boolean; + partial: boolean; +} + +export interface WorkflowRepeatStateRecord { + name: string; + kind: WorkflowRepeatStateKind; + source: { + node_id: string; + execution_id: string; + task_id?: string; + iteration_path: WorkflowIterationFrame[]; + attempt: number; + output_name: string; + }; + workflow_validation: WorkflowValidationResult; + coverage: Record; + prior_coverage?: Record; + limitations: string[]; +} + export interface WorkflowLoopItemRecord { item_id: string; index: number; @@ -226,6 +275,7 @@ function isExecution(value: unknown): value is WorkflowExecutionRecord { function isAttempt(value: unknown): value is WorkflowExecutionAttemptRecord { return isRecord(value) && validIdentity(value.execution_id) && validIdentity(value.node_id) && + (value.node_kind === undefined || validIdentity(value.node_kind)) && validIdentity(value.state) && typeof value.attempt === 'number' && Number.isInteger(value.attempt) && value.attempt >= 1 && validPath(value.iteration_path) && validResultMetadata(value); } @@ -234,7 +284,12 @@ function isDecision(value: unknown): value is WorkflowRuntimeDecisionRecord { return isRecord(value) && validPath(value.iteration_path) && (value.execution_id === undefined || validIdentity(value.execution_id)) && (value.node_id === undefined || validIdentity(value.node_id)) && - (value.attempt === undefined || typeof value.attempt === 'number' && Number.isInteger(value.attempt) && value.attempt >= 0); + (value.attempt === undefined || typeof value.attempt === 'number' && Number.isInteger(value.attempt) && value.attempt >= 0) && + (value.repeat === undefined || isWorkflowRepeatProgress(value.repeat)) && + (value.condition_result === undefined || typeof value.condition_result === 'boolean') && + ['iteration', 'batch_number', 'batch_size', 'batch_usage'].every((key) => + value[key] === undefined || typeof value[key] === 'number' && Number.isSafeInteger(value[key]) && + Number(value[key]) >= 0 && Number(value[key]) <= DEFAULT_FLOW_LIMITS.max_executions); } function pageParams(cursor: string | null, limit: number): URLSearchParams { @@ -247,7 +302,7 @@ function pageParams(cursor: string | null, limit: number): URLSearchParams { function pageFromResponse( response: unknown, - key: 'executions' | 'attempts' | 'decisions' | 'items' | 'records' | 'contributors', + key: 'executions' | 'attempts' | 'decisions' | 'items' | 'records' | 'contributors' | 'iterations' | 'states', isItem: (value: unknown) => value is T, identity?: (value: T) => string, limit = 100, @@ -321,13 +376,15 @@ export async function fetchWorkflowExecutionAttemptResult( offset: number, limit = 2000, signal?: AbortSignal, + output = 'authoritative', ): Promise { const resultLimit = boundedLimit(limit, 2000); - if (!validIdentity(executionId) || !Number.isInteger(attempt) || attempt < 1 || !Number.isInteger(offset) || offset < 0) { + if (!validIdentity(executionId) || !Number.isInteger(attempt) || attempt < 1 || + !Number.isInteger(offset) || offset < 0 || !FLOW_ALIAS_PATTERN.test(output)) { throw new Error('The requested execution attempt or result range is invalid.'); } const params = new URLSearchParams({ - output: 'authoritative', + output, offset: String(offset), limit: String(resultLimit), }); @@ -381,7 +438,7 @@ export async function fetchWorkflowLoopItemsPage( !validIdentity(value.state) || typeof value.index !== 'number' || !Number.isSafeInteger(value.index) || value.index < 0 || !validWorkflowIterationPath(value.iteration_path)) return false; const frame = value.iteration_path.at(-1); - return frame?.item_id === value.item_id && frame.index === value.index && + return frame !== undefined && 'item_id' in frame && frame.item_id === value.item_id && frame.index === value.index && (value.execution_ids === undefined || Array.isArray(value.execution_ids) && value.execution_ids.length <= 256 && value.execution_ids.every(validIdentity)) && (value.record_count === undefined || typeof value.record_count === 'number' && Number.isSafeInteger(value.record_count) && value.record_count >= 0); }, (item) => item.item_id, limit); @@ -392,6 +449,122 @@ export async function fetchWorkflowLoopItemsPage( } }; } +export async function fetchWorkflowRepeatIterationsPage( + scope: WorkflowScope, + workflowId: string, + runId: string, + executionId: string, + cursor: string | null, + limit = 50, + signal?: AbortSignal, +): Promise> { + const response = await api.get(workflowUrl(scope, workflowId, + `/runs/${encodeURIComponent(runId)}/executions/${encodeURIComponent(executionId)}/iterations`, + pageParams(cursor, limit)), signal); + if (!isRecord(response) || response.repeat_execution_id !== executionId || + !isWorkflowRepeatProgress(response.repeat) || response.repeat.execution_id !== executionId || + !Number.isSafeInteger(response.total_count) || Number(response.total_count) < 0 || + Number(response.total_count) > DEFAULT_FLOW_LIMITS.max_executions || + typeof response.source_snapshot_changed !== 'boolean') { + throw new Error('The Repeat rounds returned an unsupported response.'); + } + const repeat = response.repeat; + const page = pageFromResponse(response, 'iterations', (value): value is WorkflowRepeatIterationRecord => { + if (!isRecord(value) || Object.keys(value).some((key) => ![ + 'iteration', 'iteration_path', 'batch_number', 'batch_size', 'batch_usage', 'state', 'condition_result', + 'execution_ids', 'before_available', 'after_available', 'partial', + ].includes(key)) || !validWorkflowIterationPath(value.iteration_path) || + typeof value.state !== 'string' || !['running', 'completed', 'completed_partial', 'cancelled'].includes(value.state) || + typeof value.partial !== 'boolean' || value.before_available !== true || + typeof value.after_available !== 'boolean' || + value.condition_result !== null && typeof value.condition_result !== 'boolean' || + !value.after_available && value.condition_result !== null || + !Array.isArray(value.execution_ids) || value.execution_ids.length > 256 || !value.execution_ids.every(validIdentity) || + ['iteration', 'batch_number', 'batch_size', 'batch_usage'].some((key) => + typeof value[key] !== 'number' || !Number.isSafeInteger(value[key]) || Number(value[key]) < 0)) return false; + const frame = value.iteration_path.at(-1); + return frame !== undefined && 'iteration' in frame && frame.iteration === value.iteration && frame.loop_id === repeat.node_id && + value.batch_size === repeat.batch_size && Number(value.batch_size) <= MAX_REPEAT_ITERATIONS && + Number(value.batch_usage) >= 1 && Number(value.batch_usage) <= Number(value.batch_size) && + Number(value.batch_number) * Number(value.batch_size) + Number(value.batch_usage) - 1 === value.iteration; + }, (item) => String(item.iteration), limit); + if (page.items.some((item, index) => item.iteration >= Number(response.total_count) || + index > 0 && item.iteration <= page.items[index - 1].iteration)) { + throw new Error('The Repeat round page contains conflicting lifetime identities.'); + } + return { ...page, metadata: { repeat, sourceSnapshotChanged: response.source_snapshot_changed === true } }; +} + +function isRepeatValidation(value: unknown): value is WorkflowValidationResult { + return isRecord(value) && value.version === 1 && + ['valid', 'invalid', 'incomplete', 'accepted_partial', 'not_requested'].includes(String(value.status)) && + (value.eligible === undefined || typeof value.eligible === 'boolean') && + (value.reason_codes === undefined || Array.isArray(value.reason_codes) && value.reason_codes.length <= 100 && + value.reason_codes.every((code) => typeof code === 'string' && code.length <= 256)) && + (value.counts === undefined || isRecord(value.counts) && Object.values(value.counts).every((count) => + typeof count === 'number' && Number.isFinite(count) && count >= 0)); +} + +function isRepeatCoverage(value: unknown): value is WorkflowRepeatStateRecord['coverage'] { + return isRecord(value) && Object.values(value).every((item) => + item === null || typeof item === 'string' || typeof item === 'boolean' || + typeof item === 'number' && Number.isFinite(item)); +} + +function isRepeatStateRecord(value: unknown): value is WorkflowRepeatStateRecord { + if (!isRecord(value) || Object.keys(value).some((key) => + !['name', 'kind', 'source', 'workflow_validation', 'coverage', 'prior_coverage', 'limitations'].includes(key)) || + typeof value.name !== 'string' || !FLOW_ALIAS_PATTERN.test(value.name) || + !REPEAT_STATE_KINDS.some((kind) => kind === value.kind) || !isRecord(value.source) || + !isRepeatValidation(value.workflow_validation) || !isRepeatCoverage(value.coverage) || + value.prior_coverage !== undefined && !isRepeatCoverage(value.prior_coverage) || + !Array.isArray(value.limitations) || !value.limitations.every((item) => typeof item === 'string')) return false; + const source = value.source; + return Object.keys(source).every((key) => ['node_id', 'execution_id', 'task_id', 'iteration_path', 'attempt', 'output_name'].includes(key)) && + validIdentity(source.node_id) && validIdentity(source.execution_id) && + (source.task_id === undefined || validIdentity(source.task_id)) && + typeof source.attempt === 'number' && Number.isSafeInteger(source.attempt) && source.attempt >= 1 && + validWorkflowIterationPath(source.iteration_path) && + typeof source.output_name === 'string' && FLOW_ALIAS_PATTERN.test(source.output_name); +} + +export async function fetchWorkflowRepeatStatePage( + scope: WorkflowScope, + workflowId: string, + runId: string, + executionId: string, + iteration: number, + phase: 'before' | 'after', + cursor: string | null, + limit = 50, + signal?: AbortSignal, +): Promise> { + if (!Number.isSafeInteger(iteration) || iteration < 0 || iteration >= DEFAULT_FLOW_LIMITS.max_executions || + !['before', 'after'].includes(phase)) { + throw new Error('Select an exact admitted Repeat round and before or after state.'); + } + const params = pageParams(cursor, limit); + params.set('phase', phase); + const response = await api.get(workflowUrl(scope, workflowId, + `/runs/${encodeURIComponent(runId)}/executions/${encodeURIComponent(executionId)}/iterations/${iteration}/state`, params), signal); + if (!isRecord(response) || response.repeat_execution_id !== executionId || response.iteration !== iteration || + response.phase !== phase || typeof response.available !== 'boolean' || + !Number.isSafeInteger(response.total_count) || Number(response.total_count) < 0 || Number(response.total_count) > 100 || + response.available && (typeof response.partial !== 'boolean' || typeof response.source_snapshot_changed !== 'boolean') || + !response.available && (phase !== 'after' || response.partial !== undefined || response.source_snapshot_changed !== undefined)) { + throw new Error('The Repeat state returned an unsupported response.'); + } + const page = pageFromResponse(response, 'states', isRepeatStateRecord, (item) => item.name, limit); + if (!response.available && (page.items.length || response.total_count !== 0 || page.next_cursor !== null) || + response.available && (Number(response.total_count) < 1 || page.items.length > Number(response.total_count))) { + throw new Error('The Repeat state page has an invalid availability or count.'); + } + return { ...page, metadata: { + stateAvailable: response.available, partial: response.partial === true, + sourceSnapshotChanged: response.source_snapshot_changed === true, + } }; +} + export async function fetchWorkflowExecutionRecordsPage( scope: WorkflowScope, workflowId: string, @@ -403,7 +576,7 @@ export async function fetchWorkflowExecutionRecordsPage( limit = 100, signal?: AbortSignal, ): Promise> { - if (!validIdentity(executionId) || !Number.isInteger(attempt) || attempt < 1 || !['records', 'documents'].includes(output)) { + if (!validIdentity(executionId) || !Number.isInteger(attempt) || attempt < 1 || !FLOW_ALIAS_PATTERN.test(output)) { throw new Error('The requested execution collection is invalid.'); } const params = pageParams(cursor, limit); diff --git a/application/v2_ui/src/lib/workflowFlow.ts b/application/v2_ui/src/lib/workflowFlow.ts index 49082073c..bdfa2a74c 100644 --- a/application/v2_ui/src/lib/workflowFlow.ts +++ b/application/v2_ui/src/lib/workflowFlow.ts @@ -34,7 +34,27 @@ export interface WorkflowLoopItemSource { scope: 'current'; } -export type WorkflowFlowSource = WorkflowNodeOutputSource | WorkflowLoopItemSource; +export interface WorkflowRepeatStateSource { + kind: 'repeat_state'; + loop_id: string; + state_name: string; + scope: 'current'; +} + +export type WorkflowFlowSource = WorkflowNodeOutputSource | WorkflowLoopItemSource | WorkflowRepeatStateSource; +export type WorkflowRepeatStateKind = Exclude; +export type WorkflowRepeatStateContract = Omit & { + kind: WorkflowRepeatStateKind; + allow_partial?: boolean; + require_complete_coverage?: boolean; +}; + +export interface WorkflowRepeatState { + name: string; + initial: WorkflowNodeOutputSource | WorkflowRepeatStateSource; + next: string; + output_contract: WorkflowRepeatStateContract; +} export interface WorkflowFlowBinding { name: string; @@ -128,7 +148,18 @@ export interface WorkflowCollectNode { output_contract: WorkflowOutputContract & { kind: 'records' | 'document_results' }; } -export type WorkflowFlowNode = WorkflowTaskNode | WorkflowIfNode | WorkflowRouteNode | WorkflowForEachNode | WorkflowCollectNode; +export interface WorkflowRepeatUntilNode { + id: string; + kind: 'repeat_until'; + max_iterations: number; + state: WorkflowRepeatState[]; + body: WorkflowFlowRegion & { outputs: WorkflowFlowBinding[] }; + until: WorkflowPredicate; + exports: { name: string; output: string }[]; +} + +export type WorkflowLoopControl = WorkflowForEachNode | WorkflowRepeatUntilNode; +export type WorkflowFlowNode = WorkflowTaskNode | WorkflowIfNode | WorkflowRouteNode | WorkflowLoopControl | WorkflowCollectNode; export interface WorkflowFlowRegion { id: string; @@ -149,6 +180,8 @@ export const FLOW_MAX_NODES = 256; export const FLOW_MAX_DEPTH = 4; export const DEFAULT_LOOP_MAX_ITEMS = 500; export const MAX_LOOP_ITEMS = 5000; +export const MAX_REPEAT_ITERATIONS = 1000; +export const REPEAT_STATE_KINDS: WorkflowRepeatStateKind[] = ['text', 'json', 'records', 'document_results']; export const FLOW_MAX_PREDICATE_DEPTH = 8; export const FLOW_MAX_PREDICATE_NODES = 100; export const FLOW_ALIAS_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; @@ -161,11 +194,27 @@ export function isFlowBinding(value: unknown): value is WorkflowFlowBinding { const source = value.source; return typeof value.name === 'string' && (source.kind === 'node_output' && typeof source.node_id === 'string' && typeof source.output === 'string' || - source.kind === 'loop_item' && typeof source.loop_id === 'string' && ['json', 'any'].includes(String(value.expected_kind))) && + source.kind === 'loop_item' && typeof source.loop_id === 'string' && ['json', 'any'].includes(String(value.expected_kind)) || + source.kind === 'repeat_state' && typeof source.loop_id === 'string' && typeof source.state_name === 'string') && typeof value.required === 'boolean' && typeof value.allow_partial === 'boolean' && FLOW_OUTPUT_KINDS.some((kind) => kind === value.expected_kind); } +function isRepeatState(value: unknown): value is WorkflowRepeatState { + if (!isRecord(value) || typeof value.name !== 'string' || typeof value.next !== 'string' || + !isRecord(value.initial) || value.initial.scope !== 'current' || !isRecord(value.output_contract)) return false; + const source = value.initial; + const contract = value.output_contract; + return (source.kind === 'node_output' && typeof source.node_id === 'string' && typeof source.output === 'string' || + source.kind === 'repeat_state' && typeof source.loop_id === 'string' && typeof source.state_name === 'string') && + REPEAT_STATE_KINDS.some((kind) => kind === contract.kind) && + (contract.allow_partial === undefined || typeof contract.allow_partial === 'boolean') && + (contract.require_complete_coverage === undefined || typeof contract.require_complete_coverage === 'boolean') && + (contract.schema === undefined || isRecord(contract.schema)) && + (contract.expected_count === undefined || typeof contract.expected_count === 'number') && + (contract.identity_field === undefined || typeof contract.identity_field === 'string'); +} + function isLoopScope(value: unknown): value is WorkflowLoopScope { return isRecord(value) && ['personal', 'group', 'public'].includes(String(value.scope_type)) && (value.scope_id === undefined || typeof value.scope_id === 'string'); @@ -248,6 +297,12 @@ export function isFlowRegion(value: unknown, depth = 0): value is WorkflowFlowRe typeof node.max_items === 'number' && isFlowRegion(node.body, depth + 1) && Array.isArray(node.body.outputs); } + if (node.kind === 'repeat_until') { + return typeof node.max_iterations === 'number' && Array.isArray(node.state) && node.state.every(isRepeatState) && + isFlowRegion(node.body, depth + 1) && Array.isArray(node.body.outputs) && isFlowPredicate(node.until) && + Array.isArray(node.exports) && node.exports.every((output) => + isRecord(output) && typeof output.name === 'string' && typeof output.output === 'string'); + } if (!Array.isArray(node.inputs) || !node.inputs.every(isFlowBinding) || !isFlowPredicate(node.condition)) return false; if (node.kind === 'route') { return isRecord(node.target) && @@ -280,6 +335,17 @@ export function loopItemBinding(name: string, loopId: string): WorkflowFlowBindi }; } +export function repeatStateBinding(name: string, loopId: string, slot: WorkflowRepeatState): WorkflowFlowBinding { + return { + name, source: { kind: 'repeat_state', loop_id: loopId, state_name: slot.name, scope: 'current' }, + expected_kind: slot.output_contract.kind, required: true, allow_partial: false, + }; +} + +export function repeatUntilBindings(node: WorkflowRepeatUntilNode): WorkflowFlowBinding[] { + return node.state.map((slot) => repeatStateBinding(slot.name, node.id, slot)); +} + export interface FlowProducer { id: string; label: string; @@ -293,7 +359,20 @@ export function isRecordsFlowOutput(output: FlowProducer['outputs'][number]): bo export function flowProducers(workflow: WorkflowDefinition): FlowProducer[] { if (!isFlowRegion(workflow.flow)) return []; const tasks = new Map(workflow.tasks.map((task) => [task.id, task])); + const repeats = new Map(flowRepeats(workflow).map(({ node }) => [node.id, node])); const result: FlowProducer[] = []; + const resolveOutput = (source: WorkflowFlowSource): FlowProducer['outputs'][number] | undefined => { + if (source.kind === 'node_output') return result.find((producer) => producer.id === source.node_id)?.outputs + .find((output) => output.name === source.output); + if (source.kind === 'repeat_state') { + const slot = repeats.get(source.loop_id)?.state.find((item) => item.name === source.state_name); + return slot ? { + name: slot.name, kind: slot.output_contract.kind, required: true, + schema: slot.output_contract.schema ?? (slot.output_contract.kind === 'text' ? { type: 'string' } : undefined), + } : undefined; + } + return undefined; + }; const walk = (region: WorkflowFlowRegion) => region.nodes.forEach((node) => { if (node.kind === 'task') { const task = tasks.get(node.task_id); @@ -315,6 +394,21 @@ export function flowProducers(workflow: WorkflowDefinition): FlowProducer[] { result.push({ id: node.id, label: task.name || node.id, outputs }); } else if (node.kind === 'for_each') { walk(node.body); + } else if (node.kind === 'repeat_until') { + walk(node.body); + result.push({ + id: node.id, label: `Repeat ${node.id}`, + outputs: node.exports.map((item) => { + const binding = node.body.outputs.find((output) => output.name === item.output); + const output = binding && resolveOutput(binding.source); + return { + name: item.name, kind: output?.kind ?? binding?.expected_kind ?? 'any', + required: binding?.required === true, + ...(output?.kinds ? { kinds: output.kinds } : {}), + ...(output?.schema ? { schema: output.schema } : {}), + }; + }), + }); } else if (node.kind === 'collect') { result.push({ id: node.id, @@ -361,7 +455,7 @@ export function flowRegions(flow: WorkflowFlowRegion): { id: string; label: stri if (node.kind === 'if') { walk(node.then, `${label} / ${node.id} / Then`, depth + 1); walk(node.else, `${label} / ${node.id} / Else`, depth + 1); - } else if (node.kind === 'for_each') { + } else if (node.kind === 'for_each' || node.kind === 'repeat_until') { walk(node.body, `${label} / ${node.id} / Body`, depth + 1); } }); @@ -384,7 +478,7 @@ export function updateFlowRegion( then: updateFlowRegion(node.then, regionId, update), else: updateFlowRegion(node.else, regionId, update), }; - if (node.kind === 'for_each') { + if (node.kind === 'for_each' || node.kind === 'repeat_until') { const body = updateFlowRegion(node.body, regionId, update); return { ...node, body: { ...body, outputs: body.outputs ?? [] } }; } @@ -396,7 +490,7 @@ export function updateFlowRegion( export function flowTaskIds(node: WorkflowFlowNode): string[] { if (node.kind === 'task') return [node.task_id]; if (node.kind === 'route' || node.kind === 'collect') return []; - if (node.kind === 'for_each') return node.body.nodes.flatMap(flowTaskIds); + if (node.kind === 'for_each' || node.kind === 'repeat_until') return node.body.nodes.flatMap(flowTaskIds); return [...node.then.nodes, ...node.else.nodes].flatMap(flowTaskIds); } @@ -405,7 +499,7 @@ export function flowTaskNodeId(workflow: WorkflowDefinition, taskId: string): st const walk = (region: WorkflowFlowRegion): string => { for (const node of region.nodes) { if (node.kind === 'task' && node.task_id === taskId) return node.id; - const id = node.kind === 'for_each' ? walk(node.body) + const id = node.kind === 'for_each' || node.kind === 'repeat_until' ? walk(node.body) : node.kind === 'if' ? walk(node.then) || walk(node.else) : ''; if (id) return id; } @@ -414,11 +508,11 @@ export function flowTaskNodeId(workflow: WorkflowDefinition, taskId: string): st return walk(workflow.flow); } -export function flowLoops(workflow: WorkflowDefinition): { node: WorkflowForEachNode; regionId: string }[] { - const loops: { node: WorkflowForEachNode; regionId: string }[] = []; +export function flowLoopControls(workflow: WorkflowDefinition): { node: WorkflowLoopControl; regionId: string }[] { + const loops: { node: WorkflowLoopControl; regionId: string }[] = []; if (!isFlowRegion(workflow.flow)) return loops; const walk = (region: WorkflowFlowRegion) => region.nodes.forEach((node) => { - if (node.kind === 'for_each') { + if (node.kind === 'for_each' || node.kind === 'repeat_until') { loops.push({ node, regionId: region.id }); walk(node.body); } else if (node.kind === 'if') { @@ -430,13 +524,21 @@ export function flowLoops(workflow: WorkflowDefinition): { node: WorkflowForEach return loops; } -export function enclosingFlowLoops(workflow: WorkflowDefinition, targetId: string): WorkflowForEachNode[] { +export function flowLoops(workflow: WorkflowDefinition): { node: WorkflowForEachNode; regionId: string }[] { + return flowLoopControls(workflow).flatMap((loop) => loop.node.kind === 'for_each' ? [{ ...loop, node: loop.node }] : []); +} + +export function flowRepeats(workflow: WorkflowDefinition): { node: WorkflowRepeatUntilNode; regionId: string }[] { + return flowLoopControls(workflow).flatMap((loop) => loop.node.kind === 'repeat_until' ? [{ ...loop, node: loop.node }] : []); +} + +export function enclosingFlowLoopControls(workflow: WorkflowDefinition, targetId: string): WorkflowLoopControl[] { if (!isFlowRegion(workflow.flow)) return []; - const walk = (region: WorkflowFlowRegion, parents: WorkflowForEachNode[]): WorkflowForEachNode[] | undefined => { + const walk = (region: WorkflowFlowRegion, parents: WorkflowLoopControl[]): WorkflowLoopControl[] | undefined => { if (region.id === targetId) return parents; for (const node of region.nodes) { if (node.id === targetId || node.kind === 'if' && node.join.id === targetId) return parents; - const found = node.kind === 'for_each' ? walk(node.body, [...parents, node]) + const found = node.kind === 'for_each' || node.kind === 'repeat_until' ? walk(node.body, [...parents, node]) : node.kind === 'if' ? walk(node.then, parents) ?? walk(node.else, parents) : undefined; if (found) return found; } @@ -445,20 +547,41 @@ export function enclosingFlowLoops(workflow: WorkflowDefinition, targetId: strin return walk(workflow.flow, []) ?? []; } +export function enclosingFlowLoops(workflow: WorkflowDefinition, targetId: string): WorkflowForEachNode[] { + return enclosingFlowLoopControls(workflow, targetId).filter((node): node is WorkflowForEachNode => node.kind === 'for_each'); +} + +export function enclosingFlowRepeats(workflow: WorkflowDefinition, targetId: string): WorkflowRepeatUntilNode[] { + return enclosingFlowLoopControls(workflow, targetId).filter((node): node is WorkflowRepeatUntilNode => node.kind === 'repeat_until'); +} + +export function flowSourceOutput(workflow: WorkflowDefinition, source: WorkflowFlowSource): FlowProducer['outputs'][number] | undefined { + if (source.kind === 'node_output') return flowProducers(workflow).find((item) => item.id === source.node_id)?.outputs + .find((item) => item.name === source.output); + if (source.kind === 'repeat_state') { + const slot = flowRepeats(workflow).find(({ node }) => node.id === source.loop_id)?.node.state + .find((item) => item.name === source.state_name); + return slot ? { + name: slot.name, kind: slot.output_contract.kind, required: true, + schema: slot.output_contract.schema ?? (slot.output_contract.kind === 'text' ? { type: 'string' } : undefined), + } : undefined; + } + return undefined; +} + export function flowBindingSchema(workflow: WorkflowDefinition, binding?: WorkflowFlowBinding): Record | undefined { if (!binding) return undefined; const source = binding.source; - const producers = flowProducers(workflow); - if (source.kind === 'node_output') { - return producers.find((item) => item.id === source.node_id)?.outputs.find((item) => item.name === source.output)?.schema; + if (source.kind !== 'loop_item') { + return flowSourceOutput(workflow, source)?.schema; } const loop = flowLoops(workflow).find((item) => item.node.id === source.loop_id)?.node; let value: Record = {}; if (loop?.iterable.kind === 'input') { const name = loop.iterable.name; const input = loop.inputs.find((item) => item.name === name)?.source; - if (input?.kind === 'node_output') { - const schema = producers.find((item) => item.id === input.node_id)?.outputs.find((item) => item.name === input.output)?.schema; + if (input) { + const schema = flowSourceOutput(workflow, input)?.schema; if (isRecord(schema?.items)) value = schema.items; } } else if (loop) { @@ -475,6 +598,26 @@ export function workflowLoopLimit(options: WorkflowEditorOptions): number { ? Number(limit) : DEFAULT_LOOP_MAX_ITEMS; } +export function workflowRepeatLimit(options: WorkflowEditorOptions): number | undefined { + const limit = options.flow_limits?.max_repeat_iterations; + return options.flow_limits?.hard_repeat_iterations === MAX_REPEAT_ITERATIONS && + typeof limit === 'number' && Number.isInteger(limit) && limit >= 1 && limit <= MAX_REPEAT_ITERATIONS ? limit : undefined; +} + +export function supportsWorkflowRepeat(options: WorkflowEditorOptions): boolean { + return options.supported_node_kinds?.includes('repeat_until') === true && + options.supported_binding_sources?.includes('repeat_state') === true && workflowRepeatLimit(options) !== undefined; +} + +export function repeatIterationErrors(node: WorkflowRepeatUntilNode, ceiling = MAX_REPEAT_ITERATIONS): string[] { + if (!Number.isInteger(node.max_iterations) || node.max_iterations < 1 || node.max_iterations > MAX_REPEAT_ITERATIONS) { + return ['Choose an explicit maximum rounds before manual continuation: a whole number from 1 to 1,000.']; + } + return node.max_iterations > ceiling + ? [`Maximum rounds ${node.max_iterations} exceeds the current administrator ceiling of ${ceiling} for new runs. The authored value is preserved; choose an allowed maximum before starting a new run.`] + : []; +} + export function loopSelectionErrors(node: WorkflowForEachNode, ceiling = MAX_LOOP_ITEMS): string[] { const errors: string[] = []; const limit = Math.min(node.max_items, ceiling); @@ -636,19 +779,20 @@ export function flowUnsupportedReason(workflow: WorkflowDefinition, options?: Wo return; } onlyFields(binding, ['name', 'source', 'required', 'expected_kind', 'allow_partial'], errors, 'Input'); - onlyFields(binding.source, binding.source.kind === 'loop_item' - ? ['kind', 'loop_id', 'scope'] : ['kind', 'node_id', 'output', 'scope'], errors, 'Input source'); - if (options && binding.source.kind === 'loop_item' && !options.supported_binding_sources?.includes('loop_item')) { - errors.push('This server does not support current loop item bindings. The saved definition is preserved.'); + onlyFields(binding.source, binding.source.kind === 'loop_item' ? ['kind', 'loop_id', 'scope'] + : binding.source.kind === 'repeat_state' ? ['kind', 'loop_id', 'state_name', 'scope'] + : ['kind', 'node_id', 'output', 'scope'], errors, 'Input source'); + if (options && binding.source.kind !== 'node_output' && !options.supported_binding_sources?.includes(binding.source.kind)) { + errors.push('This server does not support the saved current loop item or Repeat state binding. The saved definition is preserved.'); } }); - const outputContract = (contract: WorkflowOutputContract) => + const outputContract = (contract: WorkflowOutputContract | WorkflowRepeatStateContract) => onlyFields(contract, ['kind', 'schema', 'expected_count', 'identity_field', 'require_complete_coverage', 'allow_partial'], errors, 'Output contract'); const walk = (region: WorkflowFlowRegion, exports = false) => { onlyFields(region, exports ? ['id', 'nodes', 'outputs'] : ['id', 'nodes'], errors, 'Region'); bindings(region.outputs ?? []); region.nodes.forEach((node) => { - if (options && ['for_each', 'collect'].includes(node.kind) && !options.supported_node_kinds?.includes(node.kind)) { + if (options && ['for_each', 'collect', 'repeat_until'].includes(node.kind) && !options.supported_node_kinds?.includes(node.kind)) { errors.push(`This server does not support ${node.kind} nodes. The saved definition is preserved.`); } if (node.kind === 'task') { @@ -658,6 +802,18 @@ export function flowUnsupportedReason(workflow: WorkflowDefinition, options?: Wo onlyFields(node, ['id', 'kind', 'source', 'output_contract'], errors, 'Collect'); onlyFields(node.source, ['loop_id', 'output'], errors, 'Collect source'); outputContract(node.output_contract); + } else if (node.kind === 'repeat_until') { + onlyFields(node, ['id', 'kind', 'max_iterations', 'state', 'body', 'until', 'exports'], errors, 'Repeat until'); + if (options && !supportsWorkflowRepeat(options)) errors.push('This server does not expose supported Repeat state and iteration policy capabilities. The saved definition is preserved.'); + node.state.forEach((slot) => { + onlyFields(slot, ['name', 'initial', 'next', 'output_contract'], errors, 'Repeat state'); + bindings([{ name: slot.name, source: slot.initial, required: true, + expected_kind: slot.output_contract.kind, allow_partial: slot.output_contract.allow_partial === true }]); + outputContract(slot.output_contract); + }); + node.exports.forEach((item) => onlyFields(item, ['name', 'output'], errors, 'Repeat export')); + predicate(node.until); + walk(node.body, true); } else if (node.kind === 'for_each') { onlyFields(node, ['id', 'kind', 'inputs', 'iterable', 'item_key', 'max_items', 'body'], errors, 'For each'); bindings(node.inputs); @@ -789,9 +945,10 @@ export function analyzeWorkflowFlow(workflow: WorkflowDefinition): WorkflowFlowA register(node.join.id); shape(node.then, depth + 1); shape(node.else, depth + 1); - } else if (node.kind === 'for_each') { + } else if (node.kind === 'for_each' || node.kind === 'repeat_until') { shape(node.body, depth + 1); - errors.push(...loopSelectionErrors(node).map((error) => `${node.id}: ${error}`)); + errors.push(...(node.kind === 'for_each' ? loopSelectionErrors(node) : repeatIterationErrors(node)) + .map((error) => `${node.id}: ${error}`)); } }); }; @@ -814,7 +971,7 @@ export function analyzeWorkflowFlow(workflow: WorkflowDefinition): WorkflowFlowA errors.push(`${label} has an expected kind that disagrees with its producer.`); } }; - const checkBindings = (bindings: WorkflowFlowBinding[], state: Availability, label: string, parents: WorkflowForEachNode[]) => { + const checkBindings = (bindings: WorkflowFlowBinding[], state: Availability, label: string, parents: WorkflowLoopControl[]) => { const names = new Set(); bindings.forEach((binding) => { if (!FLOW_ALIAS_PATTERN.test(binding.name)) errors.push(`${label} needs valid input aliases (letter first, up to 64 letters, digits, underscores or dashes).`); @@ -822,8 +979,16 @@ export function analyzeWorkflowFlow(workflow: WorkflowDefinition): WorkflowFlowA names.add(binding.name); if (binding.source.kind === 'loop_item') { const loopId = binding.source.loop_id; - if (!parents.some((node) => node.id === loopId)) errors.push(`${label}: ${binding.name} must select a current item from an enclosing For each.`); + if (!parents.some((node) => node.kind === 'for_each' && node.id === loopId)) errors.push(`${label}: ${binding.name} must select a current item from an enclosing For each.`); if (!['json', 'any'].includes(binding.expected_kind) || binding.allow_partial) errors.push(`${label}: current items are complete JSON values, not partial outputs.`); + } else if (binding.source.kind === 'repeat_state') { + const source = binding.source; + const repeat = parents.find((node): node is WorkflowRepeatUntilNode => node.kind === 'repeat_until' && node.id === source.loop_id); + const slot = repeat?.state.find((item) => item.name === source.state_name); + if (!slot) errors.push(`${label}: ${binding.name} must select a named current state slot from an enclosing Repeat.`); + else if (binding.expected_kind !== 'any' && binding.expected_kind !== slot.output_contract.kind) { + errors.push(`${label}: ${binding.name} must retain its Repeat state's declared kind.`); + } } else checkSource(binding.source, binding.required, state, `${label}: ${binding.name || 'input'}`, binding.expected_kind); }); }; @@ -884,7 +1049,8 @@ export function analyzeWorkflowFlow(workflow: WorkflowDefinition): WorkflowFlowA walk(condition, 1); if (count > FLOW_MAX_PREDICATE_NODES || JSON.stringify(condition).length > 16384) errors.push(`${label} exceeds the condition size limit.`); }; - const walk = (region: WorkflowFlowRegion, incoming: Availability, parents: WorkflowForEachNode[] = [], branch = false): Availability => { + const walk = (region: WorkflowFlowRegion, incoming: Availability, parents: WorkflowLoopControl[] = [], branch = false): Availability => { + if (parents.length > 3) errors.push('At most three enclosing For each or Repeat frames are supported.'); const inputs = new Map(); const exits: Availability[] = []; let current = incoming; @@ -897,7 +1063,7 @@ export function analyzeWorkflowFlow(workflow: WorkflowDefinition): WorkflowFlowA const bindings = (task.inputs ?? []).filter(isFlowBinding); checkBindings(bindings, current, task.name || node.id, parents); if (task.document_action?.target_mode === 'current_item') { - const loop = parents.find((item) => item.id === task.document_action?.loop_id); + const loop = parents.find((item): item is WorkflowForEachNode => item.kind === 'for_each' && item.id === task.document_action?.loop_id); if (!loop || loop.iterable.kind === 'input') { errors.push(`${task.name}: current-document Analyze requires an enclosing document selection or workspace query, not a saved record containing a document ID.`); } @@ -936,28 +1102,67 @@ export function analyzeWorkflowFlow(workflow: WorkflowDefinition): WorkflowFlowA const name = node.iterable.name; const binding = node.inputs.find((item) => item.name === name); const source = binding?.source; - const output = source?.kind === 'node_output' - ? producers.get(source.node_id)?.outputs.find((item) => item.name === source.output) : undefined; - if (!binding || !binding.required || binding.allow_partial || source?.kind !== 'node_output' || + const output = source && flowSourceOutput(workflow, source); + if (!binding || !binding.required || binding.allow_partial || + source?.kind !== 'node_output' && source?.kind !== 'repeat_state' || !output || !['records', 'document_results'].includes(output.kind)) { - errors.push(`${node.id}: choose a required, complete saved records or document-results producer.`); + errors.push(`${node.id}: choose a required, complete saved records or document-results producer or current Repeat state.`); } } const bodyEnd = walk(node.body, current, [...parents, node]); node.body.outputs.forEach((binding) => { if (binding.source.kind !== 'node_output' || - !sameEditorValue(enclosingFlowLoops(workflow, binding.source.node_id).map((loop) => loop.id), [...parents, node].map((loop) => loop.id))) { + !sameEditorValue(enclosingFlowLoopControls(workflow, binding.source.node_id).map((loop) => loop.id), [...parents, node].map((loop) => loop.id))) { errors.push(`Body outputs for ${node.id} must select an exact producer in this loop's body scope.`); } }); checkBindings(node.body.outputs, bodyEnd, `Body outputs for ${node.id}`, [...parents, node]); current = withProducer(current, node.id, true); + } else if (node.kind === 'repeat_until') { + if (!node.state.length || node.state.length > 100) errors.push(`${node.id}: declare between 1 and 100 named state slots.`); + checkBindings(node.state.map((slot) => ({ + name: slot.name, source: slot.initial, required: true, expected_kind: slot.output_contract.kind, + allow_partial: slot.output_contract.allow_partial === true, + })), current, `Initial state for ${node.id}`, parents); + const bodyParents = [...parents, node]; + const bodyEnd = walk(node.body, current, bodyParents); + checkBindings(node.body.outputs, bodyEnd, `Body outputs for ${node.id}`, bodyParents); + node.body.outputs.forEach((binding) => { + if (binding.source.kind === 'loop_item' || + binding.source.kind === 'node_output' && !sameEditorValue( + enclosingFlowLoopControls(workflow, binding.source.node_id).map((loop) => loop.id), + bodyParents.map((loop) => loop.id))) { + errors.push(`Body outputs for ${node.id} must select a producer in this body scope or explicitly pass current Repeat state unchanged.`); + } + }); + node.state.forEach((slot) => { + const next = node.body.outputs.find((binding) => binding.name === slot.next); + if (!next || !next.required) errors.push(`${node.id}: state ${slot.name} needs a required, declared next body output.`); + const initialOutput = flowSourceOutput(workflow, slot.initial); + const nextOutput = next && flowSourceOutput(workflow, next.source); + for (const [phase, output] of [['Initial', initialOutput], ['Next', nextOutput]] as const) { + if (output && (output.kinds ?? [output.kind]).some((kind) => kind !== slot.output_contract.kind)) { + errors.push(`${node.id}: ${phase} state ${slot.name} must have exactly kind ${slot.output_contract.kind}; kinds are never coerced.`); + } + } + if (slot.output_contract.kind === 'json' && !slot.output_contract.schema) { + errors.push(`${node.id}: JSON state ${slot.name} needs an explicit supported schema.`); + } + }); + checkPredicate(node.until, repeatUntilBindings(node), `Stop after a round for ${node.id}`); + const names = new Set(); + node.exports.forEach((item) => { + if (!FLOW_ALIAS_PATTERN.test(item.name) || names.has(item.name)) errors.push(`${node.id}: final exports need unique, valid names.`); + names.add(item.name); + if (!node.body.outputs.some((binding) => binding.name === item.output)) errors.push(`${node.id}: final export ${item.name} selects a missing body output.`); + }); + current = withProducer(current, node.id, true); } else if (node.kind === 'collect') { const source = loops.get(node.source.loop_id); const output = source?.node.body.outputs.find((item) => item.name === node.source.output); if (!source || !output) errors.push(`${node.id}: select an existing loop and declared body output for Collect.`); else { - if (!sameEditorValue(enclosingFlowLoops(workflow, source.node.id).map((loop) => loop.id), parents.map((loop) => loop.id)) || + if (!sameEditorValue(enclosingFlowLoopControls(workflow, source.node.id).map((loop) => loop.id), parents.map((loop) => loop.id)) || !current.definite.has(source.node.id)) { errors.push(`${node.id}: Collect must follow its loop in that loop's enclosing scope on every reaching path.`); } diff --git a/docs/admin/workflow.md b/docs/admin/workflow.md index 94b2fe591..d2e02b107 100644 --- a/docs/admin/workflow.md +++ b/docs/admin/workflow.md @@ -57,9 +57,10 @@ visible to its members. Turning one on does not turn on the other. | Enable Group Workflows | Lets permitted members create, manage and run workflows from group workspaces. Owners and Admins may author them unless Workspaces restricts group agent, action and workflow management to Owners. | Off | `allow_group_workflows` | | Require Group Assignment to Use Workflow | Narrows group workflows to an explicit allow list instead of every group. Groups outside the list lose the capability. | Off | `require_group_assignment_for_group_workflows` | | Assigned Groups | The groups that may use group workflows while assignment is required. Ignored when it is not. | Empty list | `group_workflow_allowed_group_ids` | -| Workflow Agent Action Limit | Caps the automatic tool and action calls an agent may make in one workflow run, which is what stops a run from looping. Large document sets need a higher cap. Values above 100 are capacity-sensitive: enable Cosmos DB throughput automation and watch Azure OpenAI throttling, App Service CPU and memory, and downstream latency. | 60 | `workflow_max_auto_invoke_attempts` | +| Workflow Agent Action Limit | Caps the automatic tool and action calls an agent may make in one workflow run, independently of authored For each or Repeat blocks. Large document sets may need a higher cap. Values above 100 are capacity-sensitive: enable Cosmos DB throughput automation and watch Azure OpenAI throttling, App Service CPU and memory, and downstream latency. | 60 | `workflow_max_auto_invoke_attempts` | | Workflow Task Limit | Caps the ordered instruction tasks a single workflow may contain. Supported range is 1–100. | 50 | `workflow_max_tasks` | | Workflow Loop Item Limit | Bounds the actual per-item body visits in a For each block, not the number of documents that may be searched. A collection above the effective limit must be narrowed before its body can run; it is never silently trimmed. | 500 | `workflow_max_loop_items`; supported range 1-5,000; applies to new runs | +| Workflow Repeat Iteration Limit | Bounds one automatic Repeat until batch, including its first round. Authors must choose an explicit per-block maximum; a new run above this ceiling is rejected rather than shortened. | 25 | `workflow_max_repeat_iterations`; supported range 1-1,000; new runs only; active runs and manual continuation retain the admitted policy | The action and task limits apply to personal and group runs alike, so they stay in effect whichever capability is enabled. @@ -121,6 +122,32 @@ is no cumulative run-token/spend cap in this slice. See [Serial For each and exact Collect](../explanation/features/WORKFLOW_FOR_EACH_COLLECT.md) for retained-data behavior, partial coverage, and inspection. +### Repeat batches and manual continuation + +Version **0.261.120** adds **Repeat until** to the durable V2 List editor. +Its separate setting limits automatic rounds, not selected document counts. +The value 25 is the administrator default, never an implicit authored block +maximum. A saved workflow above a newly lowered ceiling stays unchanged, but +cannot start a new run until its authored maximum or administrator policy is +deliberately adjusted. + +When the condition remains false at the block's maximum, the run pauses with +its saved state and earlier rounds retained. An authorized person can explicitly +grant another batch of the same frozen size. Changing this administrator +setting cannot enlarge an active run, and ordinary Resume cannot grant a batch. + +Manual continuation resets only batch usage. It preserves lifetime round +numbers, the admitted execution budget (at most 5,000), and the original elapsed +deadline (at most 86,400 seconds, including the time waiting for a person). +Remaining global budgets can prevent the grant or stop a later round before +the batch allowance is used. + +The run retains bounded exhaustion and continuation counters/audit records. +Control Center personal-user monitoring, group monitoring, and a dedicated +**Workflow Monitoring** section are an approved future follow-up, **not +implemented here**. This setting adds no monitoring role or cross-run access. +See [Repeat until](../explanation/features/WORKFLOW_REPEAT_UNTIL.md). + ## Common tasks ### Publication completion @@ -158,6 +185,8 @@ See [Workflow publication completion](../explanation/features/WORKFLOW_PUBLICATI | A group has no Workflows section | Group workflows are off, or assignment is required and the group is not assigned. | Check Enable Group Workflows, then add the group under Assigned Groups. | | A workflow run stops before its last task | The run hit the agent action limit. | Raise Workflow Agent Action Limit, and review capacity before going above 100. | | A workflow rejects a new task | The workflow already holds the maximum number of tasks. | Raise Workflow Task Limit, or split the work across two workflows. | +| A saved Repeat workflow cannot start a new run | Its explicit block maximum exceeds the current Repeat ceiling. | Deliberately reduce the authored maximum or adjust Workflow Repeat Iteration Limit; the app does not silently clamp it. | +| Repeat pauses with its condition unmet | The automatic batch ended, or a separate global budget blocked progress. | Inspect the gate and remaining budgets. Only a Repeat-limit gate can receive an explicit same-sized manual continuation; global budget exhaustion cannot be reset. | ## Related diff --git a/docs/explanation/features/WORKFLOW_DURABLE_EXECUTION.md b/docs/explanation/features/WORKFLOW_DURABLE_EXECUTION.md index cb64d8c6c..ea2a744eb 100644 --- a/docs/explanation/features/WORKFLOW_DURABLE_EXECUTION.md +++ b/docs/explanation/features/WORKFLOW_DURABLE_EXECUTION.md @@ -2,6 +2,8 @@ Implemented in version: **0.261.111** +Updated in version: **0.261.120**. + Application version tracking: `application/single_app/config.py`. Structured control-flow integration in **0.261.116** adds an opt-in version-3 @@ -101,7 +103,7 @@ provider response or make an arbitrary external tool transactional. | `waiting_approval` | A task is blocked on an authorized human decision. | | `waiting_output` | A submitted background operation has not produced the required final output. | | `waiting_recovery` | An interrupted or failed operation may have performed external actions; review is required before replay. | -| `paused` | Changed inputs, unavailable access, or an unsupported continuation needs attention. | +| `paused` | Changed inputs, unavailable access, an unmet Repeat batch limit, or an unsupported continuation needs attention. | | `completed` / `completed_partial` | The normal output-validation policy determines the final deliverable outcome. | | `failed` / `invalid` / `incomplete` | A task or deliverable requirement was not satisfied. | | `cancelled` | Cancellation fenced further checkpoint/publication work. | @@ -118,6 +120,31 @@ ownership or group management rights are checked at the decision endpoint. Readers without decision rights can inspect authorized progress but cannot approve or resume it. +### Finite Repeat batches + +[Repeat until](WORKFLOW_REPEAT_UNTIL.md) adds post-body state transitions in +**0.261.120**. Each block requires an explicit maximum for one automatic batch. +New runs must fit the separate administrator ceiling (default 25, range +1-1,000); admitted runs freeze that policy and authored batch size. + +An unmet condition at the batch maximum creates a `paused` gate with +`reason_code: "repeat_iteration_limit"` and choices `continue_repeat` or +`cancel`. Ordinary Resume is not a batch grant. The decision is bound to the +exact exhausted transition, saved next state, gate/version, and request ID. +Replays and duplicate acknowledgments cannot grant extra batches. + +Manual continuation grants the same batch size and resets only its usage. +Lifetime round indexes, accumulated execution admissions (at most 5,000), and +elapsed deadline (at most 86,400 seconds) remain intact. Human waiting counts +toward the deadline. The grant cannot repair invalid state, clear another +pause, approve body tasks, or authorize destination publication. + +Committed rounds and before/after state remain reference-based and inspectable. +Exhaustion/continuation counters and audit records are durable; sanitized event +delivery is not a separate execution ledger or a promise of exactly-once +telemetry. Control Center workflow monitoring remains a separate future +capability. + ### Readiness Pending output is not task success. The run retains the original child-run @@ -199,5 +226,6 @@ after reload, and explicit decision permissions. No new Cosmos container or external workflow service is required. Structured If/else, Run when, and restricted forward routing are available through the -version-3 List editor. General For each, Repeat until, exact Collect, and visual -Flow authoring remain later milestones. +version-3 List editor. Serial For each and exact Collect were added in +**0.261.117**, followed by finite Repeat until in **0.261.120**. M5A read-only +Flow and M5B accessible visual authoring remain separate later milestones. diff --git a/docs/explanation/features/WORKFLOW_FOR_EACH_COLLECT.md b/docs/explanation/features/WORKFLOW_FOR_EACH_COLLECT.md index a31959cc4..cf431a70e 100644 --- a/docs/explanation/features/WORKFLOW_FOR_EACH_COLLECT.md +++ b/docs/explanation/features/WORKFLOW_FOR_EACH_COLLECT.md @@ -2,7 +2,7 @@ Implemented in version: **0.261.117** -Updated in version: **0.261.119**. +Updated in version: **0.261.120**. Application version source: `application\single_app\config.py`. @@ -27,13 +27,18 @@ The List editor supports three input sources: | Source | What becomes an item | Ordering | | --- | --- | --- | | Selected documents | Each explicitly selected, currently authorized document | Selection order | -| Saved collection | Each complete record or per-document result from a declared earlier output | Original record order | +| Saved collection | Each complete record or per-document result from a declared earlier output, or from named current Repeat state inside its body | Original record order | | Workspace query | Each distinct authorized logical document selected when the loop starts | Stable identity order for exhaustive selection, or the selected ranking for Best N | Searching a workspace does not make every searchable document an item. A search can cover a large workspace and select 80 documents; processing that selection creates 80 loop visits. +In **0.261.120**, a For each inside Repeat may select a collection through a +required, non-partial `repeat_state` binding. The frozen source retains that +round's exact state receipt. Reading it requires authorization of that receipt; +a prior-node ancestry prefix or a lookup of the latest state is insufficient. + ### Frozen workspace queries **All matches** means exhaustive metadata/filter or keyword-content matching. @@ -71,6 +76,12 @@ policy and frozen inputs. Raising the item ceiling does not increase the separate execution-admission ceiling or elapsed deadline. A multi-task body can exhaust those limits before using every permitted item. +Since **0.261.120**, [Repeat until](WORKFLOW_REPEAT_UNTIL.md) has a separate +administrator ceiling: default 25 rounds, range 1-1,000 per automatic batch. +Changing that setting does not change For each's default 500 items or its +1-5,000 range. A Repeat continuation cannot reset the shared run's execution +admissions or elapsed deadline. + Model context capacity is a different limit. It comes from each task's effective catalog/deployment settings, including instructions, tools, and output reservation. This slice does not impose a cumulative run-token or spending cap. @@ -109,9 +120,11 @@ Each logical task appears once in the authored tree, but executes separately for each item. Ancestor outputs can be explicit inputs; another item's latest reply cannot. -Body results cross the loop boundary through Collect, not through an implicit -last-child output. Nested loop results must cross their own Collect boundary -before an outer body can export them. +For each body results cross the loop boundary through Collect, not through an +implicit last-child output. Nested For each results must cross their own +Collect boundary before an outer body can export them. A nested Repeat instead +exposes its explicitly selected final exports after Until is satisfied; it is +not an implicit Collect of all earlier rounds. Collect supports homogeneous `records` and `document_results` exports and preserves the selected kind. Text concatenation and arbitrary JSON-object @@ -152,6 +165,11 @@ records, page coverage, intermediate checkpoints, and source-linked support. This mode requires text output, declared collection inputs, no document action or publication, and a locally metered runner. +Inside a Repeat body, those collection inputs can explicitly select +`repeat_state`. This reads the exact admitted collection and retains its +original records and applicable partial policies. It does not authorize direct +state publication or make saved-record reporting an automatic fallback. + Ordinary full-input tasks do not silently become summary tasks. If the complete input cannot fit and cannot safely be split for the requested operation, the run pauses with the original data retained. Compact interpretations are not a @@ -182,6 +200,14 @@ A retry retains its execution ID and advances its attempt. Approval and recovery bind the exact item, execution, attempt, input digest, and gate. Approval for item A cannot authorize item B, even when their data looks identical. +Repeat's distinct `{loop_id, iteration}` frame can appear in the same path +without changing For each frames or their execution hashes. The Repeat index +counts lifetime rounds, not the position inside a newly granted batch. +A For each inside Repeat freezes its inputs once per exact inner-loop +execution. Resume reuses them; a genuinely new outer round creates a new +inner-loop identity and can run its authored selection afresh. Previously +frozen selections remain unchanged. + V2 inspection pages show frozen items, their executions and attempts, complete record pages, and contributor receipts. Byte excerpts remain separate transport views and may show index metadata; they are not semantic record pages. @@ -253,9 +279,10 @@ the source contract, authorization, recovery and validation commands. Generic CSV, Markdown, Word/DOCX, PDF, PowerPoint/PPTX and XML mappings remain future extensions of that same shared framework; existing native formats keep -their behavior. Repeat until, M5 read-only Flow and accessible visual authoring -remain separate future slices. Parallel iteration and hosted-agent loops remain -unsupported; cumulative run-token/spending caps remain deferred. +their behavior. Repeat until is added in **0.261.120** without changing this +For each/Collect contract. M5A read-only Flow and M5B accessible visual +authoring remain separate future slices. Parallel iteration and hosted-agent +loops remain unsupported; cumulative run-token/spending caps remain deferred. Validation uses fictional data and isolated services. It is not evidence of a live deployment or permission change. diff --git a/docs/explanation/features/WORKFLOW_PUBLICATION_COMPLETION.md b/docs/explanation/features/WORKFLOW_PUBLICATION_COMPLETION.md index a7564e421..fcfbc737c 100644 --- a/docs/explanation/features/WORKFLOW_PUBLICATION_COMPLETION.md +++ b/docs/explanation/features/WORKFLOW_PUBLICATION_COMPLETION.md @@ -4,6 +4,8 @@ Implemented in version: **0.261.118**. Saved-output integration implemented in version: **0.261.119**. +Updated in version: **0.261.120** for Repeat final records. + Application version tracking: `application/single_app/config.py`. ## Purpose and scope @@ -28,6 +30,13 @@ Analyze artifacts. See [Saved workflow output publication](WORKFLOW_SAVED_OUTPUT_PUBLICATION.md) for source eligibility, partial acceptance and file materialization. +In **0.261.120**, a satisfied Repeat boundary can supply an eligible final +records export to that same source path. An exhausted automatic batch does +not complete Repeat or expose its final exports. A manual continuation grants +neither task approval nor destination approval, and cannot clear a publication +wait or reset the run's original deadline. The shared renderer, source-bound +immutable file, sole destination ledger, and completion proof remain unchanged. + ## Choose the completion level In the V2 List editor, enable **Publish a workflow file** and choose **Existing diff --git a/docs/explanation/features/WORKFLOW_REPEAT_UNTIL.md b/docs/explanation/features/WORKFLOW_REPEAT_UNTIL.md new file mode 100644 index 000000000..7004b86ae --- /dev/null +++ b/docs/explanation/features/WORKFLOW_REPEAT_UNTIL.md @@ -0,0 +1,529 @@ +# Repeat until with saved typed state + +Implemented in version: **0.261.120**. + +Application version tracking: `application\single_app\config.py`. + +## Overview and dependencies + +Repeat until runs a serial body at least once, saves its next state, and then +evaluates a typed stopping condition. Use it for a bounded refinement process, +such as improving a report while retaining its latest draft and a separate +structured review decision. A model saying "finished" is not an engine command. + +This M4C-3 capability extends definition-version-3 +[structured workflows](WORKFLOW_STRUCTURED_CONTROL_FLOW.md), the existing +[durable runner](WORKFLOW_DURABLE_EXECUTION.md), schema-2 journal, exact result +readers, and personal/group permissions. It requires durable execution and +locally metered models or local agents. Ordinary hosted non-loop workflows +remain supported. There is no second scheduler, result store, publication +ledger, database container, or browser runtime dependency. + +## Administrator policy and authored limits + +| Control | Value and meaning | +| --- | --- | +| **Workflow Repeat Iteration Limit** | `workflow_max_repeat_iterations`; administrator default **25**, supported range **1-1,000** | +| **Maximum rounds before manual continuation** | Required explicit `max_iterations` on each Repeat block; includes the first body iteration | +| Technical batch ceiling | **1,000** iterations per automatic batch, not a lifetime limit of 1,000 | +| Run execution admissions | Existing authored/admitted budget, at most **5,000**, shared with tasks, retries, and nested control work | +| Run elapsed deadline | Existing authored/admitted deadline, at most **86,400 seconds**, including human and output waits | + +The administrator default is not an authored default. A missing block maximum +is invalid; the editor starts that field unset. A new run whose authored +maximum exceeds the current administrator ceiling is rejected, not clamped or +silently shortened. No unadmitted snapshot is written for that rejected +submission. Lowering the ceiling does not rewrite saved definitions. + +An admitted run freezes its Repeat policy and batch size. Administrator changes +affect new runs only, including when an existing run later receives a manual +continuation. A grant of another batch does not promise that the remaining +global budgets can accommodate the whole batch. + +Editor `flow_limits.max_repeat_iterations` shows the current administrator +ceiling. Runtime `limits.max_repeat_iterations` is the frozen admitted ceiling; +`gate.repeat.batch_size` is the particular block's frozen authored maximum. +The administrator ceiling is not a replacement for that authored batch size. + +The separate **Workflow Loop Item Limit** remains default **500**, range +**1-5,000**, for actual selected For each items. Searching many documents is +not itself a Repeat iteration or a For each visit. Neither setting is a model +context limit or cumulative run-token/spending cap. + +Configure the policy in [Workflow settings](../../admin/workflow.md). + +## Definition contract + +Keep `definition_version: 3` and `durable_execution: true`. The `tasks` array +remains the task catalogue; `flow` is the executable structure. + +| Repeat field | Contract | +| --- | --- | +| `id` | Unique stable engine-node ID | +| `kind` | Exactly `repeat_until` | +| `max_iterations` | Required positive integer, at most 1,000 per automatic batch | +| `state` | One to 100 uniquely named typed state declarations | +| `body` | Region with `id`, `nodes`, and explicit `outputs` | +| `until` | Existing bounded data-only predicate, selecting validated next-state slots by name | +| `exports` | Explicit final names selecting declared body-output names; an explicit empty list is allowed | + +Every state declaration has `name`, `initial`, `next`, and `output_contract`. +For example, this declaration carries a schema-validated review decision: + +```json +{ + "name": "review", + "initial": { + "kind": "node_output", + "node_id": "seed_review", + "output": "json", + "scope": "current" + }, + "next": "next_review", + "output_contract": { + "kind": "json", + "schema": { + "type": "object", + "required": ["ready"], + "properties": {"ready": {"type": "boolean"}} + }, + "allow_partial": false + } +} +``` + +Here `seed_review` must be a visible eligible earlier producer, and +`next_review` must be an explicitly declared body output. This is a declaration +excerpt, not a complete workflow. + +### Initial and current state + +State kinds must be explicit: `text`, `json`, `records`, or `document_results`, +not `any`. Initialize each slot from an exact saved output visible at the +Repeat entry, or from explicitly named enclosing Repeat state. Starting +literals, arbitrary document IDs, result-store locators, foreign-run +references, and "latest result" selectors are not supported. + +A body task selects the state admitted for its exact round: + +```json +{ + "name": "review", + "source": { + "kind": "repeat_state", + "loop_id": "refine_report", + "state_name": "review", + "scope": "current" + }, + "required": true, + "expected_kind": "json", + "allow_partial": false +} +``` + +The admitted state is read-only throughout that body. A nested Repeat can read +named enclosing state but cannot mutate it. The outer state changes only at +the outer transition. + +Each slot needs its own explicit next-body-output selection. To retain a slot +unchanged, export its current-state receipt from the body rather than asking a +model to echo the value. There is no implicit merge, append, transcript +accumulation, deduplication, truncation, or conversion into downloaded files. + +A For each inside the body can select a saved collection through a required, +non-partial `repeat_state` input. Its frozen source must retain and authorize +the exact admitted state receipt, not apply an earlier-node ancestry shortcut. +Explicit **Saved-record report** collection bindings can also select current +Repeat state; the report keeps the original collection stored and follows the +existing processing and partial-data rules. Neither path permits direct +`repeat_state` publication. + +### Transition and condition + +After all required body work completes, the engine resolves every next-state +output, checks its exact producer and slot contract, and evaluates Until +against that **next** state. JSON decision fields need an explicit supported +schema. The existing typed comparison, ordered short-circuit, missing/null, +and `exists` rules apply. + +An example condition for the declaration above is: + +```json +{ + "op": "eq", + "left": {"input": "review", "path": "/ready"}, + "right": {"literal": true} +} +``` + +Even if the initial state already meets that condition, the body runs once. +A true condition on the last permitted round succeeds. A false condition at +the batch maximum pauses without completing the Repeat boundary or releasing +its final exports. + +Body outputs do not leak across the boundary as "the last task." A downstream +task binds a named Repeat export through `node_output`, for example an export +declared as `{"name": "review", "output": "next_review"}`. Use explicit joins +when different branches provide a required next value. + +Schema validity does not prove factual correctness. An over-budget condition +read or model input is an explicit retained-data blocker, not permission to +evaluate a prefix, summarize automatically, or silently switch sources. + +## Durable state and mixed execution paths + +Large values remain at their original result references. Private state +snapshots retain exact slot receipts, validation, coverage, and predecessor +transitions; they are not a growing transcript in the runtime control row. +Earlier saved state versions and original outputs remain retained. + +A Repeat frame is exactly: + +```json +{"loop_id": "refine_report", "iteration": 26} +``` + +`iteration` is the zero-based **lifetime** index for this exact Repeat +invocation. The example is displayed as round 27. It does not reset after a +manual continuation. Existing For each frames remain +`{loop_id, item_id, index}` without a new discriminator or changed hashes. + +Mixed nesting retains the existing limits of four regions including the root +and at most three enclosing loop frames. A new round gets a distinct execution +identity; a retry retains that identity and advances its attempt. A new batch +does not rewrite prior executions. + +A For each nested inside Repeat freezes membership per exact inner-loop +execution. Resume reuses that saved selection. A genuinely new outer round +creates a new inner execution and may perform the authored selection afresh, +without changing any earlier frozen collection. + +One Repeat-entry admission and one admission per round are charged alongside +existing body tasks, retries, and nested control work. Replaying a committed +transition or writing result pages does not charge the round again. + +Transitions and continuation decisions use the existing conditional journal +transaction, lease, cancellation, and tombstone fences. A prepared but +uncommitted state is not eligible output. Recovered workers reuse committed +body units and transitions; uncertain external effects still require the +existing recovery gate. + +## Partial data, authorization, and lifecycle + +Partial input is rejected by default. Acceptance requires eligible producer +output, explicit state-slot acceptance, and the relevant body or downstream +binding acceptance. Retain coverage and limitations through subsequent state +and final exports; meeting Until does not relabel partial work as complete. + +Failed, invalid, pending, missing-required, or unauthorized results cannot be +made eligible by a partial flag or a manual grant. Current personal/group, +initiator, source-revision, and contributor access is rechecked at the existing +read, continuation, model/tool, and external-effect boundaries. + +The exact path must prove every frozen For each membership and admitted Repeat +round with its before-state receipt. A syntactically valid frame, current head, +or cached digest is not historical authorization. State and selected-producer +lineage use bounded, cycle-detecting traversal rather than starting an +unbounded recursive walk for each preceding round. + +Cancellation or deletion fences later state, transition, result, continuation, +and publication writes. It does not undo an email, upload, or other completed +external action. Native Analyze retains both workflow and work-unit fences. + +Cancellation targets the run's frozen Repeat identities even if the live +workflow definition has since changed. Editing a definition cannot redirect +cancellation to a newer body or strand an older body's approval pause. + +## Manual continuation + +At an unmet batch limit, the run uses `paused` with a pause gate whose +`reason_code` is `repeat_iteration_limit`. Its choices are +`continue_repeat` and `cancel`; ordinary Resume is not another batch grant. + +An authorized decision uses the existing `runtime/decision` endpoint with the +current `expected_version`, `gate_id`, `choice`, and stable `request_id`. The +client cannot submit a new maximum, state value, source, or execution identity. +The gate binds the exact exhausted transition and saved next state. + +The read-only gate retains `id`, `unit_id`, `input_digest`, `execution_id`, +`node_id`, `iteration_path`, `attempt`, `definition_revision`, `reason`, and +the safe `repeat` summary alongside its kind/reason code/choices. The path is +the Repeat boundary's enclosing path, and the boundary attempt is 1. These are +server-owned selectors, not additional client decision fields. The V2 request +uses a UUID for `request_id`. + +**Continue Repeat for up to another N rounds** grants the same frozen batch +size. It resets only current-batch usage and advances the batch number/start; +lifetime numbering, saved state, cumulative admissions, and elapsed deadline +remain intact. A stale gate cannot grant another batch, and repeated +acknowledgment of the same request does not increment counters again. + +Continuation independently rechecks authority, saved contributors, lifecycle, +remaining admissions, and deadline. It does not approve body tasks, authorize +a publication destination, waive invalid output, or clear a different pause. +Global-budget blocker gates offer cancellation only, not another batch. +Schedulers, polling, retries, tool/model text, and readiness checks cannot +issue a manual grant. + +## V2 usage and inspection + +1. In a personal or group V2 List workflow, enable structured control flow and + add **Repeat until** after the producers that initialize state. +2. Explicitly choose **Maximum rounds before manual continuation**, considering + the administrator ceiling and separate global budgets. +3. Add named typed state, select its saved initial output, and declare body + outputs for every next-state slot. Body tasks use **Current Repeat state** + rather than a latest-task lookup. +4. Configure **Stop after a round when** using typed next-state fields. Select + final exports explicitly, and bind later consumers to those exports. +5. Inspect the exact round, batch, attempts, state before/after, condition + outcome, and remaining budgets. At exhaustion, review retained state before + explicitly confirming another same-sized batch. + +Unsupported definitions and unadvertised capabilities remain intact and +read-only. Invalid moves/removals retain bindings and explain the error rather +than silently selecting another producer. Classic does not gain a reduced +Repeat editor. + +Repeat authoring requires advertised `repeat_until` and `repeat_state` support, +a valid current administrator ceiling, and `hard_repeat_iterations: 1000`. +Missing capability metadata does not enable authoring; malformed advertised +limits produce an explicit editor-options error. + +A saved maximum within 1-1,000 that exceeds a lowered administrator ceiling +remains visible and editable. The editor reports a new-run policy conflict +without clamping the saved value; choose an allowed maximum before saving. +This is not an unsupported definition and does not change an active run's +frozen batch size. + +When a Repeat body task inherits the workflow runner, that runner's picker +also enforces local-loop eligibility. Tasks outside the body retain their +own ordinary runner choices. + +Inspection pages are bounded and source-authorized. An uncommitted after-state +is unavailable, not an eligible empty result. Content uses the existing exact +result, record, and provenance readers rather than exposing private locators. +Open a round's body execution to inspect its exact attempts. If that execution +is another loop, **Inspect nested Repeat rounds** or **Inspect nested For each +items** opens that specific loop instance, not all executions of its authored +node. +See [Trigger a workflow](../../guides/trigger-a-workflow.md) for operator steps. + +### Read and decision APIs + +Personal inspection uses these scoped resources: + +```text +GET /api/user/workflows//runs//executions//iterations +GET /api/user/workflows//runs//executions//iterations//state +POST /api/user/workflows//runs//runtime/decision +``` + +Group paths replace `user` with `group` and retain the existing explicit +`group_id` query parameter and authorization policy. State inspection selects +`phase=before` or `phase=after`, defaulting to `before`. Pages use opaque +cursors and `limit`, default 50 and supported range 1-100. A separate +response-size bound of about 240 KiB prevents metadata pages from growing +with the full state content. + +| Response or entry | Fields | +| --- | --- | +| Iterations page | `iterations`, `total_count`, `next_cursor`, `repeat_execution_id`, `repeat`, `source_snapshot_changed` | +| Iteration entry | `iteration`, `iteration_path`, `batch_number`, `batch_size`, `batch_usage`, `state`, `condition_result`, `execution_ids`, `before_available`, `after_available`, `partial` | +| State page | `states`, `iteration`, `phase`, `available`, `total_count`, `next_cursor`, `repeat_execution_id`; optional `partial` and `source_snapshot_changed` | +| State entry | `name`, `kind`, `workflow_validation`, `coverage`, `limitations`, optional `prior_coverage`, and `source` | +| State source selector | `node_id`, `execution_id`, optional `task_id`, `iteration_path`, `attempt`, `output_name` | + +Round `state` is `running`, `completed`, `completed_partial`, or `cancelled`. +`condition_result` is a Boolean or null. A completed round whose condition is +false is still completed; the Repeat head/gate, not that round status alone, +identifies batch exhaustion. + +A cursor binds the owning scope/run, exact Repeat execution, immutable +snapshot, page boundary, and iteration/phase where applicable. Every request +rechecks access; neither a cursor nor a result digest is an access grant. +Content selectors are resolved by the server through the existing exact +result, record, and provenance readers. The decision endpoint does not accept +caller-authored state or a replacement batch maximum. + +The safe Repeat summary appears as `gate.repeat`, `runtime.repeat_progress`, +and the iteration page's `repeat`. It exposes bounded metadata, not state values: + +| Purpose | Fields | +| --- | --- | +| Exact boundary | `execution_id`, `node_id` | +| Lifetime progress | `completed_iteration`, `next_iteration`, `completed_count` | +| Current automatic batch | `batch_number` (zero-based), `batch_size`, `batch_usage` | +| Durable counters | `exhaustion_count`, `continuation_count` | +| Outcome | `state`, `partial` | + +The summary's head `state` is `running`, `waiting_manual_continue`, +`completed`, or `cancelled`. `completed_iteration` is -1 before any round has +completed; actual iteration selectors remain zero-based lifetime indexes. + +Iteration entries include `condition_result`, `before_available`, and +`after_available`. State entries carry exact saved-source selectors with +`workflow_validation`, `coverage`, and a required `limitations` string array. +Availability flags and selectors do not replace the source authorization check +or make uncommitted state eligible. + +An uncommitted after-state reports `available: false`, `states: []`, +`total_count: 0`, and `next_cursor: null`, not an eligible empty value. State +entries may also include `prior_coverage` to retain initial partial coverage +counts after a later output passes validation. This is flat, safe metadata with +primitive values, displayed separately as retained earlier coverage. The V2 +reader rejects private or nested prior metadata instead of rendering it. A +later valid result does not erase those earlier limitations or relabel the +carried state complete. + +Metadata contains no state values, private `state_ref`, or provider locator. +Read content through the existing exact result, record, and provenance readers +using the returned source selectors. + +## Preserve shared saved-record publication + +Only a satisfied Repeat boundary can supply its final eligible `records` +export, directly or through an explicit join, to the existing required +`node_output` records binding for **Saved workflow output** publication. +Repeat state/control metadata and a flattened `document_results` bundle are +not records producers. + +The M4C-2 [saved-output contract](WORKFLOW_SAVED_OUTPUT_PUBLICATION.md) remains +unchanged: `GeneratedFileExportRequest`, `GeneratedRecordExportSource`, +`GeneratedFileExportStream`, and `build_generated_file_export` use +`exact_records_v1` JSON. Every original object, order, duplicate, nested value, +Unicode value, false, zero, null, and eligible empty collection is retained. + +The representation identity binds the frozen definition, actual selected +producer execution/path/attempt, exact output references, partial policy, +profile, and format. A genuinely new producer round is not a retry of an old +source. Retrying a publisher does not select newer data merely because its +display position or publishing attempt changed. + +Actual encoded-byte digests, immutable prepare/ready units, stable addresses, +quota enforcement, and private artifact transport remain in use. Never +overwrite different bytes at the same address or expose a quota-breach prefix. +The existing publication service and sole destination ledger own submission, +approval, notifications, and recovery. + +Submitted, Approved, and Indexed-and-ready retain their separate completion +rules, immutable fulfilled observations, and readiness proof. A downloadable +JSON file is not destination completion. Repeat adds neither a publish-only +loophole nor a download-only task, and does not expand shared export formats. + +## Monitoring included now and deferred follow-up + +Committed exhaustion and manual-continuation decisions retain authoritative +counts, actor/time, gate/request correlation, batch size/number, and lifetime +progress. Replays, polls, and duplicate acknowledgments must not increase +those durable counts. + +`runtime.repeat_counts` holds `exhaustion_count` and `continuation_count`, +aggregated across all Repeat boundaries in the run. `runtime.repeat_progress` +retains the bounded safe progress summary rather than state values or private +result references. + +Structured events `workflow_repeat_batch_exhausted` and +`workflow_repeat_manually_continued` use the existing `log_event` path and +`[WORKFLOW_SCHEDULER]` tag with stable committed-decision event IDs. They +exclude state values, prompts, records, credentials, and private locators. +Telemetry delivery is not +transactionally exactly-once with Cosmos; deduplicate by event ID rather than +treating a logging failure as reversal of a committed grant. + +**Not implemented in M4C-3:** the approved future Control Center follow-up +includes personal-workflow monitoring in the personal-user context, +group-workflow monitoring in the group context, and a dedicated **Workflow +Monitoring** section. It must separately decide roles/visibility, cross-run +queries, aggregation, retention, filters, and any alerts/actions. Logs and IDs +do not grant monitoring access. Reuse the existing execution/decision records; +do not create a competing ledger. + +M5A read-only Flow comes next in the milestone sequence, followed separately +by M5B accessible visual authoring. Neither is included here, and no graph +library evaluation or installation is part of Repeat. Parallel loops, general +cycles/reducers, hosted-agent loops, and cumulative token/spending caps remain +out of scope. + +## Implementation and validation + +`functions_workflow_editor.py` advertises `repeat_until`, `repeat_state`, and +`flow_limits.max_repeat_iterations`/`hard_repeat_iterations` without returning +raw settings. `functions_workflow_limits.py` supplies the shared constants and +strict validator. `admin_settings_fields.py`, `functions_settings.py`, +`route_frontend_admin_settings.py`, and the existing Workflow pane keep +Classic/V2 numeric bounds, absent-value preservation, and persistence aligned. + +The compiler, identity, structured runner, journal, runtime decisions, and +source-authorized readers remain the implementation boundaries described in +the linked workflow features; Repeat does not replace those services. + +`functions_workflow_repeat_execution.py` supplies the serial body transitions, +`functions_workflow_repeat_state.py` binds saved state to sealed admissions, +and `functions_workflow_repeat_history.py` serves the bounded inspection +pages. The existing Flow runner and runtime decision transaction call these +helpers. The existing journal's `journal_commit_many` and `journal_decide` +provide the conditional multi-row transition and human-decision boundaries. +V2's `WorkflowRepeatFields.tsx` and `WorkflowRepeatProgress.tsx` integrate +authoring and progress into the existing List and runtime controls. + +`functional_tests/test_workflow_repeat_editor_options.py` covers defaults, +actual 1/1,000 bounds, invalid-value rejection, current scoped editor policies, +safe projection, and Classic validation. `test_workflow_loop_limits.py` +additionally exercises the production settings writer with isolated storage, +and `test_v2_admin_workflow_parity.py` checks both surfaces' fields and bounds. + +`ui_tests/test_workflow_loop_admin_limits.py` exercises Classic and V2 admin +controls with closed browser fixtures and the production schema, template, +and normalizer. It covers independent limits, help/ARIA, native validity, +mobile/keyboard use, V2 narrow updates and reload, valid 1,000 saves, retained +invalid drafts without changing saved policy, and unrelated V2 updates that +preserve an absent Repeat key. + +`test_workflow_repeat_schema.py` and `test_workflow_repeat_limits.py` cover +typed bindings, mixed ancestry, legacy execution hashes, and independent +limits. The Repeat execution, state, recovery, publication, and route-policy +suites exercise the existing engine, readers, journal, and publication +boundaries. `test_workflow_repeat_dispatcher.py` runs the production dispatcher, +including the committed-task-unit/result-checkpoint crash gap on both result +stores, with no extra execution admissions or repeated provider invocation. +It also verifies cancellation after a live definition edit against the +original admitted Repeat and round. +`ui_tests/test_v2_workflow_repeat_until.py` covers the local V2 surfaces. + +### Verified closed-fixture runtime coverage + +Runtime coverage exercises actual 1,000-round exhaustion and an explicit grant +into lifetime round 1,001, with the same absolute deadline and no replay +admissions or duplicate body work. Cold history/state reads reauthorize the +saved lineage. Other cases cover frozen policy, typed/pass-through state, +collections larger than 8 MiB, retained partials, three mixed frames, approvals, +lost acknowledgments, lease/cancellation fences, cached-proof source +revocation, cursor scope/phase binding, sanitized logs, and policy preflight. + +Native Analyze and exact final-record publication use the existing production +adapters with both Cosmos and Blob fixtures and both nesting directions. +Crashes after destination submission/ledger commitment reuse the same artifact +and destination on restart, without duplicate submissions or changed native +publication fingerprints. + +The focused runtime selections verified for this slice are reproducible from +the repository root with an isolated test interpreter and repository-pinned +dependencies, including Flask/Werkzeug: + +```powershell +python -m pytest -q .\functional_tests\test_workflow_repeat_execution.py .\functional_tests\test_workflow_repeat_recovery.py .\functional_tests\test_workflow_repeat_state.py .\functional_tests\test_workflow_repeat_publication.py .\functional_tests\route_tests\test_workflow_repeat_policy.py -k 'not real_thousand' +python -m pytest -q .\functional_tests\test_workflow_repeat_execution.py -k real_thousand +``` + +The first selection passed **64 tests**, with the threshold case deselected. +The separate threshold selection passed **1 test** in approximately **402 +seconds**. That duration is local test evidence, not a production performance +guarantee. + +**These are fictional, closed fixtures, not live Azure, Cosmos, Blob, model, +or publication-service validation.** They exercise the production runner, +journal, result-store, native Analyze, exporter, and destination-ledger +functions without live-service operations. Settings/options checks alone do +not establish runtime behavior. No live workflow, external publication, +deployment, or permission change was performed as part of this validation. diff --git a/docs/explanation/features/WORKFLOW_RESULT_READERS.md b/docs/explanation/features/WORKFLOW_RESULT_READERS.md index b2c835a77..4946bab87 100644 --- a/docs/explanation/features/WORKFLOW_RESULT_READERS.md +++ b/docs/explanation/features/WORKFLOW_RESULT_READERS.md @@ -13,6 +13,9 @@ raw-model result as an original Analyze run. Updated in version: **0.261.117** with execution-scoped complete-record handles and incremental collection indexes. +Updated in version: **0.261.120** for exact Repeat state receipts and mixed +iteration paths. + ## Purpose and dependencies This incremental foundation extends the existing `workflow-result-v1` store. @@ -108,6 +111,36 @@ iteration and aggregate consumption. Explicit saved-record reporting can process supported large inputs in batches while retaining originals. Inspection-only readers do not grant engine eligibility to invalid results. +## Repeat state and exact historical reads + +[Repeat until](WORKFLOW_REPEAT_UNTIL.md) reuses the same typed result store for +`text`, `json`, `records`, and `document_results`. `repeat_state` is a binding +source, not a new public result kind or a request to convert typed data into +a downloaded file. Each slot selects an exact saved producer receipt; earlier +state versions and original values remain retained. + +A read must prove every frame of a mixed iteration path: frozen item membership +for For each and sealed round admission with its before-state receipt for +Repeat. `{loop_id, iteration}` uses a zero-based lifetime index, which does not +reset on manual continuation. A current Repeat head or a well-formed path is +not permission to invent a historical round. + +State and producer lineage use shared bounded, cycle-detecting authorization +traversal. Reusing an exact receipt does not cache authority indefinitely: +current workflow/group, contributor, source revision, producer attempt, and +lifecycle checks still apply at their existing boundaries. + +Before/after inspection is paged. Uncommitted after-state is unavailable, not +an eligible empty value. Partial state requires explicit acceptance and +retains coverage and limitations through later state and final exports. +Invalid, failed, pending, missing-required, or unauthorized state cannot become +usable through manual continuation. + +Only a satisfied Repeat boundary exposes its declared final outputs through +ordinary `node_output` readers. This preserves the selected body producer, +attempt, and representation for later tasks and exact saved-record publication +without a latest-task lookup or a fabricated native Analyze producer. + ## Validation and integration boundary The foundation's functional coverage includes named sections and receipts, diff --git a/docs/explanation/features/WORKFLOW_SAVED_OUTPUT_PUBLICATION.md b/docs/explanation/features/WORKFLOW_SAVED_OUTPUT_PUBLICATION.md index 72b8ff2e7..76731e6e2 100644 --- a/docs/explanation/features/WORKFLOW_SAVED_OUTPUT_PUBLICATION.md +++ b/docs/explanation/features/WORKFLOW_SAVED_OUTPUT_PUBLICATION.md @@ -2,8 +2,9 @@ Implemented in version: **0.261.119**. -Related application version update: `VERSION = "0.261.119"` in -`application\single_app\config.py`. +Updated in version: **0.261.120** for Repeat final records. + +Application version tracking: `application\single_app\config.py`. ## Overview and purpose @@ -18,6 +19,11 @@ This M4C-2 slice uses the existing [Generated File Export Framework](GENERATED_FILE_EXPORT_FRAMEWORK.md). There is no workflow-only renderer or second publication service. +In **0.261.120**, a satisfied [Repeat until](WORKFLOW_REPEAT_UNTIL.md) boundary +can also provide a named final records export. It remains a real engine-node +output with its exact selected-producer receipt, not an invented Analyze task. +An unmet batch limit exposes no final Repeat output for publication. + | Representation | Purpose | | --- | --- | | Saved task or engine output | Durable data consumed through explicit typed bindings; no workspace upload or indexing is needed for the next task. | @@ -91,8 +97,9 @@ records output in the frozen definition. A task's authoritative records output is also eligible. An explicit join retains its selected-producer receipt and branch lineage; it is not resolved by looking for a recent task ID. -The adapter rejects loop-item inputs, optional missing inputs, diagnostics, -preview rows, text, scalar or untyped JSON, and `document_results` bundles. +The adapter rejects direct loop-item or Repeat-state inputs, optional missing +inputs, diagnostics, preview rows, text, scalar or untyped JSON, and +`document_results` bundles. Nested objects and arrays **inside supported record objects** remain supported. Invalid, failed, pending or unreadable sources cannot become valid files by changing the format. @@ -103,6 +110,11 @@ coverage remain visibly partial; missing work is not relabeled complete. Duplicates and record order are preserved. A declared uniqueness-contract violation remains invalid: export never deduplicates it into a passing result. +For Repeat exports, the state-slot and body/downstream partial policies must +also have accepted the carried data. A later true Until condition never removes +its coverage limitations. Body-state receipts are not a shortcut around the +required final `node_output` records binding. + `source_kind` describes the source, not the destination. Personal, group and public destination fields keep their existing meaning. The selected workspace is explicit and does not follow the user's active workspace. @@ -190,6 +202,12 @@ generic identities additionally bind the validated saved-output source. Different publication nodes or destinations may reuse a source artifact while retaining their own destination receipts. +A genuinely new Repeat round changes the source identity when a new producer +actually runs. Retrying a publication of already materialized output keeps +the same exact source, immutable bytes, and destination receipt; changes to +the publisher's attempt or display order do not select a newer round. +Manual continuation grants no publication permission or completion bypass. + ## Authorization and existing APIs Source receipts, digests and artifact locators are not permissions. Current @@ -335,6 +353,7 @@ do not publish documents to live workspaces or establish deployment acceptance. defaults to **500 actual selected items**; administrators can set **1-5,000** for new runs only. A searchable corpus is not itself a loop selection. - The owner-deferred cumulative run-token/spend cap is not implemented here. -- Repeat until, M5 read-only Flow and accessible visual authoring remain - separate future slices. This feature adds no automation of those steps, new - scheduler, promotion service or destination ledger. +- Repeat until is added separately in **0.261.120** and reuses this exact source + and publication contract. M5A read-only Flow and M5B accessible visual + authoring remain separate future slices. This feature adds no automation of + those steps, new scheduler, promotion service, or destination ledger. diff --git a/docs/explanation/features/WORKFLOW_STRUCTURED_CONTROL_FLOW.md b/docs/explanation/features/WORKFLOW_STRUCTURED_CONTROL_FLOW.md index 1afc81a25..c1c3d52e0 100644 --- a/docs/explanation/features/WORKFLOW_STRUCTURED_CONTROL_FLOW.md +++ b/docs/explanation/features/WORKFLOW_STRUCTURED_CONTROL_FLOW.md @@ -2,7 +2,7 @@ Implemented in version: **0.261.116** -Updated in version: **0.261.119**. +Updated in version: **0.261.120**. Application version tracking: `application/single_app/config.py`. @@ -21,8 +21,10 @@ This page describes the M4A foundation. Version **0.261.117** adds [serial For each and exact Collect](WORKFLOW_FOR_EACH_COLLECT.md) to the same definition version and journal. Version **0.261.119** adds [saved-record JSON publication](WORKFLOW_SAVED_OUTPUT_PUBLICATION.md). -Repeat until, M5 read-only Flow and accessible visual authoring remain separate -future slices. M4A itself did not admit loops. +Version **0.261.120** adds [Repeat until](WORKFLOW_REPEAT_UNTIL.md), with saved +typed state and explicit manual grants after finite automatic batches. M5A +read-only Flow and M5B accessible visual authoring remain separate subsequent +milestones. M4A itself did not admit loops. ## Dependencies and compatibility @@ -103,6 +105,7 @@ indexes are not a second stored graph. | Join | Stable `id` and named `exports`, each selecting a Then and Else producer/output | | Forward route | `id`, `kind: "route"`, `inputs`, `condition`, `target` | | Route target | A later sibling `node_id`, or the current branch's `exit_region_id` | +| Repeat until node (0.261.120) | `id`, `kind: "repeat_until"`, required `max_iterations`, typed `state`, `body`, post-body `until`, and explicit final `exports` | A version-3 binding is explicit: @@ -126,6 +129,13 @@ input mode. Version-1 and version-2 omitted/null predecessor behavior remains unchanged. Definitions cannot supply result-store references or choose another run's execution. +Inside Repeat, an explicit `repeat_state` source names an enclosing `loop_id`, +`state_name`, and `scope: "current"`. It selects the saved state admitted for +that exact round, not another iteration's latest task output. Until reads +validated next-state slots, and downstream consumers use the Repeat node's +named final exports only after the condition succeeds. See +[Repeat's state contract](WORKFLOW_REPEAT_UNTIL.md#definition-contract). + The server validates versions, executable fields, node kinds, unique IDs, region depth, routing, and producer availability. Advanced saves retain the existing definition-revision and active-run protections. @@ -164,6 +174,12 @@ workflow, run, logical node, server-derived `execution_id`, `iteration_path`, and attempt. Real task results also retain `task_id`; engine results do not invent task IDs. M4A iteration paths are empty. +For each adds unchanged `{loop_id, item_id, index}` frames. Repeat adds the +distinct `{loop_id, iteration}` shape in **0.261.120**, with a zero-based +lifetime round index that survives manual continuation. Mixed paths must prove +their ordered ancestry and exact frozen/admitted membership. Engine-boundary +results retain their actual selected-producer receipts. + Retries retain the execution ID and advance the attempt. Approval and recovery decisions bind the exact execution, attempt, gate, definition, and input digest. Receipts retain the actual producer and representation. Branch-control @@ -177,7 +193,7 @@ history items, and deletion preserves lifecycle tombstones. ## Limits and inspection -| Limit | M4A policy | +| Limit | Structured policy | | --- | --- | | Authored tasks | Existing administrator setting: default 50, supported range 1-100 | | Structural IDs | At most 256 | @@ -185,8 +201,14 @@ history items, and deletion preserves lifecycle tombstones. | Predicate size | At most 100 nodes, depth 8, and 16 KiB | | Execution admissions | Default/maximum 5,000; includes retries, not result-chunk writes | | Elapsed deadline | Default/maximum 86,400 seconds, including waits | +| Repeat automatic batch (0.261.120) | Required authored maximum; administrator default 25, range 1-1,000; new runs above policy are rejected, not clamped | | Model input | Existing effective catalog/deployment budget; complete required input is never clipped | +At an unmet Repeat batch maximum, an authorized manual decision can grant the +same frozen batch again. Only batch usage resets; lifetime round identity, +execution admissions, and elapsed deadline do not. Ordinary Resume cannot make +this grant or clear the global limits. + Run history distinguishes selected paths, intentionally skipped nodes, output validation, attempts, and exact consumed-result receipts. Large execution and decision histories are paginated. Full result content is requested separately; @@ -223,6 +245,12 @@ the downloadable file and submits it to the chosen destination using the same completion policies. It is not a download-only task or a native Analyze artifact. Omitting the source choice preserves existing native publication. +In **0.261.120**, a satisfied Repeat boundary can expose a final records export +to the same required records binding. The selected body producer remains real, +with its exact path/attempt and retained partial coverage. An exhausted batch +has no eligible final export, and Repeat does not add a different renderer, +file format, or destination ledger. + Saved-record serialization rechecks current scope and the exact source attempt every 100 records. Reusing a materialized file still verifies its exact ready checkpoint. Generic destination approval rechecks current source and destination diff --git a/docs/guides/create-a-workflow.md b/docs/guides/create-a-workflow.md index 1724c375f..cfe5861c6 100644 --- a/docs/guides/create-a-workflow.md +++ b/docs/guides/create-a-workflow.md @@ -73,7 +73,8 @@ Declare promised deliverables under **Final outputs**. This prevents a run from reporting completion when a selected path did not produce the required result. Structured definitions require durable execution and preserve their choices across waits and restarts. For each and Collect are added in **0.261.117** below. -Repeat until and the visual Flow editor remain separate. +Repeat until is added in **0.261.120** below. M5A read-only Flow and M5B visual +authoring remain separate later milestones. See [Structured workflow control flow](../explanation/features/WORKFLOW_STRUCTURED_CONTROL_FLOW.md) for condition semantics, execution identity, limits, and compatibility. @@ -105,6 +106,46 @@ their full input cannot safely fit; they do not silently become summary tasks. See [Serial For each and exact Collect](../explanation/features/WORKFLOW_FOR_EACH_COLLECT.md) for local-runner requirements, partial coverage, nested scopes, and limitations. +## Refine saved state with Repeat until + +In **0.261.120**, use **Repeat until** when each round should work on saved +state from the preceding round, such as a report draft and a structured review +decision. It is a serial, post-body loop: the body always runs at least once. +Select an eligible model or local agent; hosted non-loop workflows are +unchanged, but hosted loop execution is unavailable. + +1. Produce the initial data in earlier tasks, then add **Repeat until**. + Declare named state with explicit `text`, `json`, `records`, or + `document_results` contracts. Select each initial saved output; entering + starting literals is not supported. +2. Choose **Maximum rounds before manual continuation** explicitly. The field + starts unset. The administrator ceiling defaults to 25 and can be 1-1,000; + this is separate from For each's 500-item default and the global run budgets. +3. Bind body tasks to **Current Repeat state**, declare their body outputs, and + select a next body output for every state slot. To keep data unchanged, + explicitly pass through its current-state receipt rather than asking a + model to echo it. +4. Under **Stop after a round when**, select typed next-state fields. Use a + schema-validated Boolean such as `review.ready`, not a sentence saying the + work is complete. Configure explicit final exports for later tasks. +5. Save and reopen the workflow to review those exact bindings. An invalid + removal or move keeps the binding and reports the problem rather than + silently choosing another producer. + +The final exports become available only when the condition is true, including +when it first becomes true on the last allowed round. Otherwise the workflow +pauses at the batch limit with prior outputs and next state retained. See +[manual continuation](trigger-a-workflow.md#continue-a-repeat-batch) before +granting another batch. + +Partial state is rejected unless the producer, state slot, and relevant +consumers explicitly accept it. Accepted coverage limitations remain visible +through later rounds and final results. Approval cannot repair failed, +invalid, pending, missing, or unauthorized data. + +See [Repeat until with saved typed state](../explanation/features/WORKFLOW_REPEAT_UNTIL.md) +for state contracts, frozen policy, mixed nesting, and exact result identity. + ## Durable execution and task approval Starting in **0.261.111**, new V2 workflows enable **Durable execution**. @@ -193,6 +234,12 @@ task, **Collect**, or an explicit **Join outputs** selection. Use this when you need the complete collected dataset as a file, rather than an explanation of the dataset or a copy of one native Analyze artifact. +In **0.261.120**, a satisfied **Repeat until** boundary can also supply a named +eligible records export, directly or through a join. An exhausted batch does +not expose a final export; current-state metadata is not a publication source. +The same exact JSON renderer, immutable file identity, and destination receipt +remain in use. + 1. Produce an eligible records output. For example, Analyze each document in a frozen For each selection, then Collect the records outside the loop. 2. In a later task, enable **Publish a workflow file** and explicitly select diff --git a/docs/guides/trigger-a-workflow.md b/docs/guides/trigger-a-workflow.md index a2933db77..5c79ed2fc 100644 --- a/docs/guides/trigger-a-workflow.md +++ b/docs/guides/trigger-a-workflow.md @@ -109,6 +109,37 @@ task retains its data rather than receiving a truncated substitute. See [Serial For each and exact Collect](../explanation/features/WORKFLOW_FOR_EACH_COLLECT.md). +## Continue a Repeat batch + +In **0.261.120**, **Repeat until** saves the state admitted for each round and +its validated next state. Inspect lifetime round, current automatic batch, +batch usage/limit, condition outcome, exact producer attempts, partial coverage, +and remaining global budgets. Paged inspection avoids loading every round or +record at once; an uncommitted after-state is unavailable, not an empty result. + +If the condition is still false at the authored maximum, the workflow pauses. +Review the retained state before choosing **Continue Repeat for up to another +N rounds** and confirming the grant. Only users with the current workflow +decision permission may do this. A stale or already-used gate must refresh +rather than create another grant. + +This grants the same frozen batch size, even if an administrator has since +changed the Repeat setting. Only batch usage resets; lifetime round numbers, +cumulative execution admissions, and the original elapsed deadline do not. +Waiting for a person consumes elapsed time. The shared limits remain at most +5,000 admissions and 86,400 seconds, and can block further continuation. + +Ordinary Resume, polling, scheduled triggers, retries, and a model's response +cannot grant another batch. The grant does not approve body tasks or workspace +publication, and cannot make failed, invalid, pending, or unauthorized state +usable. If a separate budget or access gate is the blocker, address that exact +gate rather than treating it as a Repeat-limit pause. + +A true condition completes Repeat, including on the last allowed round. +Only then are its declared final exports eligible for downstream tasks. +Earlier saved state and accepted partial limitations remain retained. +See [Repeat until](../explanation/features/WORKFLOW_REPEAT_UNTIL.md). + ## Inspect a saved-output publication In **0.261.119**, a task explicitly configured with **Saved workflow output** @@ -116,6 +147,11 @@ renders its selected saved records as JSON and submits that file to its chosen destination. Run inspection identifies the exact producer, output and attempt; a repeated task name or latest chat reply is not the source identity. +Repeat final records in **0.261.120** retain that same source-bound identity. +A new round with a genuinely new producer is distinct from retrying a +publication of one already saved output. Continuation never redirects an +existing immutable file to a newer source. + Use the existing generated-file card to download the full JSON, not a preview of the first records. Record order, duplicates, nested values and retained provenance are preserved. Accepted partial output remains visibly partial; diff --git a/functional_tests/route_tests/test_workflow_execution_journal_policy.py b/functional_tests/route_tests/test_workflow_execution_journal_policy.py index 928e6649b..85490d6d3 100644 --- a/functional_tests/route_tests/test_workflow_execution_journal_policy.py +++ b/functional_tests/route_tests/test_workflow_execution_journal_policy.py @@ -1,7 +1,7 @@ # test_workflow_execution_journal_policy.py """ Structured workflow execution API policy and exact result regression coverage. -Version: 0.261.117 +Version: 0.261.120 Implemented in: 0.261.116 Production route helpers and journal readers execute with isolated Flask request @@ -53,7 +53,7 @@ def test_all_execution_routes_retain_blueprint_and_swagger_security(): assert any("enabled_required" in value for value in decorators) if "/user/" in path: assert "workflow_user_required" in decorators - assert len(routes) == 14 + assert len(routes) == 18 @pytest.fixture diff --git a/functional_tests/route_tests/test_workflow_repeat_policy.py b/functional_tests/route_tests/test_workflow_repeat_policy.py new file mode 100644 index 000000000..5bf661c1a --- /dev/null +++ b/functional_tests/route_tests/test_workflow_repeat_policy.py @@ -0,0 +1,127 @@ +# test_workflow_repeat_policy.py +""" +Functional policy tests for authorized Repeat iteration and state inspection. +Version: 0.261.120 +Implemented in: 0.261.120 + +Real route helpers and readers use closed Flask and transactional store fixtures. +No live application, permissions, credentials or services are used. +""" + +import ast +import logging +import sys +from pathlib import Path + +import pytest +from azure.core.exceptions import AzureError +from azure.cosmos.exceptions import CosmosResourceNotFoundError +from flask import Flask, jsonify, request + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "application" / "single_app")) +sys.path.insert(0, str(ROOT / "functional_tests")) + +# Shared test helpers follow the isolated worktree import setup. +from functions_analysis_access import AnalysisResultUnavailable +from functions_workflow_identity import workflow_execution_id +from functions_workflow_node_results import WorkflowRecordPageTooLarge +from functions_workflow_repeat_history import workflow_repeat_iterations_page, workflow_repeat_state_page +from functions_workflow_result_store import WorkflowResultStorageUnavailableError +from functions_workflow_runtime_store import RuntimeUnavailable, WorkflowRuntimeConflict +from test_workflow_repeat_execution import execute_repeat, repeat_runtime + + +ROUTES = ROOT / "application" / "single_app" / "route_backend_workflows.py" + + +def test_repeat_routes_keep_exact_existing_blueprint_swagger_and_scope_policies(): + tree = ast.parse(ROUTES.read_text(encoding="utf-8")) + registrar = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "register_route_backend_workflows") + paths = [] + for function in registrar.body: + if not isinstance(function, ast.FunctionDef) or not function.decorator_list: + continue + route = function.decorator_list[0] + path = route.args[0].value + if not (path.endswith("/iterations") or path.endswith("/iterations//state")): + continue + decorators = [ast.unparse(value) for value in function.decorator_list] + assert ast.unparse(route.func) == "bp.route" + assert decorators[1] == "swagger_route(security=get_auth_security())" + assert "login_required" in decorators and "user_required" in decorators + if "/user/" in path: + assert "workflow_user_required" in decorators + assert "enabled_required('allow_user_workflows')" in decorators + else: + assert "enabled_required('enable_group_workspaces')" in decorators + assert "enabled_required('allow_group_workflows')" in decorators + paths.append(path) + assert len(paths) == 4 + + +@pytest.fixture +def repeat_api(monkeypatch): + workflow, store, _, _, _ = repeat_runtime(monkeypatch) + execute_repeat(workflow, store, target=3) + access = {"owner": True, "group": True} + + def group_scope(user): + if not access["group"]: + raise PermissionError + assert request.args.get("group_id") == "fictional-group" + return "fictional-group", {} + + namespace = { + "get_current_user_id": lambda: "owner", "jsonify": jsonify, "request": request, "logging": logging, + "get_personal_workflow": lambda user, key: workflow if access["owner"] and key == workflow["id"] else None, + "get_group_workflow": lambda group, key: workflow if key == workflow["id"] else None, + "get_personal_workflow_run": lambda user, run: {"id": run, "workflow_id": workflow["id"]} if run == "run" else None, + "get_group_workflow_run": lambda group, run: {"id": run, "workflow_id": workflow["id"]} if run == "run" else None, + "_resolve_group_workflow_request_group": group_scope, + "workflow_repeat_iterations_page": workflow_repeat_iterations_page, + "workflow_repeat_state_page": workflow_repeat_state_page, + "WorkflowRuntimeConflict": WorkflowRuntimeConflict, "RuntimeUnavailable": RuntimeUnavailable, + "WorkflowResultStorageUnavailableError": WorkflowResultStorageUnavailableError, + "WorkflowRecordPageTooLarge": WorkflowRecordPageTooLarge, + "AnalysisResultUnavailable": AnalysisResultUnavailable, + "AzureError": AzureError, "CosmosResourceNotFoundError": CosmosResourceNotFoundError, + "log_event": lambda *args, **kwargs: None, + } + function = next(node for node in ast.parse(ROUTES.read_text(encoding="utf-8")).body + if isinstance(node, ast.FunctionDef) and node.name == "_workflow_execution_history_response") + exec(compile(ast.Module(body=[function], type_ignores=[]), str(ROUTES), "exec"), namespace) + app = Flask("workflow-repeat") + + def inspect(scope, run_id, execution_id, kind, iteration=None): + return namespace["_workflow_execution_history_response"]( + "workflow", run_id, group=scope == "group", execution_id=execution_id, kind=kind, iteration=iteration, + ) + + app.add_url_rule("////iterations", endpoint="iterations", + view_func=lambda scope, run_id, execution_id: inspect(scope, run_id, execution_id, "iterations")) + app.add_url_rule("////iterations//state", endpoint="states", + view_func=lambda scope, run_id, execution_id, iteration: inspect(scope, run_id, execution_id, "states", iteration)) + return app.test_client(), workflow_execution_id(workflow, "run", "repeat"), access + + +def test_iteration_and_state_route_shapes_and_exact_scope_binding(repeat_api): + client, execution_id, access = repeat_api + base = f"/user/run/{execution_id}/iterations" + page = client.get(base, query_string={"limit": 2}) + assert page.status_code == 200 and len(page.json["iterations"]) == 2 + assert page.json["total_count"] == 3 and page.json["next_cursor"] + state = client.get(f"{base}/1/state?phase=after") + assert state.status_code == 200 and state.json["available"] + assert state.json["states"][0]["source"]["iteration_path"][-1]["iteration"] == 1 + assert client.get(f"{base}/1/state?phase=current").status_code == 400 + assert client.get(f"{base}/1/state", query_string={"cursor": page.json["next_cursor"]}).status_code == 400 + assert client.get(base, query_string={"limit": 101}).status_code == 400 + assert client.get(f"{base}/1001/state").status_code == 404 + assert client.get(f"/user/foreign-run/{execution_id}/iterations").status_code == 404 + assert client.get(f"/user/run/{'f' * 64}/iterations").status_code == 404 + assert client.get(f"/group/run/{execution_id}/iterations?group_id=fictional-group").status_code == 200 + access["owner"] = False + assert client.get(base).status_code == 404 + access["group"] = False + assert client.get(f"/group/run/{execution_id}/iterations?group_id=fictional-group").status_code == 403 diff --git a/functional_tests/test_v2_admin_workflow_parity.py b/functional_tests/test_v2_admin_workflow_parity.py index 23fcdf46e..51cc204a4 100644 --- a/functional_tests/test_v2_admin_workflow_parity.py +++ b/functional_tests/test_v2_admin_workflow_parity.py @@ -2,7 +2,7 @@ # test_v2_admin_workflow_parity.py """ Functional test pinning V1/V2 parity for the Admin Settings Workflow group. -Version: 0.261.059 +Version: 0.261.120 Implemented in: 0.261.059 The Workflow group rendered completely empty in the V2 React admin surface. The @@ -19,7 +19,7 @@ - every form field the V1 pane submits is claimed by the schema; - the schema invents no workflow field that V1 does not have; - the section is not empty, which is the specific regression; - - the two numeric limits share identical bounds with the V1 inputs, since a V2 + - the numeric limits share identical bounds with the V1 inputs, since a V2 control offering a wider range would save a value V1 refuses to show; and - the gating chain matches the capability each sub-setting belongs to. """ @@ -49,7 +49,7 @@ NUMBER_BLOCK_RE = re.compile(r']*type="number"(?P[^>]*)>', re.DOTALL) ATTR_RE = re.compile(r'(\w[\w-]*)="([^"]*)"') -# Which capability each sub-setting belongs to. The two run limits are absent on +# Which capability each sub-setting belongs to. The run limits are absent on # purpose: they bound personal *and* group runs, and `depends_on` names a single # key, so gating either one on a single capability would hide a live limit from # an administrator who only uses the other. @@ -59,7 +59,12 @@ "group_workflow_allowed_group_ids": "require_group_assignment_for_group_workflows", } -UNGATED_KEYS = ("workflow_max_auto_invoke_attempts", "workflow_max_tasks") +UNGATED_KEYS = ( + "workflow_max_auto_invoke_attempts", + "workflow_max_tasks", + "workflow_max_loop_items", + "workflow_max_repeat_iterations", +) fields_module = import_app_module("admin_settings_fields") diff --git a/functional_tests/test_workflow_loop_limits.py b/functional_tests/test_workflow_loop_limits.py index 47eb920e1..298a7b580 100644 --- a/functional_tests/test_workflow_loop_limits.py +++ b/functional_tests/test_workflow_loop_limits.py @@ -1,7 +1,7 @@ # test_workflow_loop_limits.py """ Functional tests for strict workflow loop policy and non-secret editor options. -Version: 0.261.117 +Version: 0.261.120 Implemented in: 0.261.117 Exercises production settings normalization, the Classic POST validation block, @@ -34,6 +34,7 @@ get_workflow_loop_item_limit, get_workflow_max_loop_items, validate_workflow_max_loop_items, + validate_workflow_max_repeat_iterations, ) from test_support.app_stubs import import_app_module @@ -45,6 +46,32 @@ def _production_function(filename, name, namespace): return namespace[name] +def _classic_limit_validator(key, validator, getter): + source = ast.parse((APP_ROOT / "route_frontend_admin_settings.py").read_text(encoding="utf-8")) + validation = next( + node for node in ast.walk(source) + if isinstance(node, ast.Try) + and any( + isinstance(child, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == key for target in child.targets) + for child in node.body + ) + ) + wrapper = ast.parse("def validate_form(form_data, settings):\n pass\n").body[0] + wrapper.body = [validation, ast.Return(value=ast.Name(id=key, ctx=ast.Load()))] + flashes = [] + namespace = { + "WorkflowLoopLimitError": WorkflowLoopLimitError, + validator.__name__: validator, + getter.__name__: getter, + "flash": lambda message, category: flashes.append((message, category)), + "redirect": lambda path: ("redirect", path), + "url_for": lambda endpoint: endpoint, + } + exec(compile(ast.fix_missing_locations(ast.Module(body=[wrapper], type_ignores=[])), "classic_post", "exec"), namespace) + return namespace["validate_form"], flashes + + class WorkflowLoopPolicyTests(unittest.TestCase): def test_default_and_supported_boundaries(self): self.assertEqual(get_workflow_max_loop_items({}), 500) @@ -117,30 +144,9 @@ def test_v2_registry_rejects_invalid_and_preserves_absent_field(self): self.assertEqual(normalized["workflow_max_loop_items"], 1700) def test_classic_post_validation_and_absent_preservation(self): - source = ast.parse((APP_ROOT / "route_frontend_admin_settings.py").read_text(encoding="utf-8")) - validation = next( - node for node in ast.walk(source) - if isinstance(node, ast.Try) - and any( - isinstance(child, ast.Assign) - and any(isinstance(target, ast.Name) and target.id == "workflow_max_loop_items" - for target in child.targets) - for child in node.body - ) + validate, flashes = _classic_limit_validator( + "workflow_max_loop_items", validate_workflow_max_loop_items, get_workflow_max_loop_items, ) - wrapper = ast.parse("def validate_form(form_data, settings):\n pass\n").body[0] - wrapper.body = [validation, ast.Return(value=ast.Name(id="workflow_max_loop_items", ctx=ast.Load()))] - flashes = [] - namespace = { - "WorkflowLoopLimitError": WorkflowLoopLimitError, - "validate_workflow_max_loop_items": validate_workflow_max_loop_items, - "get_workflow_max_loop_items": get_workflow_max_loop_items, - "flash": lambda message, category: flashes.append((message, category)), - "redirect": lambda path: ("redirect", path), - "url_for": lambda endpoint: endpoint, - } - exec(compile(ast.fix_missing_locations(ast.Module(body=[wrapper], type_ignores=[])), "classic_post", "exec"), namespace) - validate = namespace["validate_form"] self.assertEqual(validate({}, {"workflow_max_loop_items": 2100}), 2100) self.assertEqual(validate({}, {}), 500) self.assertEqual(validate({"workflow_max_loop_items": "2600"}, {}), 2600) @@ -151,10 +157,15 @@ def test_classic_post_validation_and_absent_preservation(self): def test_settings_writer_validates_before_storage_and_preserves_absent_value(self): class Storage: def __init__(self): - self.current = {"id": "app_settings", "_etag": "etag-1", "workflow_max_loop_items": 1800} + self.current = { + "id": "app_settings", "_etag": "etag-1", + "workflow_max_loop_items": 1800, "workflow_max_repeat_iterations": 200, + } + self.reads = 0 self.writes = [] def read_item(self, **_kwargs): + self.reads += 1 return copy.deepcopy(self.current) def replace_item(self, *, body, **_kwargs): @@ -169,6 +180,7 @@ class ClosedError(Exception): namespace = { "copy": copy, "logging": logging, "validate_workflow_max_loop_items": validate_workflow_max_loop_items, + "validate_workflow_max_repeat_iterations": validate_workflow_max_repeat_iterations, "cosmos_settings_container": storage, "validate_content_screening_settings": lambda *_args: None, "coerce_multi_model_endpoint_enablement": lambda _old, requested: requested, @@ -195,14 +207,26 @@ class ClosedError(Exception): writer = _production_function("functions_settings.py", "update_settings", namespace) with self.assertRaises(WorkflowLoopLimitError): writer({"workflow_max_loop_items": False}) + for value in (None, "", 0, 1001, True, 25.0, "invalid-secret"): + with self.subTest(value=value), self.assertRaises(WorkflowLoopLimitError): + writer({"workflow_max_repeat_iterations": value, "workflow_max_loop_items": 2500}) + self.assertEqual(storage.reads, 0) self.assertFalse(storage.writes) embedding = ModuleType("functions_embedding_compatibility") embedding.embedding_settings_write_guard = lambda *_args, **_kwargs: nullcontext() with patch.dict(sys.modules, {"functions_embedding_compatibility": embedding}): self.assertTrue(writer({"allow_user_workflows": True})) self.assertEqual(storage.current["workflow_max_loop_items"], 1800) + self.assertEqual(storage.current["workflow_max_repeat_iterations"], 200) self.assertTrue(writer({"workflow_max_loop_items": "2500"})) self.assertEqual(storage.current["workflow_max_loop_items"], 2500) + self.assertEqual(storage.current["workflow_max_repeat_iterations"], 200) + for value in ("1", "750", "1000"): + update = {"workflow_max_repeat_iterations": value} + self.assertTrue(writer(update)) + self.assertEqual(storage.current["workflow_max_repeat_iterations"], int(value)) + self.assertEqual(storage.current["workflow_max_loop_items"], 2500) + self.assertEqual(update, {"workflow_max_repeat_iterations": value}) def test_editor_capabilities_and_eligibility_are_safe_and_backwards_compatible(self): options = build_workflow_editor_options( @@ -214,10 +238,13 @@ def test_editor_capabilities_and_eligibility_are_safe_and_backwards_compatible(s {"id": "unknown", "name": "Unknown", "loop_eligible": True, "secret": "PRIVATE"}, ], ) - self.assertEqual(options["supported_node_kinds"], ["task", "if", "route", "for_each", "collect"]) + self.assertEqual( + options["supported_node_kinds"], + ["task", "if", "route", "for_each", "collect", "repeat_until"], + ) self.assertEqual(options["supported_iterable_kinds"], ["input", "documents", "workspace_query"]) self.assertEqual(options["supported_query_modes"], ["all_matches", "best_n"]) - self.assertEqual(options["supported_binding_sources"], ["node_output", "loop_item"]) + self.assertEqual(options["supported_binding_sources"], ["node_output", "loop_item", "repeat_state"]) self.assertEqual(options["supported_input_processing_modes"], ["full", "saved_record_report"]) self.assertEqual(options["flow_limits"]["max_loop_items"], 1450) self.assertEqual([agent["loop_eligible"] for agent in options["agents"]], [True, False, False]) diff --git a/functional_tests/test_workflow_loop_schema.py b/functional_tests/test_workflow_loop_schema.py index 9f46cafaf..e9f0ee4ea 100644 --- a/functional_tests/test_workflow_loop_schema.py +++ b/functional_tests/test_workflow_loop_schema.py @@ -1,7 +1,7 @@ # test_workflow_loop_schema.py """ Isolated production-backed compiler and iteration identity regression tests. -Version: 0.261.117 +Version: 0.261.120 Implemented in: 0.261.117 Validates additive loop schemas, frozen-source descriptors, lexical availability, @@ -859,7 +859,7 @@ def test_group_iterable_scope_is_bound_to_the_server_normalization_context(): @pytest.mark.parametrize("path", [ {}, (), "", [None], [{}], [{**frame(), "iteration": 0}], [{**frame(), "repeat_id": "again"}], - [{"loop_id": "each_source", "iteration": 0}], [{**frame(), "item_id": "A" * 64}], + [{"loop_id": "each_source", "iteration": -1}], [{**frame(), "item_id": "A" * 64}], [{**frame(), "item_id": "a" * 63}], [{**frame(), "item_id": "z" * 64}], [{**frame(), "index": True}], [{**frame(), "index": -1}], [{**frame(), "index": 5000}], [{**frame(), "index": 0.0}], [{**frame(), "index": "0"}], [{**frame(), "loop_id": "bad/path"}], @@ -879,6 +879,13 @@ def test_iteration_paths_copy_values_and_none_remains_root_scope(): assert path[0]["index"] == 0 +def test_repeat_frame_cannot_replace_a_for_each_item_frame(): + path = [{"loop_id": "each_source", "iteration": 0}] + assert normalize_workflow_iteration_path(path) == path + with pytest.raises(ValueError): + workflow_execution_id(definition(), "run", "analyze_node", path) + + def test_execution_and_producer_identities_follow_every_engine_and_task_ancestor(): workflow = nested_definition() compiled = compile_workflow_flow(workflow) diff --git a/functional_tests/test_workflow_repeat_dispatcher.py b/functional_tests/test_workflow_repeat_dispatcher.py new file mode 100644 index 000000000..251c9c61b --- /dev/null +++ b/functional_tests/test_workflow_repeat_dispatcher.py @@ -0,0 +1,157 @@ +# test_workflow_repeat_dispatcher.py +""" +Functional tests for post-body Repeat in the production task dispatcher. +Version: 0.261.120 +Implemented in: 0.261.120 + +Exercises real journal, result transport, typed state, and dispatch checkpoints +using a closed model client. Recovery interrupts after the task unit commits but +before its result checkpoint, not merely after a completed flow traversal. +""" + +import copy +import json +from types import SimpleNamespace + +import pytest + +from test_workflow_repeat_execution import repeat_runtime +from test_workflow_repeat_schema import repeat_definition +from test_workflow_result_store import FakeBlobService +from test_workflow_task_result_handoff import build_inventory_run +from functions_workflow_definitions import workflow_definition_revision +from functions_workflow_execution import WorkflowSuspended, current_workflow_execution, workflow_execution_scope +from functions_workflow_results import authorize_workflow_task_result_read, persist_workflow_task_result +from functions_workflow_result_store import WorkflowResultStore +from functions_workflow_runtime_store import WorkflowRuntimeLease, WorkflowRuntimeStore +from functions_workflow_structured_execution import StructuredWorkflowExecution + + +def production_dispatcher(reply): + runner, _, _, _, _, _ = build_inventory_run() + calls = [] + + def completion(**_kwargs): + execution = current_workflow_execution() + selectors = copy.deepcopy(execution.selectors()) + calls.append(selectors) + content = json.dumps(reply(selectors)) + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=content))], usage=None, + ) + + client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=completion))) + runner.update({ + "get_workflow_alert_signals": lambda: [], + "_resolve_model_workflow_client": lambda *args, **kwargs: ( + runner["WorkflowModelClient"](client, "gpt-4.1", "aoai"), "gpt-4.1", "aoai", + ), + "persist_workflow_task_result": lambda envelope, **kwargs: persist_workflow_task_result( + envelope, **{**kwargs, "settings": {"max_file_size_mb": 10}}, + ), + "authorize_workflow_task_result_read": authorize_workflow_task_result_read, + }) + return runner, calls + + +def dispatch(runner, workflow, store): + with WorkflowRuntimeLease(store, owner_id="repeat-dispatcher") as lease: + execution = StructuredWorkflowExecution(store, lease, workflow, "run", settings={}) + with workflow_execution_scope(execution): + return runner["_execute_workflow_task_sequence"]( + workflow, {}, "conversation", "run", None, {}, actor_user_id="owner", + ) + + +@pytest.mark.parametrize("initial_ready", [False, True]) +@pytest.mark.parametrize("export_final", [False, True]) +def test_dispatcher_always_runs_one_body_before_testing_until(monkeypatch, initial_ready, export_final): + definition = repeat_definition(maximum=1) + definition["limits"]["max_executions"] = 4 + if not export_final: + definition["flow"]["nodes"][1]["exports"] = [] + definition["flow"]["outputs"] = [] + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + runner, calls = production_dispatcher( + lambda producer: {"ready": True if producer["iteration_path"] else initial_ready}, + ) + result = dispatch(runner, workflow, store) + assert [entry["iteration_path"] for entry in calls] == [[], [{"loop_id": "repeat", "iteration": 0}]] + assert result["workflow_outcome"] == {"status": "completed", "success": True} + assert store.read()["admitted_count"] == 4 + if export_final: + assert result["workflow_outputs"][0]["producer"]["node_id"] == "repeat" + assert not result["workflow_outputs"][0]["producer"].get("task_id") + else: + assert result["workflow_outputs"] == [] + + +@pytest.mark.parametrize("storage", ["cosmos", "blob"]) +def test_dispatcher_crash_gap_preserves_task_ordinal_and_does_not_reinvoke(monkeypatch, storage): + definition = repeat_definition(maximum=3) + definition["limits"]["max_executions"] = 6 + workflow, store, container, clock, _ = repeat_runtime(monkeypatch, definition=definition) + if storage == "blob": + blobs = FakeBlobService() + configured = lambda *args, **kwargs: WorkflowResultStore(container, blobs, "private-workflow-results") + monkeypatch.setattr("functions_workflow_result_store._configured_store", configured) + monkeypatch.setattr("functions_workflow_result_store._configured_result_store", configured) + runner, calls = production_dispatcher(lambda producer: { + "ready": bool(producer["iteration_path"] and producer["iteration_path"][-1]["iteration"] == 1), + }) + original_persist = runner["persist_workflow_task_result"] + original_cache = StructuredWorkflowExecution.cache + interruptions, orders = [], [] + + def persist(envelope, **kwargs): + execution = current_workflow_execution() + if ( + execution.iteration_path and execution.iteration_path[-1]["iteration"] == 1 + and not interruptions + ): + assert execution.unit("task:body")["state"] == "completed" + assert execution.snapshot("task-result:body") is None + interruptions.append(execution.execution_id()) + raise SystemExit("Closed worker loss after task-unit commit.") + return original_persist(envelope, **kwargs) + + def observe_order(execution, key, value): + retained = original_cache(execution, key, value) + if key == "task-order:body": + orders.append((execution.iteration_path[-1]["iteration"], value["order"], retained["order"])) + return retained + + runner["persist_workflow_task_result"] = persist + monkeypatch.setattr(StructuredWorkflowExecution, "cache", observe_order) + with pytest.raises(SystemExit, match="after task-unit commit"): + dispatch(runner, workflow, store) + assert len(calls) == 3 + clock.advance() + result = dispatch(runner, workflow, store) + assert len(calls) == 3 + assert (1, 2, 3) in orders + assert result["workflow_outcome"] == {"status": "completed", "success": True} + assert store.read()["admitted_count"] == 6 + assert store.read()["gate"] is None + + +def test_cancel_uses_frozen_repeat_identity_after_live_definition_changes(monkeypatch): + definition = repeat_definition(maximum=2) + definition["tasks"][1]["approval"] = {"required": True} + workflow, store, container, clock, _ = repeat_runtime(monkeypatch, definition=definition) + runner, calls = production_dispatcher(lambda _: {"ready": True}) + with pytest.raises(WorkflowSuspended): + dispatch(runner, workflow, store) + assert store.read()["state"] == "waiting_approval" and len(calls) == 1 + + edited = copy.deepcopy(workflow) + edited["flow"]["nodes"] = edited["flow"]["nodes"][:1] + edited["flow"]["outputs"] = [] + edited["tasks"] = edited["tasks"][:1] + edited["definition_revision"] = workflow_definition_revision(edited) + current_store = WorkflowRuntimeStore(container, edited, "run", clock=clock) + cancelled = current_store.request_cancel(actor_user_id="owner", request_id="cancel-edited-definition") + assert cancelled["state"] == "cancelled" + repeat_id = store.read()["repeat_progress"]["execution_id"] + assert store.journal_read("loop", repeat_id)["payload"]["state"] == "cancelled" + assert store.journal_read("iteration", [repeat_id, 0])["payload"]["state"] == "cancelled" diff --git a/functional_tests/test_workflow_repeat_editor_options.py b/functional_tests/test_workflow_repeat_editor_options.py new file mode 100644 index 000000000..6c87b7f89 --- /dev/null +++ b/functional_tests/test_workflow_repeat_editor_options.py @@ -0,0 +1,223 @@ +# test_workflow_repeat_editor_options.py +""" +Functional tests for Repeat administration and non-secret editor capabilities. +Version: 0.261.120 +Implemented in: 0.261.120 + +Exercises real registry normalization, Classic POST validation, editor projection, +and authorized scope adapters with fictional stores and no application clients. +The settings-writer regression is shared with test_workflow_loop_limits.py. +""" + +import ast +import copy +import json +from pathlib import Path +import sys +from types import ModuleType +import unittest +from unittest.mock import Mock, patch + + +ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" +sys.path.insert(0, str(APP_ROOT)) +sys.path.insert(0, str(ROOT / "functional_tests")) + +# Configure local paths before importing the production leaf modules and shared fixture. +from functions_workflow_editor import build_workflow_editor_options, get_workflow_editor_options +from functions_workflow_limits import ( + WORKFLOW_REPEAT_ITERATIONS_DEFAULT, + WORKFLOW_REPEAT_LIMIT_SETTING, + WorkflowLoopLimitError, + get_workflow_max_repeat_iterations, + validate_workflow_max_repeat_iterations, +) +from test_support.app_stubs import import_app_module +from test_workflow_loop_limits import _classic_limit_validator + + +def _options(**updates): + arguments = { + "scope_type": "personal", "scope_id": "fictional-owner", "can_manage": True, + "max_tasks": 50, "agents": [], "endpoints": [], + } + return build_workflow_editor_options(**{**arguments, **updates}) + + +class WorkflowRepeatEditorOptionsTests(unittest.TestCase): + def test_independent_administrator_defaults_match_storage_and_registry(self): + fields = import_app_module("admin_settings_fields") + repeat = fields.get_field_definition(WORKFLOW_REPEAT_LIMIT_SETTING) + loop = fields.get_field_definition("workflow_max_loop_items") + self.assertEqual(repeat["label"], "Workflow Repeat Iteration Limit") + self.assertEqual((repeat["default"], repeat["min"], repeat["max"], repeat["step"]), (25, 1, 1000, 1)) + self.assertNotIn("depends_on", repeat) + self.assertEqual((loop["default"], loop["min"], loop["max"]), (500, 1, 5000)) + + tree = ast.parse((APP_ROOT / "functions_settings.py").read_text(encoding="utf-8")) + get_settings = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "get_settings") + defaults = next( + node.value for node in get_settings.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "default_settings" for target in node.targets) + ) + value = next( + value for key, value in zip(defaults.keys, defaults.values) + if isinstance(key, ast.Constant) and key.value == WORKFLOW_REPEAT_LIMIT_SETTING + ) + self.assertEqual( + eval( + compile(ast.Expression(body=value), "repeat_default", "eval"), + {"WORKFLOW_REPEAT_ITERATIONS_DEFAULT": WORKFLOW_REPEAT_ITERATIONS_DEFAULT}, + ), + 25, + ) + + def test_registry_accepts_only_supported_whole_numbers_without_clamping(self): + fields = import_app_module("admin_settings_fields") + current = {"workflow_max_loop_items": 1800, WORKFLOW_REPEAT_LIMIT_SETTING: 200} + for value in (1, 25, 1000, "1", " 700 ", "1000"): + with self.subTest(value=value): + normalized, errors, warnings = fields.normalize_admin_settings_updates( + {WORKFLOW_REPEAT_LIMIT_SETTING: value}, current, + ) + self.assertFalse(errors) + self.assertFalse(warnings) + self.assertEqual(normalized[WORKFLOW_REPEAT_LIMIT_SETTING], int(value)) + self.assertNotIn("workflow_max_loop_items", normalized) + for value in (None, "", " ", "1.5", "1e3", "invalid-secret", -1, 0, 1001, 25.0, True, False, [], {}): + with self.subTest(value=value): + normalized, errors, _warnings = fields.normalize_admin_settings_updates( + {WORKFLOW_REPEAT_LIMIT_SETTING: value}, current, + ) + self.assertNotIn(WORKFLOW_REPEAT_LIMIT_SETTING, normalized) + self.assertIn("1 to 1,000", errors[WORKFLOW_REPEAT_LIMIT_SETTING]) + self.assertNotIn("invalid-secret", errors[WORKFLOW_REPEAT_LIMIT_SETTING]) + self.assertEqual(current, {"workflow_max_loop_items": 1800, WORKFLOW_REPEAT_LIMIT_SETTING: 200}) + + def test_unrelated_admin_updates_do_not_insert_or_reset_repeat_policy(self): + fields = import_app_module("admin_settings_fields") + for current in ({}, {WORKFLOW_REPEAT_LIMIT_SETTING: 700, "workflow_max_loop_items": 1800}): + with self.subTest(current=current): + before = copy.deepcopy(current) + normalized, errors, _warnings = fields.normalize_admin_settings_updates( + {"allow_user_workflows": True}, current, + ) + self.assertFalse(errors) + self.assertNotIn(WORKFLOW_REPEAT_LIMIT_SETTING, normalized) + self.assertEqual(current, before) + + def test_classic_post_preserves_absent_policy_and_reports_safe_errors(self): + validate, flashes = _classic_limit_validator( + WORKFLOW_REPEAT_LIMIT_SETTING, + validate_workflow_max_repeat_iterations, + get_workflow_max_repeat_iterations, + ) + current = {WORKFLOW_REPEAT_LIMIT_SETTING: 700, "workflow_max_loop_items": 1800} + self.assertEqual(validate({}, {}), 25) + self.assertEqual(validate({}, current), 700) + for value in ("1", "25", "1000"): + self.assertEqual(validate({WORKFLOW_REPEAT_LIMIT_SETTING: value}, current), int(value)) + for value in ("", "0", "1001", "25.0", "invalid-secret"): + response = validate({WORKFLOW_REPEAT_LIMIT_SETTING: value}, current) + self.assertEqual(response, ("redirect", "frontend_admin_settings.admin_settings")) + message, category = flashes[-1] + self.assertEqual(category, "danger") + self.assertIn("1 to 1,000", message) + self.assertNotIn("invalid-secret", message) + self.assertEqual(current, {WORKFLOW_REPEAT_LIMIT_SETTING: 700, "workflow_max_loop_items": 1800}) + + tree = ast.parse((APP_ROOT / "route_frontend_admin_settings.py").read_text(encoding="utf-8")) + persisted = [ + value + for node in ast.walk(tree) if isinstance(node, ast.Dict) + for key, value in zip(node.keys, node.values) + if isinstance(key, ast.Constant) and key.value == WORKFLOW_REPEAT_LIMIT_SETTING + ] + self.assertEqual(len(persisted), 1) + self.assertIsInstance(persisted[0], ast.Name) + self.assertEqual(persisted[0].id, WORKFLOW_REPEAT_LIMIT_SETTING) + + def test_editor_advertises_repeat_without_changing_existing_capabilities(self): + options = _options(max_loop_items=1800, max_repeat_iterations=700) + self.assertEqual(options["supported_node_kinds"], ["task", "if", "route", "for_each", "collect", "repeat_until"]) + self.assertEqual(options["supported_binding_sources"], ["node_output", "loop_item", "repeat_state"]) + self.assertEqual(options["flow_limits"]["max_repeat_iterations"], 700) + self.assertEqual(options["flow_limits"]["hard_repeat_iterations"], 1000) + self.assertEqual(options["flow_limits"]["max_loop_items"], 1800) + self.assertEqual(options["supported_definition_versions"], [1, 2, 3]) + self.assertEqual(options["supported_iterable_kinds"], ["input", "documents", "workspace_query"]) + self.assertEqual(options["publication_source_capabilities"][-1], { + "source_kind": "saved_output", "output_kinds": ["records"], "artifact_formats": ["json"], + }) + for scope in ("personal", "group"): + with self.subTest(scope=scope): + defaults = _options(scope_type=scope, can_manage=False) + self.assertEqual(defaults["flow_limits"]["max_repeat_iterations"], 25) + self.assertEqual(defaults["flow_limits"]["max_loop_items"], 500) + self.assertFalse(defaults["can_manage"]) + + def test_invalid_explicit_editor_policy_fails_instead_of_becoming_default(self): + for value in (None, "", 0, 1001, True, 25.0, "invalid-secret"): + with self.subTest(value=value), self.assertRaises(WorkflowLoopLimitError) as raised: + _options(max_repeat_iterations=value) + self.assertEqual(raised.exception.code, "workflow_repeat_limit_invalid") + self.assertNotIn("invalid-secret", raised.exception.public_message) + self.assertEqual(_options(max_repeat_iterations=1)["flow_limits"]["max_repeat_iterations"], 1) + self.assertEqual(_options(max_repeat_iterations=1000)["flow_limits"]["max_repeat_iterations"], 1000) + + def test_hosted_nonloop_choices_and_nonsecret_projection_are_unchanged(self): + options = _options(agents=[ + {"id": "local", "name": "Local", "agent_type": "local", "loop_eligible": False}, + {"id": "hosted", "name": "Hosted", "agent_type": "foundry", "loop_eligible": True}, + {"id": "unknown", "name": "Unknown", "loop_eligible": True, "secret": "PRIVATE"}, + ]) + self.assertEqual([agent["id"] for agent in options["agents"]], ["local", "hosted", "unknown"]) + self.assertEqual([agent["loop_eligible"] for agent in options["agents"]], [True, False, False]) + self.assertNotIn("PRIVATE", json.dumps(options)) + + def test_authorized_scope_adapter_reads_current_independent_limits(self): + group = ModuleType("functions_group") + group.assert_group_role = Mock(return_value="User") + group_workflows = ModuleType("functions_group_workflows") + group_workflows.GROUP_WORKFLOW_MEMBER_ROLES = ("Owner", "Admin", "DocumentManager", "User") + group_workflows._build_model_endpoint_candidates = Mock(return_value=[]) + group_workflows.get_group_workflow_agent_options = Mock(return_value=[]) + personal_workflows = ModuleType("functions_personal_workflows") + personal_workflows._build_default_model_summary = Mock(return_value={"valid": True}) + personal_workflows._build_model_endpoint_candidates = Mock(return_value=[]) + personal_workflows._build_selectable_agents = Mock(return_value=[]) + personal_workflows.get_workflow_max_tasks = Mock(return_value=50) + settings_module = ModuleType("functions_settings") + settings_module.get_group_workflow_management_roles = Mock(return_value=["Owner", "Admin"]) + modules = { + module.__name__: module + for module in (group, group_workflows, personal_workflows, settings_module) + } + settings = { + WORKFLOW_REPEAT_LIMIT_SETTING: 750, "workflow_max_loop_items": 1700, + "private_secret": "PRIVATE", + } + before = copy.deepcopy(settings) + with patch.dict(sys.modules, modules): + for group_id in ("", "fictional-group"): + with self.subTest(group_id=group_id): + result = get_workflow_editor_options("fictional-owner", settings, group_id=group_id) + self.assertEqual(result["flow_limits"]["max_repeat_iterations"], 750) + self.assertEqual(result["flow_limits"]["max_loop_items"], 1700) + self.assertEqual(result["scope"]["id"], group_id or "fictional-owner") + self.assertEqual(result["can_manage"], not group_id) + self.assertNotIn("PRIVATE", json.dumps(result)) + group.assert_group_role.assert_called_once_with( + "fictional-owner", "fictional-group", + allowed_roles=group_workflows.GROUP_WORKFLOW_MEMBER_ROLES, + ) + group.assert_group_role.side_effect = PermissionError("Not a current group member.") + with self.assertRaises(PermissionError): + get_workflow_editor_options("fictional-owner", settings, group_id="fictional-group") + self.assertEqual(settings, before) + + +if __name__ == "__main__": + unittest.main() diff --git a/functional_tests/test_workflow_repeat_execution.py b/functional_tests/test_workflow_repeat_execution.py new file mode 100644 index 000000000..a7d9cfdf1 --- /dev/null +++ b/functional_tests/test_workflow_repeat_execution.py @@ -0,0 +1,266 @@ +# test_workflow_repeat_execution.py +""" +Production-backed post-body Repeat state, batching and mixed-path regressions. +Version: 0.261.120 +Implemented in: 0.261.120 + +The real compiler, journal, execution units and result store use closed fictional +fixtures. No model, credentials, Azure service, or publication destination is used. +""" + +import copy +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "application" / "single_app")) + +# Production imports follow the isolated worktree import setup. +from functions_analysis_access import AnalysisResultUnavailable +from functions_workflow_execution import WorkflowSuspended, workflow_execution_scope +from functions_workflow_flow_runner import WorkflowFlowRunner +from functions_workflow_identity import workflow_execution_id +from functions_workflow_iterations import authorize_iteration_path +from functions_workflow_node_results import load_workflow_node_input, open_workflow_record_input +from functions_workflow_repeat_history import workflow_repeat_iterations_page, workflow_repeat_state_page +from functions_workflow_results import build_workflow_task_result, persist_workflow_task_result, workflow_result_summary +from functions_workflow_runtime import queue_durable_workflow_run +from functions_workflow_runtime_store import WorkflowRuntimeConflict, WorkflowRuntimeLease, WorkflowRuntimeStore +from functions_workflow_structured_execution import StructuredWorkflowExecution +from functions_workflow_validation import validate_workflow_task_output +from test_workflow_for_each_execution import LoopJournalContainer +from test_workflow_runtime_integration import integration # noqa: F401 +from test_workflow_structured_flow import binding, create_structured_runtime, task + + +def state_binding(name="state", loop_id="repeat", *, slot="state", kind="json", allow_partial=False): + return { + "name": name, "source": {"kind": "repeat_state", "loop_id": loop_id, "state_name": slot, "scope": "current"}, + "required": True, "expected_kind": kind, "allow_partial": allow_partial, + } + + +def repeat_definition(maximum=25): + contract = {"kind": "json", "schema": { + "type": "object", "required": ["count", "ready"], + "properties": {"count": {"type": "integer"}, "ready": {"type": "boolean"}}, + }} + return { + "id": "workflow", "user_id": "owner", "definition_version": 3, "durable_execution": True, + "runner_type": "model", "chat_capabilities_enabled": False, + "limits": {"max_executions": 5000, "deadline_seconds": 86400}, + "error_handling": {"strategy": "halt", "retry_count": 0}, + "tasks": [task("source", contract=contract), task("body", inputs=[state_binding()], contract=contract)], + "flow": {"id": "root", "nodes": [ + {"id": "source-node", "kind": "task", "task_id": "source"}, + {"id": "repeat", "kind": "repeat_until", "max_iterations": maximum, + "state": [{"name": "state", "initial": {"kind": "node_output", "node_id": "source-node", + "output": "json", "scope": "current"}, + "next": "next", "output_contract": contract}], + "body": {"id": "body-region", "nodes": [{"id": "body-node", "kind": "task", "task_id": "body"}], + "outputs": [binding("body-node", "next", "json")]}, + "until": {"op": "eq", "left": {"input": "state", "path": "/ready"}, "right": {"literal": True}}, + "exports": [{"name": "state", "output": "next"}]}, + ], "outputs": [binding("repeat", "answer", "state")]}, + } + + +def repeat_runtime(monkeypatch, *, definition=None, maximum=25, policy=25): + monkeypatch.setattr("test_workflow_structured_flow.JournalContainer", LoopJournalContainer) + original = WorkflowRuntimeStore.initialize + + def initialize(self, **options): + return original(self, **{**options, "repeat_policy": {"max_iterations": policy}}) + + monkeypatch.setattr(WorkflowRuntimeStore, "initialize", initialize) + workflow, store, container, clock = create_structured_runtime(definition or repeat_definition(maximum), monkeypatch) + for module in ( + "functions_workflow_iterations", "functions_workflow_repeat_state", "functions_workflow_loop_history", + "functions_workflow_execution_history", "functions_workflow_repeat_history", + ): + monkeypatch.setattr(f"{module}.workflow_runtime_store", lambda *args: store) + events = [] + monkeypatch.setattr("functions_workflow_repeat_execution.log_repeat_event", lambda _, value: events.append(copy.deepcopy(value))) + monkeypatch.setattr("functions_workflow_repeat_state.log_repeat_event", lambda _, value: events.append(copy.deepcopy(value))) + return workflow, store, container, clock, events + + +def execute_repeat(workflow, store, *, target=1, initial_ready=False, calls=None, result_for_task=None, + interrupt_at=None, stream_collections=False, replay_safe=True, envelope_transform=None): + calls = calls if calls is not None else [] + outcomes = [] + with WorkflowRuntimeLease(store, owner_id="repeat-worker") as lease: + execution = StructuredWorkflowExecution(store, lease, workflow, "run", settings={}) + with workflow_execution_scope(execution): + flow = WorkflowFlowRunner(workflow, "run", execution, outcomes, actor_user_id="owner", settings={}) + for current in flow.tasks(): + if current["id"] == "body" and execution.iteration_path[-1].get("iteration") == interrupt_at: + raise SystemExit("Fictional worker interruption.") + resolved = flow.resolve(current["inputs"], stream_collections=stream_collections) + + def invoke(): + calls.append((current["id"], copy.deepcopy(execution.iteration_path), execution.execution_id())) + if result_for_task is not None: + return result_for_task(current, resolved, execution) + if current["id"] == "source": + value = {"count": 0, "ready": initial_ready} + else: + count = resolved["values"]["state"]["count"] + 1 + value = {"count": count, "ready": count >= target} + return {"reply": "", "authoritative_result": {"kind": "json", "value": value}} + + result = execution.run_unit( + f"task:{current['id']}", invoke, + inputs={"task": current, "consumed_inputs": resolved["consumed_inputs"], + "iteration_inputs": resolved["iteration_inputs"]}, + replay_safe=replay_safe, approval=current.get("approval"), + ) + attempt = execution.unit(f"task:{current['id']}")["attempt"] + envelope = build_workflow_task_result(result, workflow=workflow, run_id="run", task=current, attempt_count=attempt) + envelope["consumed_inputs"] = resolved["consumed_inputs"] + envelope["workflow_validation"] = validate_workflow_task_output(envelope, current["output_contract"]) + if envelope_transform: + envelope_transform(current, envelope) + manifest, reference = persist_workflow_task_result( + envelope, workflow=workflow, run_id="run", task_id=current["id"], settings={}, + ) + summary = workflow_result_summary(manifest, reference) + outcomes.append({ + "task": current, "status": "succeeded", "attempt_count": attempt, + "execution_id": execution.execution_id(), "iteration_path": copy.deepcopy(execution.iteration_path), + "consumed_inputs": resolved["consumed_inputs"], + "result": {**result, "workflow_result": summary, "workflow_validation": envelope["workflow_validation"]}, + }) + return flow, calls + + +def repeat_head(workflow, store): + return store.journal_read("loop", workflow_execution_id(workflow, "run", "repeat"))["payload"] + + +def continue_repeat(store, request_id="manual-batch"): + control = store.read() + return store.decide( + expected_version=control["version"], gate_id=control["gate"]["id"], choice="continue_repeat", + actor_user_id="owner", request_id=request_id, + ) + + +@pytest.mark.parametrize("target,initial_ready", [(1, True), (1, False), (2, True), (3, False), (25, False)]) +def test_repeat_runs_post_body_and_exports_only_last_exact_producer(monkeypatch, target, initial_ready): + workflow, store, _, _, events = repeat_runtime(monkeypatch) + flow, calls = execute_repeat(workflow, store, target=target, initial_ready=initial_ready) + assert flow.finished and not flow.failed and not events + body = [call for call in calls if call[0] == "body"] + assert len(body) == target and [call[1][-1]["iteration"] for call in body] == list(range(target)) + receipt = flow.final_outputs[0] + assert "task_id" not in receipt["producer"] + payload, _ = load_workflow_node_input( + workflow, "run", receipt["producer"], receipt["result_ref"], output_name="state", + ) + assert json.loads(payload)["value"] == {"count": target, "ready": True} + head = repeat_head(workflow, store) + assert head["completed_count"] == target and head["state"] == "completed" + assert store.read()["admitted_count"] == 2 + target * 2 + assert len(json.dumps(store.read())) < 16384 and len(json.dumps(head)) < 8192 + + +def test_exhausted_batch_requires_explicit_idempotent_grant(monkeypatch): + workflow, store, _, clock, events = repeat_runtime(monkeypatch, maximum=2) + calls = [] + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=3, calls=calls) + control = store.read() + assert control["state"] == "paused" and control["gate"]["reason_code"] == "repeat_iteration_limit" + assert control["gate"]["choices"] == ["continue_repeat", "cancel"] + head = repeat_head(workflow, store) + assert head["completed_count"] == 2 and head["exhaustion_count"] == 1 + assert store.journal_read("execution", head["execution_id"])["payload"].get("workflow_result") is None + with pytest.raises(WorkflowRuntimeConflict): + store.decide(expected_version=control["version"], gate_id=control["gate"]["id"], + choice="resume", actor_user_id="owner", request_id="not-a-grant") + queued = continue_repeat(store) + duplicate = store.decide( + expected_version=control["version"], gate_id=control["gate"]["id"], choice="continue_repeat", + actor_user_id="owner", request_id="manual-batch", + ) + assert duplicate["version"] == queued["version"] + with pytest.raises(WorkflowRuntimeConflict): + store.decide(expected_version=control["version"], gate_id=control["gate"]["id"], + choice="continue_repeat", actor_user_id="owner", request_id="stale-tab") + clock.advance() + flow, _ = execute_repeat(workflow, store, target=3, calls=calls) + assert flow.finished and len([call for call in calls if call[0] == "body"]) == 3 + assert [call[1][-1]["iteration"] for call in calls if call[0] == "body"] == [0, 1, 2] + assert repeat_head(workflow, store)["continuation_count"] == 1 + assert store.read()["repeat_counts"] == {"exhaustion_count": 1, "continuation_count": 1} + assert len(events) == 2 and events[0]["event_id"] != events[1]["event_id"] + + +def test_unsealed_or_substituted_repeat_path_is_not_authorized(monkeypatch): + workflow, store, _, _, _ = repeat_runtime(monkeypatch) + flow, _ = execute_repeat(workflow, store, target=2) + receipt = flow.final_outputs[0] + boundary = store.journal_read("execution", receipt["producer"]["execution_id"])["payload"]["workflow_result"] + source = boundary["outputs"]["state"]["selected_producer"]["producer"] + for forged in ( + {**source, "iteration_path": [{"loop_id": "repeat", "iteration": 1001}]}, + {**source, "iteration_path": [{"loop_id": "repeat", "iteration": 0}]}, + { + **source, "iteration_path": [{"loop_id": "repeat", "iteration": 1001}], + "execution_id": workflow_execution_id(workflow, "run", source["node_id"], [{"loop_id": "repeat", "iteration": 1001}]), + }, + ): + with pytest.raises(AnalysisResultUnavailable): + authorize_iteration_path(workflow, "run", forged, reader_user_id="owner", store=store) + + +def test_admin_policy_rejects_authored_maximum_without_clamping(monkeypatch): + with pytest.raises(WorkflowRuntimeConflict) as error: + repeat_runtime(monkeypatch, maximum=26, policy=25) + assert error.value.code == "repeat_policy_exceeded" + + +def test_queue_rejects_above_policy_before_writing_a_definition_snapshot(integration, monkeypatch): + workflow, _, services, _, _, requests = integration + workflow.clear() + workflow.update(repeat_definition(26)) + services["definitions"].upsert_item(workflow) + writes = [] + + def unexpected_snapshot(*args, **kwargs): + writes.append(True) + raise AssertionError("An unadmitted Repeat must not create result-store data.") + + monkeypatch.setattr("functions_workflow_runtime.save_workflow_runtime_result", unexpected_snapshot) + with pytest.raises(WorkflowRuntimeConflict) as error: + queue_durable_workflow_run(workflow, actor_user_id="owner") + assert error.value.code == "repeat_policy_exceeded" and writes == [] and requests == [] + assert services["runs"].items == {} + + +def test_real_thousand_round_batch_then_lifetime_round_1001(monkeypatch): + workflow, store, _, clock, _ = repeat_runtime(monkeypatch, maximum=1000, policy=1000) + calls = [] + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=1001, calls=calls) + first = [call for call in calls if call[0] == "body"] + assert len(first) == 1000 and first[-1][1][-1]["iteration"] == 999 + before = store.read() + assert before["admitted_count"] == 2002 and repeat_head(workflow, store)["next_iteration"] == 1000 + continue_repeat(store) + assert store.read()["deadline_at"] == before["deadline_at"] + clock.advance() + flow, _ = execute_repeat(workflow, store, target=1001, calls=calls) + last = [call for call in calls if call[0] == "body"] + assert flow.finished and len(last) == 1001 and last[-1][1][-1]["iteration"] == 1000 + assert last[-1][2] not in {call[2] for call in first} + assert store.read()["admitted_count"] == 2004 + assert store.read()["journal_counts"]["execution"] > 1000 + repeat_id = repeat_head(workflow, store)["execution_id"] + page = workflow_repeat_iterations_page(workflow, "run", repeat_id, reader_user_id="owner", limit=2) + assert len(page["iterations"]) == 2 and page["total_count"] == 1001 and page["next_cursor"] + state = workflow_repeat_state_page(workflow, "run", repeat_id, 1000, reader_user_id="owner", phase="after") + assert state["available"] and state["states"][0]["source"]["iteration_path"][-1]["iteration"] == 1000 diff --git a/functional_tests/test_workflow_repeat_limits.py b/functional_tests/test_workflow_repeat_limits.py new file mode 100644 index 000000000..730e23fbc --- /dev/null +++ b/functional_tests/test_workflow_repeat_limits.py @@ -0,0 +1,52 @@ +# test_workflow_repeat_limits.py +""" +Functional tests for the independent administrator Repeat batch ceiling. +Version: 0.261.120 +Implemented in: 0.261.120 + +Validates explicit settings without changing the existing For-each policy. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "application" / "single_app")) + +# Production imports follow the isolated repository path setup. +from functions_workflow_limits import ( + WORKFLOW_REPEAT_ITERATIONS_DEFAULT, WORKFLOW_REPEAT_ITERATIONS_MAX, + WorkflowLoopLimitError, get_workflow_max_loop_items, get_workflow_max_repeat_iterations, + validate_workflow_max_repeat_iterations, +) + + +def test_repeat_and_for_each_defaults_are_independent(): + assert WORKFLOW_REPEAT_ITERATIONS_DEFAULT == 25 + assert WORKFLOW_REPEAT_ITERATIONS_MAX == 1000 + assert get_workflow_max_repeat_iterations({}) == 25 + assert get_workflow_max_loop_items({}) == 500 + settings = {"workflow_max_loop_items": 5000, "workflow_max_repeat_iterations": 7} + assert get_workflow_max_repeat_iterations(settings) == 7 + assert get_workflow_max_loop_items(settings) == 5000 + + +@pytest.mark.parametrize("value,expected", [(1, 1), (25, 25), (1000, 1000), ("1", 1), (" 25 ", 25), ("1000", 1000)]) +def test_administrator_repeat_limit_accepts_whole_numbers(value, expected): + assert validate_workflow_max_repeat_iterations(value) == expected + + +@pytest.mark.parametrize("value", [None, 0, -1, 1001, 5000, True, False, 25.0, "", "25.0", "1e2", "twenty", [], {}, "\uff11\uff12"]) +def test_invalid_administrator_repeat_limit_is_not_clamped_or_defaulted(value): + with pytest.raises(WorkflowLoopLimitError) as failure: + get_workflow_max_repeat_iterations({"workflow_max_repeat_iterations": value}) + assert failure.value.code == "workflow_repeat_limit_invalid" + assert "1,000" in failure.value.public_message + + +@pytest.mark.parametrize("settings", [[], "unavailable", 25]) +def test_unavailable_settings_are_not_replaced_by_successful_defaults(settings): + with pytest.raises(WorkflowLoopLimitError) as failure: + get_workflow_max_repeat_iterations(settings) + assert failure.value.code == "workflow_repeat_limit_unavailable" diff --git a/functional_tests/test_workflow_repeat_publication.py b/functional_tests/test_workflow_repeat_publication.py new file mode 100644 index 000000000..4fa00a697 --- /dev/null +++ b/functional_tests/test_workflow_repeat_publication.py @@ -0,0 +1,193 @@ +# test_workflow_repeat_publication.py +""" +Functional tests for mixed Repeat native Analyze and exact saved-output publication. +Version: 0.261.120 +Implemented in: 0.261.120 + +The production task dispatcher, native checkpoints, shared JSON renderer and sole +publication ledger use existing closed fictional source, Blob and Cosmos fixtures. +""" + +from copy import deepcopy +import hashlib +import json + +import pytest + +import test_workflow_loop_native_analysis as native_loops +from test_analyze_native_saved_integration import native_run # noqa: F401 +from test_analysis_artifact_publication import publication, normalizers # noqa: F401 +from test_workflow_saved_output_artifacts import artifact_services # noqa: F401 +from functions_analysis_access import AnalysisResultUnavailable +from functions_workflow_node_results import open_workflow_record_input + + +def repeat_node(initial_node, body_nodes, output_node, contract): + return { + "id": "repeat", "kind": "repeat_until", "max_iterations": 1, + "state": [{ + "name": "findings", "initial": {"kind": "node_output", "node_id": initial_node, "output": "records", "scope": "current"}, + "next": "findings", "output_contract": deepcopy(contract), + }], + "body": {"id": "repeat-body", "nodes": body_nodes, "outputs": [{ + "name": "findings", "source": {"kind": "node_output", "node_id": output_node, "output": "records", "scope": "current"}, + "required": True, "expected_kind": "records", "allow_partial": False, + }]}, + "until": {"op": "eq", "left": {"literal": True}, "right": {"literal": True}}, + "exports": [{"name": "findings", "output": "findings"}], + } + + +def redirect_collect(value): + if isinstance(value, dict): + if value.get("node_id") == "collect": + value.update(node_id="repeat", output="findings") + for child in value.values(): + redirect_collect(child) + elif isinstance(value, list): + for child in value: + redirect_collect(child) + + +@pytest.fixture +def native_repeat_flow(native_run, monkeypatch, request): + options = getattr(request, "param", None) or {} + original = native_loops.loop_runtime + + def runtime(patch, *, definition): + definition = deepcopy(definition) + each, collect, *following = definition["flow"]["nodes"] + body_task = next(task for task in definition["tasks"] if task["id"] == "body") + contract = body_task["output_contract"] + if options.get("nesting") == "repeat_in_each": + refine = deepcopy(body_task) + refine.update(id="refine", name="Refine fictional document") + definition["tasks"].append(refine) + boundary = repeat_node( + "body-node", [{"id": "refine-node", "kind": "task", "task_id": "refine"}], "refine-node", contract, + ) + each["body"]["nodes"].append(boundary) + each["body"]["outputs"][0]["source"].update(node_id="repeat", output="findings") + else: + seed_each, seed_collect, seed_task = deepcopy(each), deepcopy(collect), deepcopy(body_task) + seed_task.update(id="seed-body", name="Seed fictional document") + seed_task["document_action"]["loop_id"] = "seed-each" + for binding in seed_task["inputs"]: + if binding["source"]["kind"] == "loop_item": + binding["source"]["loop_id"] = "seed-each" + seed_each["id"] = "seed-each" + seed_each["body"].update(id="seed-body-region", nodes=[ + {"id": "seed-body-node", "kind": "task", "task_id": "seed-body"}, + ]) + seed_each["body"]["outputs"][0]["source"]["node_id"] = "seed-body-node" + seed_collect["id"] = "seed-collect" + seed_collect["source"]["loop_id"] = "seed-each" + definition["tasks"].append(seed_task) + boundary = repeat_node("seed-collect", [each, collect], "collect", contract) + redirect_collect(following) + redirect_collect(definition["flow"]["outputs"]) + for task in definition["tasks"]: + if task["id"] != "body": + redirect_collect(task.get("inputs", [])) + definition["flow"]["nodes"] = [seed_each, seed_collect, boundary, *following] + return original(patch, definition=definition) + + monkeypatch.setattr(native_loops, "loop_runtime", runtime) + return native_loops.native_loop_flow.__wrapped__(native_run, monkeypatch, request) + + +@pytest.mark.parametrize("native_repeat_flow", [ + {"nesting": nesting, "storage": storage} + for nesting in ("repeat_in_each", "each_in_repeat") for storage in ("cosmos", "blob") +], indirect=True) +def test_native_analyze_keeps_exact_mixed_producers_and_original_records(native_repeat_flow): + fixture = native_repeat_flow + completed = fixture["execute"]() + assert completed["workflow_outcome"] == {"status": "completed", "success": True} + assert len(fixture["calls"]) == 4 and len({producer["execution_id"] for _, producer in fixture["calls"]}) == 4 + mixed = [producer for _, producer in fixture["calls"] if len(producer["iteration_path"]) == 2] + assert len(mixed) == 2 + assert all(any(frame.get("iteration") == 0 for frame in producer["iteration_path"]) for producer in mixed) + assert all(any("item_id" in frame for frame in producer["iteration_path"]) for producer in mixed) + receipt = completed["workflow_outputs"][0] + reader = open_workflow_record_input( + fixture["workflow"], "run", receipt["producer"], receipt["result_ref"], + output_name=receipt["output_name"], source_resolver=fixture["source_resolver"], + ) + rows = list(reader.iter_records()) + assert len(rows) == 300 and [row["values"] for row in rows[:150]] == fixture["native_run"].rows + before = len(fixture["native_run"].reads) + fixture["execute"]() + assert len(fixture["calls"]) == 4 and len(fixture["native_run"].reads) == before + fixture["allowed"]["native-source-a"] = False + with pytest.raises(AnalysisResultUnavailable): + reader.read_records(offset=0, limit=1) + + +@pytest.mark.parametrize("native_repeat_flow", [ + {"storage": storage, "join": join, "publication": { + "source_kind": "saved_output", "artifact_format": "json", "workspace_scope": "personal", "completion_policy": "submitted", + }} + for storage in ("cosmos", "blob") for join in (False, True) +], indirect=True) +def test_repeat_final_records_use_unchanged_shared_export_and_destination_ledger(artifact_services, native_repeat_flow): + fixture, services = native_repeat_flow, artifact_services + services.bind(fixture["workflow"], fixture["store"]) + + def interrupt(): + raise SystemExit("Fictional restart after Repeat completion, before publication.") + + with pytest.raises(SystemExit): + fixture["execute"](before_publication=interrupt) + assert len(fixture["calls"]) == 4 and services.blobs.writes == 0 + completed = fixture["execute"]() + assert completed["workflow_outcome"] == {"status": "completed", "success": True} + assert completed["publication"]["state"] == "submitted" and completed["publication"]["policy_satisfied"] + card = completed["generated_tabular_outputs"][-1] + artifact = services.publication.messages.records[card["artifact_message_id"]] + source = artifact["metadata"]["generated_artifact_source"] + assert source["producer"]["node_id"] == ("selected" if fixture["options"]["join"] else "repeat") + assert "task_id" not in source["producer"] + content = services.blobs.data[(artifact["blob_container"], artifact["blob_path"])] + rows = json.loads(content) + assert len(rows) == 300 and [row["values"] for row in rows[:150]] == fixture["native_run"].rows + assert artifact["metadata"]["generated_artifact_content_sha256"] == hashlib.sha256(content).hexdigest() + assert services.publication.calls["queue"][0]["file_content_bytes"] == content + before = deepcopy(services.publication.calls) + assert fixture["execute"]()["publication"] == completed["publication"] + assert services.publication.calls["create"] == before["create"] and services.publication.calls["queue"] == before["queue"] + assert services.blobs.writes == 1 and len(fixture["calls"]) == 4 + fixture["allowed"]["native-source-b"] = False + with pytest.raises((PermissionError, AnalysisResultUnavailable, ValueError)): + services.download("owner", "conversation-1", card["artifact_message_id"]) + + +@pytest.mark.parametrize("native_repeat_flow", [ + {"storage": storage, "publication": { + "source_kind": "saved_output", "artifact_format": "json", "workspace_scope": "personal", "completion_policy": "submitted", + }} for storage in ("cosmos", "blob") +], indirect=True) +def test_repeat_restart_after_destination_submission_reuses_the_existing_ledger( + artifact_services, native_repeat_flow, monkeypatch, +): + fixture, services = native_repeat_flow, artifact_services + services.bind(fixture["workflow"], fixture["store"]) + original = services.publication.module.publish_generated_chat_artifact_for_user + interrupted = [] + + def submit_then_interrupt(*args, **kwargs): + submitted = original(*args, **kwargs) + if not interrupted: + interrupted.append(True) + raise SystemExit("Fictional restart after the sole destination ledger committed.") + return submitted + + monkeypatch.setattr(services.publication.module, "publish_generated_chat_artifact_for_user", submit_then_interrupt) + with pytest.raises(SystemExit): + fixture["execute"]() + assert len(services.publication.calls["create"]) == len(services.publication.calls["queue"]) == 1 + completed = fixture["execute"]() + assert completed["workflow_outcome"] == {"status": "completed", "success": True} + assert completed["publication"]["state"] == "submitted" and completed["publication"]["policy_satisfied"] + assert len(services.publication.calls["create"]) == len(services.publication.calls["queue"]) == 1 + assert len(fixture["calls"]) == 4 and services.blobs.writes == 1 diff --git a/functional_tests/test_workflow_repeat_recovery.py b/functional_tests/test_workflow_repeat_recovery.py new file mode 100644 index 000000000..bf450a1af --- /dev/null +++ b/functional_tests/test_workflow_repeat_recovery.py @@ -0,0 +1,511 @@ +# test_workflow_repeat_recovery.py +""" +Functional tests for Repeat atomic recovery, live authority and lifetime budgets. +Version: 0.261.120 +Implemented in: 0.261.120 + +Exercises the real journal and result store with fictional transactional Cosmos, +Blob and clock fixtures. External effects are local counters only. +""" + +import copy +import json +import sys +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +import pytest +from azure.cosmos.exceptions import CosmosHttpResponseError + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "application" / "single_app")) + +# Production imports follow the isolated worktree import setup. +from functions_analysis_access import AnalysisResultUnavailable +from functions_workflow_execution import WorkflowSuspended +from functions_workflow_execution_history import workflow_execution_history +from functions_workflow_identity import workflow_execution_id +from functions_workflow_node_results import load_workflow_node_input +from functions_workflow_repeat_history import workflow_repeat_iterations_page, workflow_repeat_state_page +from functions_workflow_result_store import WorkflowResultStore +from functions_workflow_runtime_store import WorkflowRuntimeConflict +from test_workflow_for_each_execution import execute_loop, loop_runtime +from test_workflow_repeat_execution import ( + continue_repeat, execute_repeat, repeat_definition, repeat_head, repeat_runtime, +) +from test_workflow_result_store import FakeBlobService + + +@pytest.mark.parametrize("stage", ["admission_before", "admission_after", "transition_before", "transition_after"]) +def test_restart_reuses_sealed_state_and_committed_body_units(monkeypatch, stage): + workflow, store, _, clock, _ = repeat_runtime(monkeypatch, maximum=3) + original = store.journal_commit_many + interrupted = [] + calls = [] + + def stop(token, entries, **options): + key = entries[0]["key"] + wanted = "repeat-iteration" if stage.startswith("admission") else "repeat-transition" + if not interrupted and key[0] == wanted and key[-1] == 1: + interrupted.append(True) + if stage.endswith("after"): + original(token, entries, **options) + raise SystemExit("Closed fixture crash.") + return original(token, entries, **options) + + monkeypatch.setattr(store, "journal_commit_many", stop) + with pytest.raises(SystemExit): + execute_repeat(workflow, store, target=3, calls=calls) + first = list(calls) + clock.advance() + flow, _ = execute_repeat(workflow, store, target=3, calls=calls) + assert flow.finished + assert len([call for call in calls if call[0] == "body"]) == 3 + assert len({call[2] for call in calls}) == len(calls) + assert calls[:len(first)] == first and store.read()["admitted_count"] == 8 + assert repeat_head(workflow, store)["completed_count"] == 3 + + +@pytest.mark.parametrize("kind", ["repeat-iteration", "repeat-transition", "grant"]) +def test_lost_acknowledgement_does_not_duplicate_transitions_or_grants(monkeypatch, kind): + workflow, store, container, clock, events = repeat_runtime(monkeypatch, maximum=1) + original = container.execute_item_batch + lost = [] + + def commit_then_disconnect(batch_operations, partition_key): + wanted = any( + operation[0] == "create" and operation[1][0].get("record_kind") in {"admission", "decision"} + and ( + operation[1][0].get("key", [None])[0] == kind + or kind == "grant" and operation[1][0].get("payload", {}).get("choice") == "continue_repeat" + ) + for operation in batch_operations + ) + value = original(batch_operations, partition_key) + if wanted and not lost: + lost.append(True) + raise CosmosHttpResponseError(status_code=503) + return value + + monkeypatch.setattr(container, "execute_item_batch", commit_then_disconnect) + calls = [] + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, calls=calls) + continue_repeat(store) + clock.advance() + flow, _ = execute_repeat(workflow, store, target=2, calls=calls) + assert flow.finished and lost + assert store.read()["admitted_count"] == 6 + assert store.read()["repeat_counts"] == {"exhaustion_count": 1, "continuation_count": 1} + assert len(events) == 2 and len([call for call in calls if call[0] == "body"]) == 2 + + +def test_uncertain_effect_uses_existing_recovery_gate_not_manual_batch_grant(monkeypatch): + workflow, store, _, clock, _ = repeat_runtime(monkeypatch) + effects = [] + + def effect(current, resolved, execution): + if current["id"] == "source": + value = {"count": 0, "ready": False} + else: + effects.append(execution.execution_id()) + if len(effects) == 1: + raise SystemExit("Uncertain fictional effect.") + value = {"count": 1, "ready": True} + return {"reply": "", "authoritative_result": {"kind": "json", "value": value}} + + with pytest.raises(SystemExit): + execute_repeat(workflow, store, result_for_task=effect, replay_safe=False) + clock.advance() + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, result_for_task=effect, replay_safe=False) + control = store.read() + gate = control["gate"] + assert gate["kind"] == "recovery" and gate["iteration_path"] == [{"loop_id": "repeat", "iteration": 0}] + with pytest.raises(WorkflowRuntimeConflict): + continue_repeat(store, "not-a-recovery") + store.decide(expected_version=control["version"], gate_id=gate["id"], + choice="retry", actor_user_id="owner", request_id="recover-effect") + flow, _ = execute_repeat(workflow, store, result_for_task=effect, replay_safe=False) + assert flow.finished and len(effects) == 2 and effects[0] == effects[1] + assert repeat_head(workflow, store)["batch_usage"] == 1 + assert store.read()["admitted_count"] == 5 + + +@pytest.mark.parametrize("budget", [3, 4]) +def test_global_admission_limit_cannot_be_extended_by_any_repeat_decision(monkeypatch, budget): + definition = repeat_definition(1) + definition["limits"]["max_executions"] = budget + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2) + if budget == 4: + assert store.read()["gate"]["reason_code"] == "repeat_iteration_limit" + with pytest.raises(WorkflowRuntimeConflict) as error: + continue_repeat(store) + assert error.value.code == "execution_budget_exceeded" + control = store.read() + assert control["admitted_count"] == budget and control["gate"]["choices"] == ["cancel"] + with pytest.raises(WorkflowRuntimeConflict): + store.resume(expected_version=control["version"], actor_user_id="owner", request_id="reset") + assert repeat_head(workflow, store)["continuation_count"] == 0 + + +def test_human_wait_consumes_frozen_deadline_and_admin_snapshot_is_retained(monkeypatch): + workflow, store, _, clock, _ = repeat_runtime(monkeypatch, maximum=2) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=3) + before = store.read() + # Initialization replay must not replace an active run's admitted policy. + unchanged = store.initialize( + snapshot_ref=before["snapshot_ref"], definition_revision=before["definition_revision"], + actor_user_id="owner", request_id=before["request_id"], repeat_policy={"max_iterations": 1}, + ) + assert unchanged["repeat_policy"] == before["repeat_policy"] + clock.now += timedelta(seconds=86400) + with pytest.raises(WorkflowRuntimeConflict) as error: + continue_repeat(store) + assert error.value.code == "deadline_exceeded" + assert store.read()["gate"]["reason_code"] == "deadline_exceeded" + assert store.read()["deadline_at"] == before["deadline_at"] and repeat_head(workflow, store)["continuation_count"] == 0 + + +@pytest.mark.parametrize("race", ["cancel", "tombstone", "lease"]) +def test_transition_commit_is_fenced_after_preparing_next_state(monkeypatch, race): + workflow, store, container, clock, _ = repeat_runtime(monkeypatch) + original = store.journal_commit_many + stopped = [] + + def lose_ownership(token, entries, **options): + if entries[0]["key"][0] == "repeat-transition" and not stopped: + stopped.append(True) + if race == "cancel": + store.request_cancel(actor_user_id="owner", request_id="cancel-round") + elif race == "tombstone": + store.tombstone() + else: + clock.advance() + store.claim(owner_id="replacement-worker") + return original(token, entries, **options) + + monkeypatch.setattr(store, "journal_commit_many", lose_ownership) + with pytest.raises(WorkflowRuntimeConflict): + execute_repeat(workflow, store) + assert stopped + transitions = [row for row in container.items.values() + if row.get("record_kind") == "decision" and row.get("key", [None])[0] == "repeat-transition"] + assert transitions == [] + heads = [row["payload"] for row in container.items.values() if row.get("record_kind") == "loop"] + assert heads[0]["completed_count"] == 0 + if race == "tombstone": + WorkflowResultStore(container).delete_run_results(workflow, "run") + assert len(container.items) == 1 and next(iter(container.items.values()))["deleted"] + + +def test_live_state_authority_is_rechecked_before_grants_and_unfinished_history(monkeypatch): + definition = repeat_definition(1) + definition["tasks"][1]["approval"] = {"required": True, "message": "Review this exact fictional round."} + workflow, store, container, _, _ = repeat_runtime(monkeypatch, definition=definition) + allowed = {"value": True} + source = {"document_id": "fictional-source", "scope": "personal", "scope_id": "owner", "source_version": 1} + + def authorize(user, sources, **options): + if not allowed["value"]: + raise AnalysisResultUnavailable("analysis_source_access_revoked") + return {"sources": sources, "source_count": len(sources), "source_snapshot_changed": False} + + monkeypatch.setattr("functions_workflow_node_results.authorize_analysis_sources", authorize) + + def provenance(current, envelope): + if current["id"] == "source": + envelope["analysis_access"] = {"version": "analysis-source-access-v1", "sources": [source]} + + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, envelope_transform=provenance) + repeat_id = repeat_head(workflow, store)["execution_id"] + assert store.journal_read("execution", repeat_id)["payload"].get("workflow_result") is None + allowed["value"] = False + with pytest.raises(AnalysisResultUnavailable): + workflow_execution_history(workflow, "run", reader_user_id="owner") + with pytest.raises(AnalysisResultUnavailable): + workflow_repeat_state_page(workflow, "run", repeat_id, 0, reader_user_id="owner") + from functions_workflow_results import authorize_workflow_run_read + + monkeypatch.setitem(sys.modules, "config", SimpleNamespace( + cosmos_personal_workflow_run_items_container=container, cosmos_group_workflow_run_items_container=container, + )) + monkeypatch.setattr("functions_workflow_runtime_store.workflow_runtime_store", lambda *args: store) + with pytest.raises(AnalysisResultUnavailable): + authorize_workflow_run_read(workflow, "run", reader_user_id="owner") + allowed["value"] = True + approval = store.read() + store.decide(expected_version=approval["version"], gate_id=approval["gate"]["id"], + choice="approve", actor_user_id="owner", request_id="approved-round") + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, envelope_transform=provenance) + allowed["value"] = False + before = store.read()["repeat_counts"] + with pytest.raises(AnalysisResultUnavailable): + continue_repeat(store) + assert store.read()["repeat_counts"] == before + + +@pytest.mark.parametrize("backend", ["cosmos", "blob"]) +def test_exact_state_history_uses_existing_storage_and_cleanup(monkeypatch, backend): + workflow, store, container, _, _ = repeat_runtime(monkeypatch) + blobs = FakeBlobService() if backend == "blob" else None + results = WorkflowResultStore(container, blobs, "private-results" if blobs else None) + monkeypatch.setattr("functions_workflow_result_store._configured_store", lambda *args, **kwargs: results) + monkeypatch.setattr("functions_workflow_result_store._configured_result_store", lambda *args, **kwargs: results) + flow, _ = execute_repeat(workflow, store, target=3) + identity = flow.final_outputs[0]["producer"] + page = workflow_repeat_iterations_page(workflow, "run", identity["execution_id"], reader_user_id="owner", limit=2) + assert [item["iteration"] for item in page["iterations"]] == [0, 1] and page["next_cursor"] + last = workflow_repeat_iterations_page( + workflow, "run", identity["execution_id"], reader_user_id="owner", limit=2, cursor=page["next_cursor"], + ) + assert [item["iteration"] for item in last["iterations"]] == [2] + before = workflow_repeat_state_page(workflow, "run", identity["execution_id"], 1, reader_user_id="owner") + after = workflow_repeat_state_page(workflow, "run", identity["execution_id"], 1, reader_user_id="owner", phase="after") + assert before["available"] and after["available"] + assert before["states"][0]["source"]["iteration_path"][-1]["iteration"] == 0 + assert after["states"][0]["source"]["iteration_path"][-1]["iteration"] == 1 + encoded = json.dumps(after) + assert "state_ref" not in encoded and "result_ref" not in encoded and "count" not in after["states"][0] + decisions = workflow_execution_history(workflow, "run", reader_user_id="owner", kind="decision") + assert len([row for row in decisions["decisions"] if row.get("decision_kind") == "repeat_transition"]) == 3 + with pytest.raises(ValueError): + workflow_repeat_state_page( + workflow, "run", identity["execution_id"], 1, reader_user_id="owner", cursor=page["next_cursor"], + ) + payload, _ = load_workflow_node_input( + workflow, "run", identity, flow.final_outputs[0]["result_ref"], output_name="state", + ) + assert json.loads(payload)["value"] == {"count": 3, "ready": True} + store.tombstone() + results.delete_run_results(workflow, "run") + assert len(container.items) == 1 + if blobs: + assert blobs.records == {} + + +def test_pending_after_state_is_explicitly_unavailable(monkeypatch): + workflow, store, _, _, _ = repeat_runtime(monkeypatch) + with pytest.raises(SystemExit): + execute_repeat(workflow, store, interrupt_at=0) + repeat_id = repeat_head(workflow, store)["execution_id"] + before = workflow_repeat_state_page(workflow, "run", repeat_id, 0, reader_user_id="owner") + after = workflow_repeat_state_page(workflow, "run", repeat_id, 0, reader_user_id="owner", phase="after") + assert before["available"] and len(before["states"]) == 1 + assert after["available"] is False and after["states"] == [] and after["next_cursor"] is None + + +def test_batch_grant_does_not_approve_the_next_body_attempt(monkeypatch): + definition = repeat_definition(1) + definition["tasks"][1]["approval"] = {"required": True, "message": "Approve this exact fictional round."} + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + calls = [] + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, calls=calls) + first = copy.deepcopy(store.read()) + store.decide(expected_version=first["version"], gate_id=first["gate"]["id"], + choice="approve", actor_user_id="owner", request_id="round-zero-approved") + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, calls=calls) + continue_repeat(store) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, calls=calls) + second = store.read() + assert second["gate"]["kind"] == "approval" + assert second["gate"]["iteration_path"][-1]["iteration"] == 1 + assert second["gate"]["id"] != first["gate"]["id"] + assert len([call for call in calls if call[0] == "body"]) == 1 + store.decide(expected_version=second["version"], gate_id=second["gate"]["id"], + choice="approve", actor_user_id="owner", request_id="round-one-approved") + flow, _ = execute_repeat(workflow, store, target=2, calls=calls) + assert flow.finished and repeat_head(workflow, store)["continuation_count"] == 1 + + +@pytest.mark.parametrize("use_decision", [False, True]) +def test_cancelled_exhaustion_retains_rounds_without_a_continuation_grant(monkeypatch, use_decision): + workflow, store, _, _, _ = repeat_runtime(monkeypatch, maximum=1) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2) + control = store.read() + if use_decision: + store.decide(expected_version=control["version"], gate_id=control["gate"]["id"], + choice="cancel", actor_user_id="owner", request_id="cancel-exhaustion") + else: + store.request_cancel(actor_user_id="owner", request_id="cancel-run") + head = repeat_head(workflow, store) + assert head["state"] == "cancelled" and head["completed_count"] == 1 + assert head["continuation_count"] == 0 and store.read()["repeat_progress"]["state"] == "cancelled" + assert store.journal_read("attempt", [head["execution_id"], 1])["payload"]["state"] == "cancelled" + + +@pytest.mark.parametrize("stage", ["approval", "invalid_state"]) +def test_cancelled_body_gate_marks_its_admitted_round_and_repeat(monkeypatch, stage): + definition = repeat_definition() + if stage == "approval": + definition["tasks"][1]["approval"] = {"required": True, "message": "Approve this fictional round."} + else: + slot = definition["flow"]["nodes"][1]["state"][0] + slot["output_contract"] = copy.deepcopy(slot["output_contract"]) + slot["output_contract"]["schema"]["properties"]["count"]["maximum"] = 0 + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + + def produce(current, resolved, execution): + return {"reply": "", "authoritative_result": {"kind": "json", "value": { + "count": int(current["id"] == "body"), "ready": False, + }}} + + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, result_for_task=produce) + control = store.read() + store.decide(expected_version=control["version"], gate_id=control["gate"]["id"], + choice="reject" if stage == "approval" else "cancel", + actor_user_id="owner", request_id="cancel-body-gate") + head = repeat_head(workflow, store) + iteration = store.journal_read("iteration", [head["execution_id"], 0])["payload"] + assert store.read()["state"] == head["state"] == iteration["state"] == "cancelled" + assert head["completed_count"] == 0 and head["continuation_count"] == 0 + + +@pytest.mark.parametrize("scenario", [ + "queued_repeat", "queued_without_repeat", "running_without_repeat", "running_for_each", +]) +def test_cancellation_without_repeat_path_does_not_load_frozen_definition(monkeypatch, scenario): + if scenario == "running_for_each": + workflow, store, container, _ = loop_runtime(monkeypatch) + + def interrupt_body(*args, **kwargs): + raise SystemExit("Closed For each worker interruption.") + + with pytest.raises(SystemExit, match="For each worker interruption"): + execute_loop(workflow, store, [{"fictional": True}], result_for_item=interrupt_body) + assert store.read()["cursor"]["iteration_path"] + else: + definition = repeat_definition() + if scenario != "queued_repeat": + definition["tasks"] = definition["tasks"][:1] + definition["flow"]["nodes"] = definition["flow"]["nodes"][:1] + definition["flow"]["outputs"][0]["source"].update(node_id="source-node", output="json") + workflow, store, container, _, _ = repeat_runtime(monkeypatch, definition=definition) + if scenario == "running_without_repeat": + execute_repeat(workflow, store) + assert not any("iteration" in frame for frame in (store.read().get("cursor") or {}).get("iteration_path", [])) + loop_rows = copy.deepcopy([row for row in container.items.values() if row.get("record_kind") == "loop"]) + admitted = store.read()["admitted_count"] + + def unexpected_snapshot(): + raise AssertionError("Cancellation without a Repeat path must not read a definition snapshot.") + + monkeypatch.setattr(store, "run_definition", unexpected_snapshot) + cancelled = store.request_cancel(actor_user_id="owner", request_id=f"cancel-{scenario}") + assert cancelled["state"] == "cancelled" and cancelled["admitted_count"] == admitted + assert [row for row in container.items.values() if row.get("record_kind") == "loop"] == loop_rows + + +def test_cached_state_proof_never_caches_current_access_to_a_source(monkeypatch): + workflow, store, _, _, _ = repeat_runtime(monkeypatch) + allowed = {"value": True} + original = store.journal_commit + calls = [] + source = {"document_id": "fictional-source", "scope": "personal", "scope_id": "owner", "source_version": 1} + + def authorize(user, sources, **options): + if not allowed["value"]: + raise AnalysisResultUnavailable("analysis_source_access_revoked") + return {"source_count": len(sources), "sources": sources, "source_snapshot_changed": False} + + def revoke_before_model(token, kind, key, payload, **options): + row = original(token, kind, key, payload, **options) + if kind == "unit" and key[-1] == "task:body" and payload["state"] == "running": + allowed["value"] = False + return row + + def provenance(current, envelope): + if current["id"] == "source": + envelope["analysis_access"] = {"version": "analysis-source-access-v1", "sources": [source]} + + monkeypatch.setattr("functions_workflow_node_results.authorize_analysis_sources", authorize) + monkeypatch.setattr(store, "journal_commit", revoke_before_model) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, calls=calls, envelope_transform=provenance) + assert [call[0] for call in calls] == ["source"] + assert repeat_head(workflow, store)["completed_count"] == 0 + assert store.read()["gate"]["choices"] == ["cancel"] + + +def test_history_cursors_bind_phase_round_and_admitted_snapshot(monkeypatch): + definition = repeat_definition(2) + second_slot = copy.deepcopy(definition["flow"]["nodes"][1]["state"][0]) + second_slot["name"] = "copy" + definition["flow"]["nodes"][1]["state"].append(second_slot) + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=3) + repeat_id = repeat_head(workflow, store)["execution_id"] + rounds = workflow_repeat_iterations_page(workflow, "run", repeat_id, reader_user_id="owner", limit=1) + first = workflow_repeat_state_page(workflow, "run", repeat_id, 1, reader_user_id="owner", limit=1) + second = workflow_repeat_state_page( + workflow, "run", repeat_id, 1, reader_user_id="owner", limit=1, cursor=first["next_cursor"], + ) + assert [row["name"] for row in first["states"] + second["states"]] == ["state", "copy"] + for iteration, phase in ((0, "before"), (1, "after")): + with pytest.raises(ValueError): + workflow_repeat_state_page( + workflow, "run", repeat_id, iteration, reader_user_id="owner", phase=phase, cursor=first["next_cursor"], + ) + continue_repeat(store) + execute_repeat(workflow, store, target=3) + historical = workflow_repeat_iterations_page( + workflow, "run", repeat_id, reader_user_id="owner", cursor=rounds["next_cursor"], + ) + assert historical["total_count"] == 2 and [row["iteration"] for row in historical["iterations"]] == [1] + history = workflow_execution_history(workflow, "run", reader_user_id="owner") + encoded = json.dumps(history) + assert '"state_ref"' not in encoded and '"repeat_state"' not in encoded + + +def test_monitoring_events_contain_only_stable_sanitized_correlation(monkeypatch): + from functions_workflow_repeat_state import log_repeat_event + + captured = [] + monkeypatch.setitem(sys.modules, "functions_appinsights", SimpleNamespace( + log_event=lambda message, **options: captured.append((message, options["extra"])), + )) + store = SimpleNamespace(identity={"workflow_id": "fictional-workflow", "run_id": "fictional-run", + "scope_type": "personal", "scope_id": "fictional-owner"}) + decision = { + "choice": "continue_repeat", "event_id": "a" * 64, "gate_id": "gate", "request_id": "request", + "execution_id": "b" * 64, "node_id": "repeat", "actor_user_id": "fictional-owner", + "decided_at": "2026-09-17T00:00:00+00:00", "prompt": "PRIVATE-SENTINEL", + "repeat": {"batch_number": 1, "batch_size": 25, "completed_count": 25, + "exhaustion_count": 1, "continuation_count": 1, "state_ref": "PRIVATE-SENTINEL"}, + } + log_repeat_event(store, decision) + log_repeat_event(store, decision) + assert captured[0] == captured[1] + assert captured[0][1]["event_name"] == "workflow_repeat_manually_continued" + assert "PRIVATE-SENTINEL" not in json.dumps(captured) + + +def test_deadline_is_rechecked_after_continuation_source_authorization(monkeypatch): + from functions_workflow_repeat_state import prepare_repeat_grant + + workflow, store, _, clock, _ = repeat_runtime(monkeypatch, maximum=1) + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2) + + def authorize_then_expire(*args, **kwargs): + prepared = prepare_repeat_grant(*args, **kwargs) + clock.now += timedelta(seconds=86400) + return prepared + + monkeypatch.setattr("functions_workflow_repeat_state.prepare_repeat_grant", authorize_then_expire) + with pytest.raises(WorkflowRuntimeConflict) as error: + continue_repeat(store) + assert error.value.code == "deadline_exceeded" + assert repeat_head(workflow, store)["continuation_count"] == 0 + assert store.read()["gate"]["choices"] == ["cancel"] diff --git a/functional_tests/test_workflow_repeat_schema.py b/functional_tests/test_workflow_repeat_schema.py new file mode 100644 index 000000000..bb35d5faa --- /dev/null +++ b/functional_tests/test_workflow_repeat_schema.py @@ -0,0 +1,456 @@ +# test_workflow_repeat_schema.py +""" +Functional tests for typed Repeat state, lexical visibility, and mixed identities. +Version: 0.261.120 +Implemented in: 0.261.120 + +These tests exercise the production compiler and identity helpers without clients, +model calls, or an alternate execution engine. +""" + +import copy +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "application" / "single_app")) + +# Production imports follow the isolated repository path setup. +from functions_workflow_definitions import WorkflowDefinitionError +from functions_workflow_bindings import WorkflowInputError +from functions_workflow_execution import workflow_execution_scope +from functions_workflow_flow import compile_workflow_flow, normalize_flow_bindings +from functions_workflow_identity import ( + normalize_workflow_iteration_path, workflow_execution_id, workflow_node_identity, +) +from functions_workflow_loop_runners import ( + assert_workflow_loop_agent_type, validate_workflow_loop_runners, +) + + +DECISION_SCHEMA = { + "type": "object", "required": ["ready"], + "properties": {"ready": {"type": "boolean"}, "round": {"type": "integer"}}, +} + + +def node_binding(node_id, name, output="json", kind="json", **options): + return { + "name": name, "source": { + "kind": "node_output", "node_id": node_id, "output": output, "scope": "current", + }, "required": True, "expected_kind": kind, "allow_partial": False, **options, + } + + +def state_binding(loop_id="repeat", state_name="decision", *, name="state", kind="json"): + return { + "name": name, "source": { + "kind": "repeat_state", "loop_id": loop_id, "state_name": state_name, "scope": "current", + }, "required": True, "expected_kind": kind, "allow_partial": False, + } + + +def instruction_task(identifier, contract, inputs=None): + return { + "id": identifier, "name": identifier, "type": "instructions", "instructions": "Use declared saved inputs.", + "runner": {"type": "inherit"}, "document_action": {"type": "none"}, + "inputs": inputs or [], "output_contract": copy.deepcopy(contract), + } + + +def repeat_definition(maximum=25): + contract = {"kind": "json", "schema": DECISION_SCHEMA, "allow_partial": False} + return { + "id": "workflow", "user_id": "owner", "definition_version": 3, "durable_execution": True, + "runner_type": "model", "chat_capabilities_enabled": False, + "limits": {"max_executions": 5000, "deadline_seconds": 86400}, + "tasks": [ + instruction_task("seed", contract), + instruction_task("body", contract, [state_binding()]), + ], + "flow": {"id": "root", "nodes": [ + {"id": "seed-node", "kind": "task", "task_id": "seed"}, + { + "id": "repeat", "kind": "repeat_until", "max_iterations": maximum, + "state": [{ + "name": "decision", "initial": node_binding("seed-node", "seed")["source"], + "next": "decision_after", "output_contract": copy.deepcopy(contract), + }], + "body": { + "id": "repeat-body", "nodes": [{"id": "body-node", "kind": "task", "task_id": "body"}], + "outputs": [node_binding("body-node", "decision_after")], + }, + "until": { + "op": "eq", "left": {"input": "decision", "path": "/ready"}, "right": {"literal": True}, + }, + "exports": [{"name": "decision", "output": "decision_after"}], + }, + ], "outputs": [node_binding("repeat", "final", "decision")]}, + } + + +def add_data_state(workflow, kind): + schema = {"type": "string"} if kind == "text" else { + "type": "object", + } if kind == "json" else {"type": "array", "items": {"type": "object"}} + selector = "documents" if kind == "document_results" else kind + contract = {"kind": kind, "schema": schema, "allow_partial": False} + workflow["tasks"].extend([ + instruction_task("data-seed", contract), + instruction_task("data-body", contract, [state_binding(state_name="data", kind=kind)]), + ]) + workflow["flow"]["nodes"].insert(1, {"id": "data-seed-node", "kind": "task", "task_id": "data-seed"}) + repeat = workflow["flow"]["nodes"][-1] + repeat["state"].append({ + "name": "data", "initial": node_binding("data-seed-node", "data", selector, kind)["source"], + "next": "data_after", "output_contract": contract, + }) + repeat["body"]["nodes"].insert(0, {"id": "data-body-node", "kind": "task", "task_id": "data-body"}) + repeat["body"]["outputs"].append(node_binding("data-body-node", "data_after", selector, kind)) + repeat["exports"].append({"name": "data", "output": "data_after"}) + return repeat + + +def test_repeat_normalization_is_explicit_and_idempotent(): + workflow = repeat_definition() + compiled = compile_workflow_flow(workflow) + repeat = compiled["nodes"]["repeat"]["node"] + assert repeat["max_iterations"] == 25 + assert repeat["state"][0]["output_contract"]["allow_partial"] is False + assert compiled["node_loop_ids"]["body-node"] == ["repeat"] + assert compiled["node_loop_ids"]["repeat"] == [] + normalized = {**workflow, **{key: compiled[key] for key in ("flow", "tasks", "limits")}} + assert compile_workflow_flow(normalized)["flow"] == compiled["flow"] + assert workflow["flow"]["nodes"][1]["state"][0]["initial"]["node_id"] == "seed-node" + + +@pytest.mark.parametrize("maximum", [1, 25, 500, 1000]) +def test_explicit_repeat_batch_maximum_is_independent_of_for_each(maximum): + assert compile_workflow_flow(repeat_definition(maximum))["nodes"]["repeat"]["node"]["max_iterations"] == maximum + + +@pytest.mark.parametrize("maximum", [None, 0, -1, True, False, 25.0, "25", 1001, 5000]) +def test_invalid_or_omitted_authored_repeat_maximum_is_rejected(maximum): + workflow = repeat_definition(maximum) + if maximum is None: + workflow["flow"]["nodes"][1].pop("max_iterations") + with pytest.raises(WorkflowDefinitionError, match="max_iterations"): + compile_workflow_flow(workflow) + + +@pytest.mark.parametrize("kind", ["text", "json", "records", "document_results"]) +def test_typed_state_and_final_exports_preserve_original_kind(kind): + workflow = repeat_definition() + add_data_state(workflow, kind) + workflow["flow"]["outputs"].append(node_binding("repeat", "data", "data", kind)) + compiled = compile_workflow_flow(workflow) + repeat = compiled["nodes"]["repeat"]["node"] + assert repeat["state"][1]["output_contract"]["kind"] == kind + assert ("repeat", "data") in compiled["definite_outputs"] + + +@pytest.mark.parametrize("mutation", [ + lambda node: node.update(state=[]), + lambda node: node["state"].append(copy.deepcopy(node["state"][0])), + lambda node: node["state"][0]["output_contract"].update(kind="any"), + lambda node: node["state"][0].update(initial={"kind": "literal", "value": {"ready": False}}), + lambda node: node["state"][0]["initial"].update(result_ref={"sha256": "a" * 64}), + lambda node: node["state"][0].update(initial=state_binding()["source"]), + lambda node: node["state"][0].update(next="missing"), + lambda node: node["body"]["outputs"][0].update(required=False), + lambda node: node["body"]["outputs"][0].update(allow_partial=True), + lambda node: node["exports"][0].update(output="missing"), + lambda node: node["exports"].append(copy.deepcopy(node["exports"][0])), + lambda node: node["until"]["left"].update(path="/undeclared"), + lambda node: node["until"]["left"].update(input="body-node"), + lambda node: node.update(exhaustion="complete"), +]) +def test_repeat_rejects_ambiguous_unsafe_or_unapproved_shapes(mutation): + workflow = repeat_definition() + mutation(workflow["flow"]["nodes"][1]) + with pytest.raises(WorkflowDefinitionError): + compile_workflow_flow(workflow) + + +def test_initial_and_next_state_must_have_the_exact_declared_kind(): + workflow = repeat_definition() + node = add_data_state(workflow, "records") + node["state"][1]["output_contract"] = {"kind": "json", "schema": {"type": "array"}} + with pytest.raises(WorkflowDefinitionError, match="exact declared"): + compile_workflow_flow(workflow) + + +def test_explicit_partial_state_requires_both_slot_and_body_acceptance(): + workflow = repeat_definition() + repeat = workflow["flow"]["nodes"][1] + repeat["state"][0]["output_contract"]["allow_partial"] = True + repeat["body"]["outputs"][0]["allow_partial"] = True + workflow["tasks"][1]["output_contract"]["allow_partial"] = True + workflow["tasks"][1]["inputs"][0]["allow_partial"] = True + assert compile_workflow_flow(workflow)["nodes"]["repeat"]["node"]["state"][0]["output_contract"]["allow_partial"] + + +def test_body_can_explicitly_retain_state_without_rewriting_originals(): + workflow = repeat_definition() + repeat = workflow["flow"]["nodes"][1] + repeat["body"]["outputs"][0]["source"] = state_binding()["source"] + compiled = compile_workflow_flow(workflow) + assert compiled["nodes"]["repeat"]["node"]["body"]["outputs"][0]["source"]["kind"] == "repeat_state" + + +def test_body_outputs_cannot_escape_repeat_without_declared_boundary_export(): + workflow = repeat_definition() + workflow["flow"]["outputs"] = [node_binding("body-node", "escaped")] + with pytest.raises(WorkflowDefinitionError, match="out-of-region"): + compile_workflow_flow(workflow) + + +def test_repeat_body_export_cannot_silently_reselect_ancestor_output(): + workflow = repeat_definition() + workflow["flow"]["nodes"][1]["body"]["outputs"][0]["source"]["node_id"] = "seed-node" + with pytest.raises(WorkflowDefinitionError, match="own body scope"): + compile_workflow_flow(workflow) + + +def test_a_conditional_next_state_requires_an_explicit_join(): + workflow = repeat_definition() + workflow["flow"]["nodes"][1]["body"]["nodes"][0]["run_when"] = { + "op": "eq", "left": {"input": "state", "path": "/ready"}, "right": {"literal": False}, + } + with pytest.raises(WorkflowDefinitionError, match="required producer"): + compile_workflow_flow(workflow) + + +def test_repeat_state_is_not_available_outside_its_body(): + workflow = repeat_definition() + workflow["tasks"][0]["inputs"] = [state_binding()] + with pytest.raises(WorkflowDefinitionError, match="enclosing Repeat"): + compile_workflow_flow(workflow) + + +def test_repeat_is_not_a_fabricated_document_item(): + workflow = repeat_definition() + workflow["tasks"][1]["inputs"] = [{ + **state_binding(), "source": {"kind": "loop_item", "loop_id": "repeat", "scope": "current"}, + }] + with pytest.raises(WorkflowDefinitionError, match="enclosing For each"): + compile_workflow_flow(workflow) + + +def test_for_each_can_read_an_exact_nonpartial_repeat_collection(): + workflow = repeat_definition() + repeat = add_data_state(workflow, "records") + repeat["body"]["nodes"].append({ + "id": "each", "kind": "for_each", "max_items": 500, "item_key": "source_identity", + "inputs": [state_binding(state_name="data", name="rows", kind="records")], + "iterable": {"kind": "input", "name": "rows"}, + "body": {"id": "each-body", "nodes": [], "outputs": []}, + }) + compiled = compile_workflow_flow(workflow) + assert compiled["node_loop_ids"]["each"] == ["repeat"] + assert compiled["nodes"]["each"]["node"]["max_items"] == 500 + + +def test_saved_record_reporting_can_bind_repeat_collection_without_a_new_exporter(): + workflow = repeat_definition() + repeat = add_data_state(workflow, "records") + report = instruction_task("report", {"kind": "text"}, [state_binding(state_name="data", kind="records")]) + report["input_processing"] = "saved_record_report" + workflow["tasks"].append(report) + repeat["body"]["nodes"].append({"id": "report-node", "kind": "task", "task_id": "report"}) + assert compile_workflow_flow(workflow)["task_nodes"]["report"] == "report-node" + + +def test_saved_output_publication_accepts_repeat_records_only_through_final_node_export(): + workflow = repeat_definition() + add_data_state(workflow, "records") + publish = instruction_task("publish", {"kind": "text"}, [node_binding("repeat", "rows", "data", "records")]) + publish["publication"] = {"source_kind": "saved_output", "artifact_format": "json"} + workflow["tasks"].append(publish) + workflow["flow"]["nodes"].append({"id": "publish-node", "kind": "task", "task_id": "publish"}) + assert compile_workflow_flow(workflow)["task_nodes"]["publish"] == "publish-node" + publish["inputs"][0]["source"] = state_binding(state_name="data")["source"] + with pytest.raises(WorkflowDefinitionError, match="enclosing Repeat"): + compile_workflow_flow(workflow) + + +def test_repeat_export_cycle_is_rejected_without_recursive_overflow(): + workflow = repeat_definition() + workflow["flow"]["nodes"][1]["body"]["outputs"][0]["source"] = node_binding("repeat", "cycle", "decision")["source"] + with pytest.raises(WorkflowDefinitionError): + compile_workflow_flow(workflow) + + +def test_lifetime_repeat_index_changes_execution_but_retry_changes_only_attempt(): + workflow = repeat_definition() + first = [{"loop_id": "repeat", "iteration": 0}] + second = [{"loop_id": "repeat", "iteration": 1}] + identifier = workflow_execution_id(workflow, "run", "body-node", first) + assert identifier != workflow_execution_id(workflow, "run", "body-node", second) + identities = [ + workflow_node_identity( + workflow, "run", "body-node", identifier, attempt, task_id="body", iteration_path=first, + ) for attempt in (1, 2) + ] + assert identities[0]["execution_id"] == identities[1]["execution_id"] + assert identities[0]["iteration_path"] == identities[1]["iteration_path"] == first + assert identities[0]["attempt"] != identities[1]["attempt"] + assert workflow_execution_id(workflow, "run", "body-node", [{"loop_id": "repeat", "iteration": 1000}]) + + +@pytest.mark.parametrize("path", [ + [{"loop_id": "repeat", "iteration": -1}], + [{"loop_id": "repeat", "iteration": True}], + [{"loop_id": "repeat", "iteration": 5000}], + [{"loop_id": "repeat", "iteration": 0, "batch": 1}], + [{"loop_id": "repeat", "iteration": 0, "item_id": "a" * 64, "index": 0}], + [{"loop_id": "repeat", "iteration": 0}, {"loop_id": "repeat", "iteration": 1}], +]) +def test_malformed_repeat_frames_are_rejected(path): + with pytest.raises(ValueError): + normalize_workflow_iteration_path(path) + + +def test_definition_ancestry_and_frame_kind_are_part_of_identity(): + workflow = repeat_definition() + with pytest.raises(ValueError, match="For-each|Repeat"): + workflow_execution_id(workflow, "run", "body-node", [{"loop_id": "repeat", "index": 0, "item_id": "a" * 64}]) + with pytest.raises(ValueError, match="ancestors"): + workflow_execution_id(workflow, "run", "body-node", []) + with pytest.raises(ValueError, match="ancestors"): + workflow_execution_id(workflow, "run", "repeat", [{"loop_id": "repeat", "iteration": 0}]) + workflow["limits"]["max_executions"] = 10 + with pytest.raises(ValueError, match="execution limit"): + workflow_execution_id(workflow, "run", "body-node", [{"loop_id": "repeat", "iteration": 10}]) + + +def test_mixed_for_each_repeat_path_preserves_for_each_frame(): + workflow = repeat_definition() + child = workflow["flow"] + child["id"] = "outer-body" + child["outputs"] = [] + workflow["flow"] = {"id": "root", "nodes": [{ + "id": "outer", "kind": "for_each", "max_items": 500, "item_key": "source_identity", "inputs": [], + "iterable": {"kind": "documents", "documents": []}, "body": child, + }], "outputs": []} + compile_workflow_flow(workflow) + item = {"loop_id": "outer", "item_id": "a" * 64, "index": 3} + path = [item, {"loop_id": "repeat", "iteration": 1001}] + assert normalize_workflow_iteration_path(path) == path + assert workflow_execution_id(workflow, "run", "body-node", path) + assert item == {"loop_id": "outer", "item_id": "a" * 64, "index": 3} + workflow["tasks"].append(instruction_task("deep", {"kind": "text"}, [state_binding()])) + inner = { + "id": "inner", "kind": "for_each", "max_items": 10, "item_key": "source_identity", "inputs": [], + "iterable": {"kind": "documents", "documents": []}, + "body": { + "id": "inner-body", "nodes": [{"id": "deep-node", "kind": "task", "task_id": "deep"}], + "outputs": [], + }, + } + child["nodes"][1]["body"]["nodes"].insert(0, inner) + compiled = compile_workflow_flow(workflow) + assert compiled["node_loop_ids"]["deep-node"] == ["outer", "repeat", "inner"] + mixed = [*path, {"loop_id": "inner", "item_id": "b" * 64, "index": 2}] + assert workflow_execution_id(workflow, "run", "deep-node", mixed) + with pytest.raises(ValueError): + normalize_workflow_iteration_path([*mixed, {"loop_id": "extra", "iteration": 0}]) + inner["body"]["nodes"] = [{ + "id": "extra", "kind": "for_each", "max_items": 1, "item_key": "source_identity", "inputs": [], + "iterable": {"kind": "documents", "documents": []}, + "body": {"id": "extra-body", "nodes": inner["body"]["nodes"], "outputs": []}, + }] + with pytest.raises(WorkflowDefinitionError): + compile_workflow_flow(workflow) + + +def test_repeat_state_source_does_not_accept_caller_selected_execution_or_run(): + binding = state_binding() + binding["source"]["run_id"] = "foreign-run" + with pytest.raises(WorkflowDefinitionError, match="unsupported fields"): + normalize_flow_bindings([binding]) + + +def test_nested_repeat_uses_outer_state_without_mutating_its_scope(): + workflow = repeat_definition() + outer = workflow["flow"]["nodes"][1] + inner = copy.deepcopy(outer) + inner["id"] = "inner" + inner["state"][0]["initial"] = state_binding()["source"] + inner["body"] = { + "id": "inner-body", "nodes": [{"id": "inner-node", "kind": "task", "task_id": "inner-task"}], + "outputs": [node_binding("inner-node", "decision_after")], + } + workflow["tasks"].append(instruction_task( + "inner-task", {"kind": "json", "schema": DECISION_SCHEMA}, [state_binding(loop_id="inner")], + )) + outer["body"]["nodes"].append(inner) + outer["body"]["outputs"][0]["source"] = node_binding("inner", "next", "decision")["source"] + compiled = compile_workflow_flow(workflow) + assert compiled["node_loop_ids"]["inner-node"] == ["repeat", "inner"] + path = [{"loop_id": "repeat", "iteration": 1000}, {"loop_id": "inner", "iteration": 25}] + assert workflow_execution_id(workflow, "run", "inner-node", path) + + +def test_legacy_for_each_execution_hash_is_unchanged(): + # Captured with functions_workflow_identity.py from merged M4C-2 commit 80f88903. + workflow = { + "id": "workflow", "user_id": "owner", "definition_version": 3, "durable_execution": True, + "tasks": [{ + "id": "body", "instructions": "Use declared saved inputs.", + "inputs": [], "output_contract": {"kind": "text"}, + }], + "flow": {"id": "root", "nodes": [{ + "id": "each", "kind": "for_each", "inputs": [], + "iterable": {"kind": "documents", "documents": [ + {"scope_type": "personal", "document_id": "document-1"}, + ]}, + "item_key": "source_identity", "max_items": 500, + "body": { + "id": "body-region", "nodes": [{"id": "body-node", "kind": "task", "task_id": "body"}], + "outputs": [], + }, + }], "outputs": []}, + } + compile_workflow_flow(workflow) + assert workflow_execution_id( + workflow, "run", "body-node", [{"loop_id": "each", "item_id": "a" * 64, "index": 0}], + ) == "7b723c1ff080e77ea7be1ae27570cb88841bc5dbeba00510edfc444267909785" + + +@pytest.mark.parametrize("runner_type", ["model", "local", "foundry", "hosted"]) +def test_repeat_admission_requires_locally_metered_body_runners(runner_type): + workflow = repeat_definition() + workflow["runner_type"] = "agent" + workflow["selected_agent"] = {"name": "hosted-seed"} + workflow["tasks"][1]["runner"] = { + "type": "model" if runner_type == "model" else "agent", + **({} if runner_type == "model" else {"selected_agent": {"name": runner_type}}), + } + checked = [] + + def resolve(selected, **kwargs): + assert kwargs["user_id"] == "owner" + checked.append(selected["name"]) + return {"agent_type": selected["name"]} + + if runner_type in {"foundry", "hosted"}: + with pytest.raises(WorkflowInputError, match="locally metered"): + validate_workflow_loop_runners(workflow, actor_user_id="owner", settings={}, resolve_agent=resolve) + else: + validate_workflow_loop_runners(workflow, actor_user_id="owner", settings={}, resolve_agent=resolve) + assert "hosted-seed" not in checked + assert checked == ([] if runner_type == "model" else [runner_type]) + + +def test_repeat_runtime_rechecks_agent_type_without_changing_ordinary_steps(): + workflow = repeat_definition() + execution = SimpleNamespace(workflow=workflow, node={"task_id": "body"}, iteration_path=[]) + with workflow_execution_scope(execution): + assert_workflow_loop_agent_type("foundry") + execution.iteration_path = [{"loop_id": "repeat", "iteration": 0}] + assert_workflow_loop_agent_type("local") + with pytest.raises(WorkflowInputError, match="locally metered"): + assert_workflow_loop_agent_type("foundry") diff --git a/functional_tests/test_workflow_repeat_state.py b/functional_tests/test_workflow_repeat_state.py new file mode 100644 index 000000000..3b0b2b3b3 --- /dev/null +++ b/functional_tests/test_workflow_repeat_state.py @@ -0,0 +1,314 @@ +# test_workflow_repeat_state.py +""" +Functional tests for typed Repeat state, mixed nesting and retained partial data. +Version: 0.261.120 +Implemented in: 0.261.120 + +All cases use the production serial runner and exact result transport with local +fictional records. Large data is never replaced by a preview or prefix. +""" + +import copy +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "application" / "single_app")) + +# Production imports follow the isolated worktree import setup. +from functions_workflow_execution import WorkflowSuspended +from functions_workflow_identity import workflow_execution_id +from functions_workflow_node_results import load_workflow_node_input, open_workflow_record_input +from functions_workflow_repeat_history import workflow_repeat_state_page +from functions_workflow_runtime_store import WorkflowRuntimeConflict +from functions_workflow_validation import validate_workflow_task_output +from test_workflow_repeat_execution import ( + continue_repeat, execute_repeat, repeat_definition, repeat_head, repeat_runtime, state_binding, +) +from test_workflow_structured_flow import binding, task + + +def result(kind, value): + return {"reply": "", "authoritative_result": {"kind": kind, "value": value}} + + +@pytest.mark.parametrize("kind", ["text", "json", "records", "document_results"]) +def test_state_preserves_exact_supported_kind_and_original_passthrough_receipt(monkeypatch, kind): + definition = repeat_definition() + value = ( + "Exact fictional text " + chr(0x3A9) if kind == "text" else + {"nested": {"ready": False, "zero": 0, "null": None}} if kind == "json" else + [{"same": [False, 0, None, chr(0x3A9)]}, {"same": [False, 0, None, chr(0x3A9)]}] + ) + if kind == "document_results": + value = [{"document_id": f"fictional-{index}", "kind": "json", "value": record} + for index, record in enumerate(value)] + contract = {"kind": kind} + if kind == "json": + contract["schema"] = {"type": "object", "required": ["nested"], "properties": {"nested": {"type": "object"}}} + if kind in {"records", "document_results"}: + contract["schema"] = {"type": "array", "items": {"type": "object"}} + definition["tasks"][0]["output_contract"] = copy.deepcopy(contract) + definition["tasks"][1]["inputs"] = [state_binding(kind=kind)] + definition["tasks"][1]["output_contract"] = {"kind": "text"} + loop = definition["flow"]["nodes"][1] + loop["state"][0]["output_contract"] = contract + loop["state"][0]["initial"]["output"] = {"document_results": "documents"}.get(kind, kind) + loop["body"]["outputs"] = [state_binding("next", kind=kind)] + loop["until"] = {"op": "eq", "left": {"literal": True}, "right": {"literal": True}} + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + + def produce(current, resolved, execution): + if current["id"] == "source": + if kind == "document_results": + return {"reply": "", "analysis_result": { + "per_document": True, + "document_results": [ + {"document_id": item["document_id"], "full_result": { + "authoritative_result": {"kind": item["kind"], "value": item["value"]}, + }} for item in value + ], + }} + return result(kind, value) + actual = resolved["values"]["state"] + assert (list(actual.iter_records()) if kind in {"records", "document_results"} else actual) == value + return {"reply": "The fictional body ran once."} + + flow, calls = execute_repeat(workflow, store, result_for_task=produce, stream_collections=True) + assert flow.finished and len(calls) == 2 + summary = store.journal_read("execution", workflow_execution_id(workflow, "run", "repeat"))["payload"]["workflow_result"] + assert summary["outputs"]["state"]["selected_producer"]["producer"]["node_id"] == "source-node" + payload, _ = load_workflow_node_input( + workflow, "run", flow.final_outputs[0]["producer"], flow.final_outputs[0]["result_ref"], output_name="state", + ) + assert json.loads(payload)["value"] == value + + +def test_partial_state_requires_opt_in_and_retains_limitations_after_valid_later_output(monkeypatch): + definition = repeat_definition(2) + for current in definition["tasks"]: + current["output_contract"]["allow_partial"] = True + loop = definition["flow"]["nodes"][1] + loop["state"][0]["output_contract"]["allow_partial"] = True + definition["tasks"][1]["inputs"][0]["allow_partial"] = True + definition["flow"]["outputs"][0]["allow_partial"] = True + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + + def partial_seed(current, envelope): + if current["id"] == "source": + envelope["coverage"] = {"status": "incomplete", "expected_count": 2, "processed_count": 1, "failed_count": 1} + envelope["workflow_validation"] = validate_workflow_task_output(envelope, current["output_contract"]) + + flow, _ = execute_repeat(workflow, store, target=2, envelope_transform=partial_seed) + assert flow.finished and flow.partial + head = repeat_head(workflow, store) + final = store.journal_read("execution", head["execution_id"])["payload"]["workflow_result"] + assert final["workflow_validation"]["status"] == "accepted_partial" + state = workflow_repeat_state_page( + workflow, "run", head["execution_id"], 1, reader_user_id="owner", phase="after", + ) + assert state["partial"] and state["states"][0]["workflow_validation"]["status"] == "accepted_partial" + assert "producer_coverage_incomplete" in state["states"][0]["limitations"] + + +def test_default_state_contract_rejects_partial_seed_without_running_body(monkeypatch): + definition = repeat_definition() + definition["tasks"][0]["output_contract"] = {**definition["tasks"][0]["output_contract"], "allow_partial": True} + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + calls = [] + + def partial(current, envelope): + envelope["coverage"] = {"status": "incomplete", "partial_coverage": True} + envelope["workflow_validation"] = validate_workflow_task_output(envelope, current["output_contract"]) + + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, calls=calls, envelope_transform=partial) + assert [call[0] for call in calls] == ["source"] + assert store.read()["gate"]["choices"] == ["cancel"] + with pytest.raises(WorkflowRuntimeConflict): + continue_repeat(store) + + +def test_partial_export_outside_state_survives_completed_repeat_replay(monkeypatch): + definition = repeat_definition() + definition["tasks"].append(task("report", contract={ + "kind": "json", "schema": {"type": "object"}, "allow_partial": True, + })) + loop = definition["flow"]["nodes"][1] + loop["body"]["nodes"].append({"id": "report-node", "kind": "task", "task_id": "report"}) + loop["body"]["outputs"].append({**binding("report-node", "report", "json"), "allow_partial": True}) + loop["exports"].append({"name": "report", "output": "report"}) + definition["flow"]["outputs"] = [] + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + + def produce(current, resolved, execution): + if current["id"] == "report": + return result("json", {"fictional": "retained incomplete report"}) + return result("json", {"count": int(current["id"] == "body"), "ready": current["id"] == "body"}) + + def partial_report(current, envelope): + if current["id"] == "report": + envelope["coverage"] = {"status": "incomplete", "partial_coverage": True} + envelope["workflow_validation"] = validate_workflow_task_output(envelope, current["output_contract"]) + + flow, _ = execute_repeat(workflow, store, result_for_task=produce, envelope_transform=partial_report) + assert flow.finished and flow.partial and repeat_head(workflow, store)["partial"] + head = repeat_head(workflow, store) + state = workflow_repeat_state_page(workflow, "run", head["execution_id"], 0, reader_user_id="owner", phase="after") + assert state["partial"] and state["states"][0]["workflow_validation"]["status"] == "valid" + replayed, calls = execute_repeat(workflow, store) + assert replayed.finished and replayed.partial and calls == [] + + +def test_invalid_next_state_is_not_a_false_condition_or_manual_override(monkeypatch): + workflow, store, _, _, _ = repeat_runtime(monkeypatch, maximum=1) + + def produce(current, resolved, execution): + return result("json", {"count": 0, "ready": False} if current["id"] == "source" + else {"count": "invalid fictional value", "ready": True}) + + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, result_for_task=produce) + head = repeat_head(workflow, store) + assert head["completed_count"] == 0 and head["exhaustion_count"] == 0 + assert store.read()["gate"]["choices"] == ["cancel"] + assert store.journal_read("decision", ["repeat-transition", head["execution_id"], 0]) is None + + +@pytest.mark.parametrize("third_frame", [False, True]) +def test_nested_repeat_seeds_from_enclosing_state_and_grant_is_instance_bound(monkeypatch, third_frame): + definition = repeat_definition() + outer = definition["flow"]["nodes"][1] + inner = copy.deepcopy(outer) + inner.update(id="inner", max_iterations=1) + inner["state"][0]["initial"] = state_binding(loop_id="repeat")["source"] + inner["body"]["id"] = "inner-body" + definition["tasks"][1]["inputs"] = [state_binding(loop_id="inner")] + outer["body"]["nodes"] = [inner] + outer["body"]["outputs"] = [binding("inner", "next", "state")] + outer["until"] = {"op": "eq", "left": {"input": "state", "path": "/count"}, "right": {"literal": 3}} + if third_frame: + contract = {"kind": "records", "schema": {"type": "array", "items": {"type": "object"}}} + definition["tasks"].extend([ + task("rows-source", contract=contract), + task("leaf", inputs=[{"name": "item", "source": {"kind": "loop_item", "loop_id": "third", "scope": "current"}}], + contract=contract), + ]) + definition["flow"]["nodes"].insert(1, {"id": "rows-source-node", "kind": "task", "task_id": "rows-source"}) + inner["body"]["nodes"].append({ + "id": "third", "kind": "for_each", "max_items": 1, "item_key": "source_identity", + "inputs": [binding("rows-source-node", "rows", "records")], "iterable": {"kind": "input", "name": "rows"}, + "body": {"id": "third-body", "nodes": [{"id": "leaf-node", "kind": "task", "task_id": "leaf"}], "outputs": []}, + }) + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + calls = [] + + def produce(current, resolved, execution): + if current["id"] == "source": + return result("json", {"count": 0, "ready": False}) + if current["id"] == "rows-source": + return result("records", [{"fictional": True}]) + if current["id"] == "leaf": + return result("records", [resolved["values"]["item"]["value"]]) + count = resolved["values"]["state"]["count"] + 1 + return result("json", {"count": count, "ready": count >= 2}) + + with pytest.raises(WorkflowSuspended): + execute_repeat(workflow, store, target=2, calls=calls, result_for_task=produce) + gate = store.read()["gate"] + assert gate["node_id"] == "inner" and gate["iteration_path"] == [{"loop_id": "repeat", "iteration": 0}] + continue_repeat(store) + flow, _ = execute_repeat(workflow, store, target=2, calls=calls, result_for_task=produce) + body = [call for call in calls if call[0] == "body"] + assert flow.finished and [call[1] for call in body] == [ + [{"loop_id": "repeat", "iteration": 0}, {"loop_id": "inner", "iteration": 0}], + [{"loop_id": "repeat", "iteration": 0}, {"loop_id": "inner", "iteration": 1}], + [{"loop_id": "repeat", "iteration": 1}, {"loop_id": "inner", "iteration": 0}], + ] + assert store.read()["repeat_counts"] == {"exhaustion_count": 1, "continuation_count": 1} + if third_frame: + leaves = [call for call in calls if call[0] == "leaf"] + assert len(leaves) == 3 and all(len(call[1]) == 3 for call in leaves) + assert all(set(call[1][-1]) == {"loop_id", "item_id", "index"} for call in leaves) + + +def test_for_each_inside_repeat_uses_exact_current_state_collection(monkeypatch): + definition = repeat_definition() + contract = {"kind": "records", "schema": {"type": "array", "items": {"type": "object"}}} + definition["tasks"].extend([ + task("rows-source", contract=contract), + task("leaf", inputs=[{"name": "item", "source": {"kind": "loop_item", "loop_id": "each", "scope": "current"}}], + contract=contract), + ]) + definition["flow"]["nodes"].insert(1, {"id": "rows-source-node", "kind": "task", "task_id": "rows-source"}) + loop = definition["flow"]["nodes"][2] + loop["state"].append({ + "name": "rows", "initial": {"kind": "node_output", "node_id": "rows-source-node", "output": "records", "scope": "current"}, + "next": "rows", "output_contract": contract, + }) + loop["body"]["nodes"].extend([ + {"id": "each", "kind": "for_each", "max_items": 2, "item_key": "source_identity", + "inputs": [state_binding("rows", slot="rows", kind="records")], "iterable": {"kind": "input", "name": "rows"}, + "body": {"id": "each-body", "nodes": [{"id": "leaf-node", "kind": "task", "task_id": "leaf"}], + "outputs": [binding("leaf-node", "findings", "records")]}}, + {"id": "collect", "kind": "collect", "source": {"loop_id": "each", "output": "findings"}, "output_contract": contract}, + ]) + loop["body"]["outputs"].extend([ + state_binding("rows", slot="rows", kind="records"), binding("collect", "findings", "records"), + ]) + loop["exports"] = [{"name": "state", "output": "findings"}] + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + rows = [{"key": 0}, {"key": 0}] + + def produce(current, resolved, execution): + if current["id"] == "source": + return result("json", {"count": 0, "ready": False}) + if current["id"] == "rows-source": + return result("records", rows) + if current["id"] == "leaf": + return result("records", [resolved["values"]["item"]["value"]]) + count = resolved["values"]["state"]["count"] + 1 + return result("json", {"count": count, "ready": count >= 2}) + + flow, calls = execute_repeat(workflow, store, result_for_task=produce) + leaves = [call for call in calls if call[0] == "leaf"] + assert flow.finished and len(leaves) == 4 + assert [call[1][0]["iteration"] for call in leaves] == [0, 0, 1, 1] + assert all(set(call[1][1]) == {"loop_id", "item_id", "index"} for call in leaves) + receipt = flow.final_outputs[0] + assert list(open_workflow_record_input( + workflow, "run", receipt["producer"], receipt["result_ref"], output_name="state", + ).iter_records()) == rows + + +def test_large_record_state_stays_reference_only_and_streams_all_originals(monkeypatch): + definition = repeat_definition(1) + contract = {"kind": "records", "schema": {"type": "array", "items": {"type": "object"}}} + definition["tasks"][0]["output_contract"] = contract + definition["tasks"][1]["inputs"] = [state_binding(kind="records")] + definition["tasks"][1]["output_contract"] = {"kind": "text"} + loop = definition["flow"]["nodes"][1] + loop["state"][0]["output_contract"] = contract + loop["state"][0]["initial"]["output"] = "records" + loop["body"]["outputs"] = [state_binding("next", kind="records")] + loop["until"] = {"op": "eq", "left": {"literal": True}, "right": {"literal": True}} + workflow, store, _, _, _ = repeat_runtime(monkeypatch, definition=definition) + records = [{"index": index, "payload": "x" * 5000, "null": None, "false": False} for index in range(1800)] + observed = [] + + def produce(current, resolved, execution): + if current["id"] == "source": + return result("records", records) + reader = resolved["values"]["state"] + observed.extend(row["index"] for row in reader.iter_records()) + return {"reply": "All complete fictional records were supplied separately."} + + flow, _ = execute_repeat(workflow, store, result_for_task=produce, stream_collections=True) + assert flow.finished and observed == list(range(len(records))) + head = repeat_head(workflow, store) + assert head["current_state_ref"]["size_bytes"] < 8192 and len(json.dumps(head)) < 8192 + receipt = flow.final_outputs[0] + reader = open_workflow_record_input(workflow, "run", receipt["producer"], receipt["result_ref"], output_name="state") + assert list(reader.iter_records()) == records diff --git a/ui_tests/fixtures/workflow_admin_limits.py b/ui_tests/fixtures/workflow_admin_limits.py index 9f4642f6e..7579d2475 100644 --- a/ui_tests/fixtures/workflow_admin_limits.py +++ b/ui_tests/fixtures/workflow_admin_limits.py @@ -1,8 +1,10 @@ # workflow_admin_limits.py """ -Closed Classic/V2 fixtures for the production workflow loop-limit settings field. -Version: 0.261.117 +Closed Classic/V2 fixtures for production For-each and Repeat policy fields. +Version: 0.261.120 Implemented in: 0.261.117 + +Repeat-until policy coverage was added in 0.261.120. """ import copy diff --git a/ui_tests/fixtures/workflow_repeat_until.py b/ui_tests/fixtures/workflow_repeat_until.py new file mode 100644 index 000000000..54acaa546 --- /dev/null +++ b/ui_tests/fixtures/workflow_repeat_until.py @@ -0,0 +1,544 @@ +# workflow_repeat_until.py +""" +Closed fixtures for typed Repeat until authoring and exact run inspection. +Version: 0.261.120 +Implemented in: 0.261.120 + +The real local SPA and production definition normalizer are used. No live +application, Azure browser, model, document service, or workspace is contacted. +""" + +import copy +import hashlib +import json +import re +from datetime import datetime, timedelta, timezone + +import pytest + +from ui_tests.fixtures.workflow_control_definitions import flow_binding +from ui_tests.fixtures.workflow_editor import GROUP_ID, connect_options, workflow_record # noqa: F401 +from ui_tests.fixtures.workflow_loops import WorkflowLoopsFixture +from ui_tests.fixtures.workspace_authoring import OWNER_ID + + +REPEAT_WORKFLOW_ID = "repeat-review-workflow" +REPEAT_RUN_ID = "repeat-review-run" +REPEAT_ID = "refine-report" +REPEAT_EXECUTION_ID = "repeat-execution" + + +def state_binding(name, state_name, kind="text", loop_id=REPEAT_ID): + return { + "name": name, + "source": {"kind": "repeat_state", "loop_id": loop_id, "state_name": state_name, "scope": "current"}, + "required": True, "expected_kind": kind, "allow_partial": False, + } + + +def state_contract(kind): + contract = {"kind": kind, "allow_partial": False, "require_complete_coverage": False} + if kind == "json": + contract["schema"] = { + "type": "object", "properties": {"ready": {"type": "boolean"}}, "required": ["ready"], + } + elif kind in {"records", "document_results"}: + contract["schema"] = { + "type": "array", "items": { + "type": "object", "properties": {"finding": {"type": "string"}}, "required": ["finding"], + }, + } + return contract + + +def repeat_workflow_record(*, group=False, collections=False): + """Every temporal source is an exact earlier output or named current state.""" + tasks = [] + for index, (task_id, name, kind) in enumerate([ + ("seed-draft", "Seed draft", "text"), + ("seed-review", "Seed review", "json"), + ("revise", "Revise draft", "text"), + ("review", "Review draft", "json"), + ], 1): + tasks.append({ + "id": task_id, "type": "instructions", "name": name, + "instructions": f"Produce {name.lower()} with the declared output shape.", + "order": index, "runner": {"type": "inherit"}, "document_action": {"type": "none"}, + "inputs": [], "reference_ids": [], "output_contract": state_contract(kind), + }) + tasks[2]["inputs"] = [state_binding("draft", "draft"), state_binding("review", "review", "json")] + tasks[3]["inputs"] = [flow_binding("draft", "revise", "text", kind="text")] + repeat = { + "id": REPEAT_ID, "kind": "repeat_until", "max_iterations": 2, + "state": [ + {"name": "draft", "initial": flow_binding("draft", "seed-draft", "text", kind="text")["source"], + "next": "next_draft", "output_contract": state_contract("text")}, + {"name": "review", "initial": flow_binding("review", "seed-review", "json")["source"], + "next": "next_review", "output_contract": state_contract("json")}, + ], + "body": { + "id": "refinement-body", + "nodes": [{"id": task_id, "kind": "task", "task_id": task_id} for task_id in ("revise", "review")], + "outputs": [flow_binding("next_draft", "revise", "text", kind="text"), flow_binding("next_review", "review", "json")], + }, + "until": {"op": "eq", "left": {"input": "review", "path": "/ready"}, "right": {"literal": True}}, + "exports": [{"name": "report", "output": "next_draft"}, {"name": "review", "output": "next_review"}], + } + nodes = [{"id": task_id, "kind": "task", "task_id": task_id} for task_id in ("seed-draft", "seed-review")] + if collections: + for kind in ("records", "document_results"): + task_id = f"seed-{kind}" + tasks.append({ + "id": task_id, "type": "instructions", "name": f"Seed {kind}", + "instructions": "Keep complete original objects in their saved order.", + "order": len(tasks) + 1, "runner": {"type": "inherit"}, "document_action": {"type": "none"}, + "inputs": [], "reference_ids": [], "output_contract": state_contract(kind), + }) + output = "documents" if kind == "document_results" else "records" + nodes.append({"id": task_id, "kind": "task", "task_id": task_id}) + repeat["state"].append({ + "name": kind, "initial": flow_binding(kind, task_id, output, kind=kind)["source"], + "next": f"next_{kind}", "output_contract": state_contract(kind), + }) + repeat["body"]["outputs"].append(state_binding(f"next_{kind}", kind, kind)) + repeat["exports"].append({"name": kind, "output": f"next_{kind}"}) + nodes.append(repeat) + return workflow_record( + REPEAT_WORKFLOW_ID, name="Group Repeat review" if group else "Repeat review", + definition_version=3, durable_execution=True, chat_capabilities_enabled=False, + tasks=tasks, reference_inputs=[], limits={"max_executions": 5000, "deadline_seconds": 86400}, + flow={"id": "root", "nodes": nodes, "outputs": [flow_binding("report", REPEAT_ID, "report", kind="text")]}, + **({"group_id": GROUP_ID} if group else {}), + ) + + +def bounded_pages(items, page_size): + return { + ("" if offset == 0 else f"offset-{offset}"): { + "items": copy.deepcopy(items[offset:offset + page_size]), + "next_cursor": f"offset-{offset + page_size}" if offset + page_size < len(items) else None, + } + for offset in range(0, len(items), page_size) + } or {"": {"items": [], "next_cursor": None}} + + +def repeat_progress(completed_count=2, batch_size=2, *, running_round=False, partial=False): + batch_number = completed_count // batch_size if running_round else completed_count // batch_size - 1 + return { + "execution_id": REPEAT_EXECUTION_ID, "node_id": REPEAT_ID, + "completed_iteration": completed_count - 1, "next_iteration": completed_count, + "batch_number": batch_number, "batch_size": batch_size, + "batch_usage": completed_count % batch_size + 1 if running_round else batch_size, + "completed_count": completed_count, + "exhaustion_count": batch_number if running_round else batch_number + 1, + "continuation_count": batch_number, + "state": "running" if running_round else "waiting_manual_continue", "partial": partial, + } + + +class WorkflowRepeatFixture(WorkflowLoopsFixture): + """Extend the existing scoped closed server, not the application runtime.""" + + def __init__(self, page): + super().__init__(page) + self.repeat_ceiling = 25 + self.repeat_hard_ceiling = 1000 + self.repeat_capabilities = True + self.personal_workflows[REPEAT_WORKFLOW_ID] = repeat_workflow_record() + self.group_workflows[GROUP_ID][REPEAT_WORKFLOW_ID] = repeat_workflow_record(group=True) + self.repeat_iteration_pages = {} + self.nested_repeat_instances = {} + self.nested_item_instances = {} + self.repeat_state_override = None + self.repeat_iterations_override = None + self.allowed_state_sources = {} + self.allowed_iteration_executions = {} + self.frozen_repeat_definitions = {} + self.repeat_grants = [] + self._seed_repeat_run("user") + self._seed_repeat_run("group") + + def _options(self, group): + options = super()._options(group) + if self.repeat_capabilities: + options["supported_node_kinds"].append("repeat_until") + options["supported_binding_sources"].append("repeat_state") + if self.repeat_ceiling is not None: + options["flow_limits"]["max_repeat_iterations"] = self.repeat_ceiling + if self.repeat_hard_ceiling is not None: + options["flow_limits"]["hard_repeat_iterations"] = self.repeat_hard_ceiling + options["publication_source_capabilities"] = [{ + "source_kind": "saved_output", "output_kinds": ["records"], "artifact_formats": ["json"], + }] + options["supported_publication_completion_policies"] = ["submitted", "approved", "indexed_ready"] + return options + + def _seed_repeat_run(self, scope, *, completed_count=2, batch_size=2, running_round=False, partial=False): + key = (scope, REPEAT_WORKFLOW_ID, REPEAT_RUN_ID) + summary = repeat_progress(completed_count, batch_size, running_round=running_round, partial=partial) + status = "running" if running_round else "paused" + self.frozen_repeat_definitions[key] = repeat_workflow_record(group=scope == "group", collections=True) + self.frozen_repeat_definitions[key]["flow"]["nodes"][-1]["max_iterations"] = batch_size + if partial: + snapshot = self.frozen_repeat_definitions[key] + snapshot["flow"]["nodes"][-1]["state"][2]["output_contract"]["allow_partial"] = True + snapshot["flow"]["nodes"][-1]["body"]["outputs"][2]["allow_partial"] = True + snapshot["flow"]["outputs"][0]["allow_partial"] = True + next(task for task in snapshot["tasks"] if task["id"] == "seed-records")["output_contract"]["allow_partial"] = True + self.workflow_runs[REPEAT_WORKFLOW_ID] = [{ + "id": REPEAT_RUN_ID, "workflow_id": REPEAT_WORKFLOW_ID, "definition_version": 3, + "durable_execution": True, "status": status, "started_at": "2026-09-19T12:00:00Z", + }] + runtime = { + "schema_version": 2, "version": 10, "state": status, "phase": REPEAT_ID, + "repeat_progress": copy.deepcopy(summary), + "repeat_counts": {"exhaustion_count": summary["exhaustion_count"], "continuation_count": summary["continuation_count"]}, + "memory": {"execution_count": completed_count * 3 + 5, "decision_count": completed_count}, + "limits": { + "max_executions": 5000, "admitted_count": completed_count * 3 + 5, "deadline_seconds": 86400, + "deadline_at": (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + "waits_count": True, "max_repeat_iterations": max(25, batch_size), + }, + } + if not running_round: + runtime["gate"] = { + "id": "repeat-limit-gate-1", "kind": "pause", "reason_code": "repeat_iteration_limit", + "unit_id": REPEAT_ID, "execution_id": REPEAT_EXECUTION_ID, "node_id": REPEAT_ID, + "attempt": 1, "iteration_path": [], "choices": ["continue_repeat", "cancel"], + "reason": "The stop condition is still unmet. Saved state is retained.", + "repeat": copy.deepcopy(summary), + } + self.workflow_runtimes[key] = runtime + self.runtime_can_decide[key] = True + self.runtime_get_count[key] = 0 + self.execution_pages[key] = bounded_pages([{ + "execution_id": REPEAT_EXECUTION_ID, "node_id": REPEAT_ID, "node_kind": "repeat_until", + "state": status, "attempt": 1, "iteration_path": [], + **({"reason_code": "repeat_iteration_limit"} if not running_round else {}), + }], 50) + iterations = [{ + "iteration": iteration, "iteration_path": [{"loop_id": REPEAT_ID, "iteration": iteration}], + "batch_number": iteration // batch_size, "batch_size": batch_size, "batch_usage": iteration % batch_size + 1, + "state": "running" if iteration == completed_count else "completed_partial" if partial else "completed", + "condition_result": None if iteration == completed_count else False, + "before_available": True, "after_available": iteration < completed_count, + "partial": partial and iteration < completed_count, + "execution_ids": [f"revise-round-{iteration}"] if iteration == completed_count + else [f"revise-round-{iteration}", f"review-round-{iteration}"], + } for iteration in range(completed_count + int(running_round))] + self.repeat_iteration_pages[key] = bounded_pages(iterations, 1 if len(iterations) < 10 else 50) + decisions = [{ + "execution_id": REPEAT_EXECUTION_ID, "node_id": REPEAT_ID, "iteration_path": [], + "attempt": 1, "decision_kind": "repeat_transition", "iteration": completed_count - 1, + "batch_number": (completed_count - 1) // batch_size, "condition_result": False, + }] + if summary["continuation_count"]: + grant_summary = repeat_progress(summary["batch_number"] * batch_size, batch_size, partial=partial) + decisions.append({ + "execution_id": REPEAT_EXECUTION_ID, "node_id": REPEAT_ID, "iteration_path": [], + "attempt": 1, "choice": "continue_repeat", "actor_user_id": OWNER_ID, + "gate_id": "previous-repeat-limit", "decided_at": "2026-09-19T12:10:00Z", + "event_id": "fixture-repeat-grant", "reason_code": "repeat_iteration_limit", "repeat": grant_summary, + }) + self.decision_pages[key] = bounded_pages(decisions, 50) + + def _state_slots(self, iteration, phase, partial): + source_iteration = iteration - 1 if phase == "before" else iteration + slots = [] + for name, kind, node_id in ( + ("draft", "text", "revise"), ("review", "json", "review"), + ("records", "records", "seed-records"), ("document_results", "document_results", "seed-document_results"), + ): + initial = source_iteration < 0 or kind in {"records", "document_results"} + producer_node = f"seed-{name}" if initial else node_id + source = { + "node_id": producer_node, "execution_id": producer_node if initial else f"{node_id}-round-{source_iteration}", + "attempt": 1 if initial or kind == "text" else 2, + "iteration_path": [] if initial else [{"loop_id": REPEAT_ID, "iteration": source_iteration}], + "output_name": "documents" if kind == "document_results" else kind, + } + accepted_partial = partial and kind == "records" + slots.append({ + "name": name, "kind": kind, "source": source, + "workflow_validation": { + "version": 1, "status": "accepted_partial" if accepted_partial else "valid", "eligible": True, + "reason_codes": ["producer_coverage_incomplete"] if accepted_partial else [], "counts": {}, + }, + "coverage": {"complete": not accepted_partial, "record_count": 260} if kind == "records" else {}, + "limitations": ["One source could not be analyzed; original accepted records are retained."] if accepted_partial else [], + **({"prior_coverage": {"complete": False, "record_count": 260}} + if accepted_partial and source_iteration >= 0 else {}), + }) + return slots + + def complete_with_records_export(self, scope="user"): + key = (scope, REPEAT_WORKFLOW_ID, REPEAT_RUN_ID) + runtime = self.workflow_runtimes[key] + runtime.update(state="completed") + self.workflow_runs[REPEAT_WORKFLOW_ID][0]["status"] = "completed" + runtime.pop("gate", None) + runtime["repeat_progress"]["state"] = "completed" + final_round = list(self.repeat_iteration_pages[key].values())[-1]["items"][-1] + final_round["condition_result"] = True + record = self.execution_pages[key][""]["items"][0] + record.update( + state="completed", + workflow_validation={"version": 1, "status": "valid", "eligible": True, "counts": {}}, + workflow_result={ + "authoritative_output": "findings", "outputs": {"findings": {"kind": "records"}}, + "result_ref": {"storage": "cosmos", "size_bytes": 4096, "chunk_count": 2, "sha256": "a" * 64}, + }, + ) + record.pop("reason_code", None) + self.attempt_pages[(*key, REPEAT_EXECUTION_ID)] = bounded_pages([record], 50) + slot = self._state_slots(1, "after", False)[2] + slot["source"] = { + "node_id": REPEAT_ID, "execution_id": REPEAT_EXECUTION_ID, "attempt": 1, "iteration_path": [], "output_name": "findings", + } + self.allowed_state_sources[(*key, REPEAT_EXECUTION_ID, 1, "findings")] = slot + frozen_repeat = self.frozen_repeat_definitions[key]["flow"]["nodes"][-1] + frozen_repeat["exports"].append({"name": "findings", "output": "next_records"}) + + def add_nested_inspection(self, scope="user"): + key = (scope, REPEAT_WORKFLOW_ID, REPEAT_RUN_ID) + first = self.repeat_iteration_pages[key][""]["items"][0] + first["execution_ids"] = ["nested-repeat-execution", "nested-each-execution"] + for execution_id, node_id, kind in ( + ("nested-repeat-execution", "nested-refinement", "repeat_until"), + ("nested-each-execution", "each-saved-finding", "for_each"), + ): + self.attempt_pages[(*key, execution_id)] = bounded_pages([{ + "execution_id": execution_id, "node_id": node_id, "node_kind": kind, "attempt": 1, + "state": "completed", "iteration_path": copy.deepcopy(first["iteration_path"]), + "workflow_result": { + "outputs": {}, "result_ref": {"storage": "cosmos", "size_bytes": 512, "chunk_count": 1, "sha256": "c" * 64}, + }, + }], 50) + summary = { + **repeat_progress(1, 1), "execution_id": "nested-repeat-execution", "node_id": "nested-refinement", + "state": "completed", "exhaustion_count": 0, + } + self.nested_repeat_instances[(*key, "nested-repeat-execution")] = { + "summary": summary, + "pages": bounded_pages([{ + "iteration": 0, "iteration_path": [*copy.deepcopy(first["iteration_path"]), {"loop_id": "nested-refinement", "iteration": 0}], + "batch_number": 0, "batch_size": 1, "batch_usage": 1, "state": "completed", + "condition_result": True, "execution_ids": [], "before_available": True, "after_available": True, "partial": False, + }], 50), + } + self.nested_item_instances[(*key, "nested-each-execution")] = { + "loop_execution_id": "nested-each-execution", "frozen_at": "2026-09-19T12:00:00Z", + "total_count": 1, "next_cursor": None, "limit": 500, + "items": [{ + "item_id": "e" * 64, "index": 0, "label": "Exact saved finding", "state": "completed", + "iteration_path": [ + *copy.deepcopy(first["iteration_path"]), + {"loop_id": "each-saved-finding", "item_id": "e" * 64, "index": 0}, + ], + "execution_ids": [], "record_count": 1, + }], + } + + def _dispatch(self, route, entry): + prefix = rf"/api/(user|group)/workflows/{REPEAT_WORKFLOW_ID}/runs/{REPEAT_RUN_ID}/executions" + if re.fullmatch(prefix + r"/[^/]+/iterations(?:/\d+/state)?", entry.path): + self._repeat_history_resource(route, entry) + elif re.fullmatch(prefix + r"/(?:revise|review)-round-\d+/attempts", entry.path): + self._repeat_attempts_resource(route, entry) + elif re.fullmatch(prefix + r"/nested-each-execution/items", entry.path): + assert entry.query.get("limit") == ["50"], entry + self._json(route, copy.deepcopy(self.nested_item_instances[(*self._key(entry), "nested-each-execution")])) + elif re.fullmatch(prefix + r"/[^/]+/attempts/\d+/(?:result|records|provenance)", entry.path): + self._repeat_source_resource(route, entry) + else: + super()._dispatch(route, entry) + + def _repeat_history_resource(self, route, entry): + key = self._key(entry) + parts = entry.path.split("/") + execution_id = parts[8] + assert entry.query.get("limit") == ["50"], entry + runtime = self.workflow_runtimes[key] + nested = self.nested_repeat_instances.get((*key, execution_id)) + assert execution_id == REPEAT_EXECUTION_ID or nested is not None, entry + summary = nested["summary"] if nested else runtime["repeat_progress"] + pages = nested["pages"] if nested else self.repeat_iteration_pages[key] + if parts[-1] == "iterations": + if self.repeat_iterations_override is not None: + self._json(route, copy.deepcopy(self.repeat_iterations_override)) + return + page = pages[entry.query.get("cursor", [""])[0]] + self.allowed_iteration_executions.setdefault(key, set()).update( + execution_id for item in page["items"] for execution_id in item["execution_ids"] + ) + self._json(route, { + "iterations": copy.deepcopy(page["items"]), "next_cursor": page["next_cursor"], + "total_count": sum(len(value["items"]) for value in pages.values()), + "repeat_execution_id": execution_id, "repeat": copy.deepcopy(summary), + "source_snapshot_changed": False, + }) + return + iteration, phase = int(parts[10]), entry.query.get("phase", [""])[0] + assert phase in {"before", "after"}, entry + assert any(item["iteration"] == iteration for page in pages.values() for item in page["items"]), entry + if self.repeat_state_override is not None: + self._json(route, copy.deepcopy(self.repeat_state_override)) + return + available = phase == "before" or iteration < summary["completed_count"] + result = {"iteration": iteration, "phase": phase, "repeat_execution_id": execution_id, "available": available} + if not available: + self._json(route, {**result, "states": [], "next_cursor": None, "total_count": 0}) + return + partial = summary["partial"] + slots = self._state_slots(iteration, phase, partial) + pages = bounded_pages(slots, 2) + selected = pages[entry.query.get("cursor", [""])[0]] + for slot in selected["items"]: + source = slot["source"] + self.allowed_state_sources[(*key, source["execution_id"], source["attempt"], source["output_name"])] = copy.deepcopy(slot) + self._json(route, { + **result, "states": copy.deepcopy(selected["items"]), "next_cursor": selected["next_cursor"], + "total_count": len(slots), "partial": partial, "source_snapshot_changed": False, + }) + + def _repeat_attempts_resource(self, route, entry): + key = self._key(entry) + execution_id = entry.path.split("/")[8] + assert execution_id in self.allowed_iteration_executions.get(key, set()), entry + assert entry.query.get("limit") == ["50"], entry + node_id, _, round_text = execution_id.partition("-round-") + iteration = int(round_text) + if iteration >= self.workflow_runtimes[key]["repeat_progress"]["completed_count"]: + self._json(route, { + "attempts": [{ + "execution_id": execution_id, "node_id": node_id, "task_id": node_id, + "attempt": 1, "state": "running", "iteration_path": [{"loop_id": REPEAT_ID, "iteration": iteration}], + }], + "next_cursor": None, "total_count": 1, + }) + return + slot = self._state_slots(iteration, "after", self.workflow_runtimes[key]["repeat_progress"]["partial"])[ + 1 if node_id == "review" else 0 + ] + source = slot["source"] + record = { + "execution_id": execution_id, "node_id": node_id, "task_id": node_id, "attempt": source["attempt"], + "iteration_path": source["iteration_path"], "state": "completed", + "workflow_validation": copy.deepcopy(slot["workflow_validation"]), + "workflow_result": { + "authoritative_output": source["output_name"], "outputs": {source["output_name"]: {"kind": slot["kind"]}}, + "result_ref": {"storage": "cosmos", "size_bytes": 512, "chunk_count": 1, "sha256": "b" * 64}, + }, + } + self.allowed_state_sources[(*key, execution_id, source["attempt"], "authoritative")] = slot + attempts = [record] + if source["attempt"] == 2: + attempts.insert(0, { + "execution_id": execution_id, "node_id": node_id, "task_id": node_id, "attempt": 1, + "iteration_path": source["iteration_path"], "state": "failed", + }) + self._json(route, {"attempts": attempts, "next_cursor": None, "total_count": len(attempts)}) + + def _repeat_source_resource(self, route, entry): + key = self._key(entry) + parts = entry.path.split("/") + execution_id, attempt, resource = parts[8], int(parts[10]), parts[11] + output = entry.query.get("output", [""])[0] + if resource == "provenance": + assert entry.query.get("limit") == ["50"], entry + slot = next(slot for source, slot in self.allowed_state_sources.items() if source[:5] == (*key, execution_id, attempt)) + source = copy.deepcopy(slot["source"]) + source.pop("output_name") + self._json(route, { + "contributors": [{"producer": source, "output_name": slot["source"]["output_name"], + "record_offset": 0, "record_count": 260, "producer_record_offset": 0}], + "total_count": 1, "next_cursor": None, + }) + return + selected_key = (*key, execution_id, attempt, output) + assert selected_key in self.allowed_state_sources, f"Unselected producer read: {entry}" + slot = self.allowed_state_sources[selected_key] + if resource == "records": + assert entry.query.get("limit") == ["100"], entry + assert slot["kind"] in {"records", "document_results"}, entry + records = [{"finding": f"Original finding {index}", "ready": False, "amount": 0, "details": {"nullable": None}} + for index in range(260)] + pages = bounded_pages(records, 100) + cursor = entry.query.get("cursor", [""])[0] + page = pages[cursor] + self._json(route, { + "records": page["items"], "next_cursor": page["next_cursor"], "total_count": len(records), + "record_offset": int(cursor.removeprefix("offset-")) if cursor else 0, + "output_name": output, "workflow_validation": copy.deepcopy(slot["workflow_validation"]), + "coverage": copy.deepcopy(slot["coverage"]), + }) + return + assert resource == "result" and entry.query.get("limit") == ["2000"], entry + value = {"ready": False, "note": "Retained exact review"} if slot["kind"] == "json" else "Retained exact draft" + content = json.dumps({ + "kind": slot["kind"], "output_name": output, "producer": slot["source"], "value": value, + }, ensure_ascii=True, sort_keys=True) + offset = int(entry.query.get("offset", ["0"])[0]) + end = min(len(content), offset + 2000) + self._json(route, { + "content": content[offset:end], "offset": offset, "next_offset": end if end < len(content) else None, + "output_name": output, "total_bytes": len(content), "complete": end == len(content), + "sha256": hashlib.sha256(content.encode("ascii")).hexdigest(), + }) + + def _workflow_runtime(self, route, entry): + scope = entry.path.split("/")[2] + key = (scope, REPEAT_WORKFLOW_ID, REPEAT_RUN_ID) + if (f"/{REPEAT_WORKFLOW_ID}/" in entry.path and entry.path.endswith("/runtime/decision") + and entry.body.get("choice") == "cancel"): + self.workflow_runtimes[key]["repeat_progress"]["state"] = "cancelled" + if f"/{REPEAT_WORKFLOW_ID}/" not in entry.path or not entry.path.endswith("/runtime/decision") or entry.body.get("choice") != "continue_repeat": + super()._workflow_runtime(route, entry) + return + assert entry.query.get("group_id") == ([GROUP_ID] if scope == "group" else None), entry + runtime = self.workflow_runtimes[key] + assert set(entry.body) == {"expected_version", "gate_id", "choice", "request_id"}, entry + assert entry.body["expected_version"] == runtime["version"], entry + assert entry.body["gate_id"] == runtime["gate"]["id"], entry + assert re.fullmatch(r"[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}", entry.body["request_id"]), entry + if not self.runtime_can_decide[key]: + self.expected_http_errors.add((route.request.url, 403)) + self._json(route, {"error": "Decision permission was revoked."}, 403) + return + if self.fail_next_decision_status: + status, self.fail_next_decision_status = self.fail_next_decision_status, None + self.expected_http_errors.add((route.request.url, status)) + self._json(route, {"error": "Transient decision failure."}, status) + return + if self.stale_next_decision: + self.stale_next_decision = False + runtime["version"] += 1 + runtime["gate"]["id"] = "repeat-limit-refreshed" + self.expected_http_errors.add((route.request.url, 409)) + self._json(route, {"error": "The Repeat gate changed."}, 409) + return + gate = runtime.pop("gate") + self.repeat_grants.append(copy.deepcopy(entry)) + decision = { + "execution_id": REPEAT_EXECUTION_ID, "node_id": REPEAT_ID, "iteration_path": [], "attempt": 1, + "gate_id": gate["id"], "choice": "continue_repeat", "actor_user_id": OWNER_ID, + "decided_at": datetime.now(timezone.utc).isoformat(), "repeat": copy.deepcopy(gate["repeat"]), + "event_id": "fixture-committed-repeat-grant", "reason_code": "repeat_iteration_limit", + } + decisions = [item for page in self.decision_pages[key].values() for item in page["items"]] + self.decision_pages[key] = bounded_pages([*decisions, decision], 50) + runtime["version"] += 1 + runtime["state"] = "queued" + self.workflow_runs[REPEAT_WORKFLOW_ID][0]["status"] = "queued" + summary = runtime["repeat_progress"] + summary.update(batch_number=summary["batch_number"] + 1, batch_usage=0, + continuation_count=summary["continuation_count"] + 1, state="running") + runtime["repeat_counts"]["continuation_count"] += 1 + self._json(route, {"runtime": copy.deepcopy(runtime), "can_decide": True}) + + +@pytest.fixture +def workflow_repeat_ui(page): + fixture = WorkflowRepeatFixture(page) + yield fixture + fixture.assert_clean() diff --git a/ui_tests/test_v2_workflow_loops.py b/ui_tests/test_v2_workflow_loops.py index 9ab735969..b6cf044cd 100644 --- a/ui_tests/test_v2_workflow_loops.py +++ b/ui_tests/test_v2_workflow_loops.py @@ -1,7 +1,7 @@ # test_v2_workflow_loops.py """ Closed browser regressions for serial For each, exact Collect and explicit saved-record reporting. -Version: 0.261.117 +Version: 0.261.120 Implemented in: 0.261.117 Loads the real built local SPA and validates authoring payloads with production @@ -1037,13 +1037,14 @@ def test_malformed_frozen_item_pages_fail_closed(workflow_loops_ui, malformed): assert not any(request.path.endswith("/records") for request in ui.requests) -def test_repeat_runtime_gate_identity_is_not_actionable(workflow_loops_ui): +def test_hybrid_repeat_and_item_runtime_gate_identity_is_not_actionable(workflow_loops_ui): ui, page = workflow_loops_ui, workflow_loops_ui.page runtime = ui.workflow_runtimes[("user", LOOP_WORKFLOW_ID, LOOP_RUN_ID)] runtime.update(state="waiting_approval", gate={ "id": "invalid-repeat-gate", "kind": "approval", "unit_id": "task:analyze-task", "execution_id": ITEM_A_EXECUTION_ID, "node_id": "analyze-one", "attempt": 1, - "iteration_path": [{"loop_id": LOOP_ID, "iteration": 1}], "choices": ["approve", "reject"], + "iteration_path": [{"loop_id": LOOP_ID, "iteration": 1, "item_id": "a" * 64, "index": 1}], + "choices": ["approve", "reject"], }) expand_history(ui) expect(page.get_by_role("alert").filter(has_text="unsupported iteration identity")).to_be_visible() diff --git a/ui_tests/test_v2_workflow_repeat_until.py b/ui_tests/test_v2_workflow_repeat_until.py new file mode 100644 index 000000000..ba39c308a --- /dev/null +++ b/ui_tests/test_v2_workflow_repeat_until.py @@ -0,0 +1,962 @@ +# test_v2_workflow_repeat_until.py +""" +Local closed-browser regressions for typed Repeat until and manual continuation. +Version: 0.261.120 +Implemented in: 0.261.120 + +Exercises the actual built V2 SPA, strict guards, scoped API requests and production +definition normalization. Fixtures intercept every request; no live workflow runs. +""" + +import copy +import re +import sys +from pathlib import Path + +import pytest +from playwright.sync_api import expect + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "ui_tests")) +sys.path.insert(0, str(ROOT / "ui_tests" / "fixtures")) + +# Shared fixtures configure isolated production imports after local path setup. +from ui_tests.fixtures.workflow_repeat_until import ( + GROUP_ID, + REPEAT_EXECUTION_ID, + REPEAT_ID, + REPEAT_RUN_ID, + REPEAT_WORKFLOW_ID, + connect_options, # noqa: F401 + flow_binding, + repeat_workflow_record, + state_binding, + workflow_repeat_ui, # noqa: F401 +) + + +pytestmark = pytest.mark.ui + + +def repeat_block(page): + return page.get_by_role("region", name="Repeat until block", exact=True) + + +def task_block(page, name): + return page.get_by_role("region", name=f"{name} block", exact=True) + + +def details(block): + block.get_by_text("Runner, inputs, references and outputs", exact=True).click() + + +def open_repeat(ui, *, group=False, **kwargs): + if group: + ui.open("/groups", **kwargs) + ui.page.get_by_label("Group workspace", exact=True).select_option(GROUP_ID) + ui.page.get_by_role("button", name="Edit Group Repeat review", exact=True).click() + else: + ui.open(f"/workspace/workflows?workflow_id={REPEAT_WORKFLOW_ID}", **kwargs) + return repeat_block(ui.page) + + +def test_author_repeat_with_explicit_maximum_next_state_and_typed_condition(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.open("/workspace/workflows") + page.get_by_role("button", name="Create workflow", exact=True).click() + page.get_by_label("Workflow name", exact=True).fill("Authored Repeat review") + page.get_by_label("Model", exact=True).select_option(label="Workspace GPT \u00b7 aoai") + page.get_by_role("button", name="Enable structured control flow", exact=True).click() + page.get_by_role("button", name="Convert draft", exact=True).click() + page.get_by_label("Task name", exact=True).fill("Seed review") + seed = task_block(page, "Seed review") + seed.get_by_label("Instructions", exact=True).fill("Return ready false in the declared JSON schema.") + details(seed) + seed.get_by_label("Output contract for Seed review", exact=True).select_option("json") + seed.get_by_label("Decision field name", exact=True).fill("ready") + seed.get_by_role("button", name="Add decision field", exact=True).click() + page.get_by_role("button", name="Add Repeat until to Main", exact=True).click() + repeat = repeat_block(page) + maximum = repeat.get_by_label("Maximum rounds before manual continuation", exact=True) + expect(maximum).to_have_value("") + expect(maximum).to_be_focused() + expect(repeat.get_by_role("alert")).to_contain_text("Choose an explicit maximum") + maximum.fill("2") + repeat.get_by_role("button", name="Add state slot", exact=True).click() + expect(repeat.get_by_label("State 1 name", exact=True)).to_be_focused() + repeat.get_by_label("State 1 name", exact=True).fill("review") + repeat.get_by_label("State 1 kind", exact=True).select_option("json") + repeat.get_by_label("State 1 initial producer", exact=True).select_option(label="Seed review") + repeat.get_by_label("State 1 initial output", exact=True).select_option("json") + repeat.get_by_role("button", name="State 1 use initial schema", exact=True).click() + repeat.get_by_role("button", name="Add task to Repeat body", exact=True).click() + repeat.get_by_label("Task name", exact=True).fill("Review again") + review = task_block(page, "Review again") + review.get_by_label("Instructions", exact=True).fill("Review the current saved decision and produce the next ready Boolean.") + details(review) + review.get_by_role("button", name="Add review again inputs input", exact=True).click() + review.get_by_label("Review again inputs input 1 name", exact=True).fill("review") + review.get_by_label("Review again inputs input 1 source", exact=True).select_option("repeat_state") + expect(review.get_by_label("Review again inputs input 1 state", exact=True)).to_have_value("review") + expect(review.get_by_label("Review again inputs input 1 allow partial", exact=True)).not_to_be_checked() + review.get_by_label("Output contract for Review again", exact=True).select_option("json") + review.get_by_label("Decision field name", exact=True).fill("ready") + review.get_by_role("button", name="Add decision field", exact=True).click() + outputs = repeat.get_by_role("group", name="Repeat body outputs", exact=True) + outputs.get_by_role("button", name="Add repeat body outputs input", exact=True).click() + outputs.get_by_label("Repeat body outputs input 1 name", exact=True).fill("next_review") + outputs.get_by_label("Repeat body outputs input 1 producer", exact=True).select_option(label="Review again") + outputs.get_by_label("Repeat body outputs input 1 output", exact=True).select_option("json") + repeat.get_by_label("State 1 next body output", exact=True).select_option("next_review") + repeat.get_by_label("Stop after a round when left input", exact=True).select_option("review") + repeat.get_by_label("Stop after a round when left field", exact=True).select_option("/ready") + repeat.get_by_role("button", name="Add Repeat export", exact=True).click() + repeat.get_by_label("Repeat export 1 name", exact=True).fill("review") + repeat.get_by_label("Repeat export 1 output", exact=True).select_option("next_review") + ui.assert_no_overflow() + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Create workflow", exact=True)).to_have_count(0) + payload = ui.workflow_writes[-1].body + node = payload["flow"]["nodes"][1] + assert node["kind"] == "repeat_until" and node["max_iterations"] == 2 + assert set(node) == {"id", "kind", "max_iterations", "state", "body", "until", "exports"} + assert node["state"][0]["initial"] == { + "kind": "node_output", "node_id": payload["flow"]["nodes"][0]["id"], "output": "json", "scope": "current", + } + assert node["state"][0]["next"] == "next_review" + assert node["state"][0]["output_contract"]["schema"]["properties"]["ready"] == {"type": "boolean"} + assert node["state"][0]["output_contract"]["allow_partial"] is False + assert payload["tasks"][1]["inputs"][0]["source"] == { + "kind": "repeat_state", "loop_id": node["id"], "state_name": "review", "scope": "current", + } + assert node["until"] == {"op": "eq", "left": {"input": "review", "path": "/ready"}, "right": {"literal": True}} + assert node["exports"] == [{"name": "review", "output": "next_review"}] + page.get_by_role("button", name="Edit Authored Repeat review", exact=True).click() + expect(page.get_by_label("Maximum rounds before manual continuation", exact=True)).to_have_value("2") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + assert ui.workflow_writes[-1].body["flow"] == payload["flow"] + assert not any(request.path.endswith("/run") for request in ui.writes) + + +@pytest.mark.parametrize("group", [False, True]) +def test_personal_group_mobile_keyboard_roundtrip_preserves_all_typed_state(workflow_repeat_ui, group): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + record = repeat_workflow_record(group=group, collections=True) + (ui.group_workflows[GROUP_ID] if group else ui.personal_workflows)[REPEAT_WORKFLOW_ID] = record + repeat = open_repeat(ui, group=group, theme="dark", width=390, height=844) + ui.assert_no_overflow() + expect(repeat.get_by_label("State 3 kind", exact=True)).to_have_value("records") + expect(repeat.get_by_label("State 4 kind", exact=True)).to_have_value("document_results") + expect(repeat.get_by_label("Repeat body outputs input 3 source", exact=True)).to_have_value("repeat_state") + expect(repeat.get_by_label("State 3 allow partial", exact=True)).not_to_be_checked() + maximum = repeat.get_by_label("Maximum rounds before manual continuation", exact=True) + maximum.focus() + maximum.press("ArrowUp") + expect(maximum).to_have_value("3") + save = page.get_by_role("button", name="Save workflow", exact=True) + save.focus() + save.press("Enter") + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + expected = copy.deepcopy(record["flow"]) + expected["nodes"][-1]["max_iterations"] = 3 + assert ui.workflow_writes[-1].body["flow"] == expected + request = ui.workflow_writes[-1] + assert request.path == ("/api/group/workflows" if group else "/api/user/workflows") + assert request.query.get("group_id") == ([GROUP_ID] if group else None) + assert not any(request.path == "/api/user/settings" and request.method != "GET" for request in ui.requests) + ui.assert_no_overflow() + + +@pytest.mark.parametrize("maximum", ["", "0", "-1", "1.5", "1001", "26"]) +def test_invalid_or_over_policy_maximum_is_not_clamped_or_saved(workflow_repeat_ui, maximum): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + original = copy.deepcopy(ui.personal_workflows[REPEAT_WORKFLOW_ID]) + repeat = open_repeat(ui) + repeat.get_by_label("Maximum rounds before manual continuation", exact=True).fill(maximum) + expect(repeat.get_by_role("alert")).to_contain_text("administrator ceiling" if maximum == "26" else "whole number from 1 to 1,000") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_be_visible() + assert not ui.workflow_writes + assert ui.personal_workflows[REPEAT_WORKFLOW_ID] == original + + +def open_repeat_run(ui, *, group=False, **kwargs): + page = ui.page + ui.open("/groups" if group else "/workspace/workflows", **kwargs) + if group: + page.get_by_label("Group workspace", exact=True).select_option(GROUP_ID) + row = page.get_by_role("listitem").filter(has_text="Group Repeat review" if group else "Repeat review").first + row.get_by_role("button", name="Show run history", exact=True).click() + row.get_by_role("button", name="Show run task results", exact=True).click() + return row + + +def open_rounds(ui): + ui.page.get_by_role("button", name=f"Show Repeat rounds for {REPEAT_EXECUTION_ID}", exact=True).click() + return ui.page.get_by_role("region", name="Repeat round inspection", exact=True) + + +def runtime_key(group=False): + return ("group" if group else "user", REPEAT_WORKFLOW_ID, REPEAT_RUN_ID) + + +def decision_writes(ui): + return [request for request in ui.writes if request.path.endswith("/runtime/decision")] + + +def confirm_repeat(page, size=2): + page.get_by_role("button", name="Continue Repeat", exact=True).click() + dialog = page.get_by_role("dialog", name="Continue Repeat?", exact=True) + dialog.get_by_role("button", name=f"Continue Repeat for up to another {size} rounds", exact=True).click() + return dialog + + +@pytest.mark.parametrize("group", [False, True]) +def test_manual_continue_is_explicit_scoped_and_preserves_all_frozen_budgets(workflow_repeat_ui, group): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.repeat_ceiling = 1 + ui.workflow_runtimes[runtime_key(group)]["can_resume"] = True + frozen = copy.deepcopy(ui.workflow_runtimes[runtime_key(group)]["limits"]) + open_repeat_run(ui, group=group, width=390 if group else 1440, height=844 if group else 900) + progress = page.get_by_role("region", name="Repeat progress", exact=True) + expect(progress).to_contain_text("Automatic batch 1: 2 of 2 rounds admitted") + expect(page.get_by_text("The stop condition is still unmet. Saved state is retained.", exact=True)).to_be_visible() + expect(page.get_by_role("button", name="Resume run", exact=True)).to_have_count(0) + assert not decision_writes(ui) + button = page.get_by_role("button", name="Continue Repeat", exact=True) + button.focus() + button.press("Enter") + dialog = page.get_by_role("dialog", name="Continue Repeat?", exact=True) + expect(dialog).to_contain_text("original deadline are unchanged") + assert not decision_writes(ui) + ui.assert_no_overflow() + confirm = dialog.get_by_role("button", name="Continue Repeat for up to another 2 rounds", exact=True) + confirm.focus() + confirm.press("Enter") + expect(dialog).to_have_count(0) + expect(progress).to_contain_text("Automatic batch 2: 0 of 2 rounds admitted") + expect(progress).to_contain_text("Next lifetime round: 3") + assert len(ui.repeat_grants) == 1 + request = ui.repeat_grants[0] + assert request.path == f"/api/{'group' if group else 'user'}/workflows/{REPEAT_WORKFLOW_ID}/runs/{REPEAT_RUN_ID}/runtime/decision" + assert request.query.get("group_id") == ([GROUP_ID] if group else None) + assert set(request.body) == {"expected_version", "gate_id", "choice", "request_id"} + assert request.body["expected_version"] == 10 + assert request.body["gate_id"] == "repeat-limit-gate-1" + assert request.body["choice"] == "continue_repeat" + assert re.fullmatch(r"[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}", request.body["request_id"]) + updated = ui.workflow_runtimes[runtime_key(group)] + assert updated["limits"] == frozen + assert updated["repeat_progress"]["completed_iteration"] == 1 + assert updated["repeat_progress"]["next_iteration"] == 2 + assert not any(request.path.endswith("/runtime/resume") for request in ui.writes) + audit = page.locator("details").filter(has=page.get_by_text("Runtime decision history", exact=True)) + audit.get_by_role("button", name="Refresh", exact=True).click() + expect(audit.get_by_role("list", name="Runtime decisions", exact=True)).to_contain_text("fixture-committed-repeat-grant") + expect(audit).to_contain_text("Explicit grant by workspace-editor-user") + + +@pytest.mark.parametrize("group", [False, True]) +def test_repeat_manual_decisions_use_server_permission_without_hiding_read_views(workflow_repeat_ui, group): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.runtime_can_decide[runtime_key(group)] = False + open_repeat_run(ui, group=group) + expect(page.get_by_role("region", name="Repeat progress", exact=True)).to_be_visible() + expect(page.get_by_text("you do not have permission", exact=False)).to_be_visible() + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_have_count(0) + expect(page.get_by_role("button", name="Resume run", exact=True)).to_have_count(0) + open_rounds(ui) + expect(page.get_by_role("list", name="Repeat rounds", exact=True)).to_contain_text("Round 1") + assert not decision_writes(ui) + + +def test_stale_repeat_decision_refreshes_real_gate_and_requires_another_confirmation(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui) + ui.stale_next_runtime_decision() + confirm_repeat(page) + expect(page.get_by_role("alert").filter(has_text="Runtime changed before your decision")).to_be_visible() + assert not ui.repeat_grants + assert len(decision_writes(ui)) == 1 + confirm_repeat(page) + expect(page.get_by_role("dialog", name="Continue Repeat?", exact=True)).to_have_count(0) + writes = decision_writes(ui) + assert len(writes) == 2 and len(ui.repeat_grants) == 1 + assert writes[0].body["request_id"] != writes[1].body["request_id"] + assert writes[1].body["expected_version"] == 11 and writes[1].body["gate_id"] == "repeat-limit-refreshed" + assert ui.workflow_runtimes[runtime_key()]["repeat_counts"]["continuation_count"] == 1 + + +def test_transient_repeat_failure_reuses_request_uuid_without_double_grant(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui) + ui.fail_next_decision_status = 503 + confirm_repeat(page) + expect(page.get_by_role("alert").filter(has_text="Retry uses the same request id for this gate")).to_be_visible() + assert not ui.repeat_grants + confirm_repeat(page) + expect(page.get_by_role("dialog", name="Continue Repeat?", exact=True)).to_have_count(0) + writes = decision_writes(ui) + assert len(writes) == 2 and len(ui.repeat_grants) == 1 + assert writes[0].body["request_id"] == writes[1].body["request_id"] + + +def test_repeat_confirmation_cannot_transfer_to_a_changed_gate(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui) + page.get_by_role("button", name="Continue Repeat", exact=True).click() + changed = copy.deepcopy(ui.workflow_runtimes[runtime_key()]) + changed["version"] += 1 + changed["gate"]["id"] = "new-repeat-exhaustion" + ui.transition_runtime_on_get(REPEAT_WORKFLOW_ID, REPEAT_RUN_ID, ui.runtime_get_count[runtime_key()] + 1, changed) + expect(page.get_by_text("Version 11", exact=True)).to_be_visible(timeout=10000) + page.get_by_role("dialog", name="Continue Repeat?", exact=True).get_by_role( + "button", name="Continue Repeat for up to another 2 rounds", exact=True, + ).click() + expect(page.get_by_role("alert").filter(has_text="Repeat gate changed while you were reviewing")).to_be_visible() + assert not decision_writes(ui) + assert not ui.repeat_grants + + +@pytest.mark.parametrize("blocker", ["admissions", "deadline"]) +def test_repeat_grant_cannot_extend_global_admissions_or_deadline(workflow_repeat_ui, blocker): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + limits = ui.workflow_runtimes[runtime_key()]["limits"] + if blocker == "admissions": + limits["admitted_count"] = limits["max_executions"] + else: + limits["deadline_at"] = "2000-01-01T00:00:00Z" + open_repeat_run(ui) + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_be_disabled() + expect(page.get_by_role("alert").filter(has_text="exhausted" if blocker == "admissions" else "expired")).to_be_visible() + expect(page.get_by_role("button", name="Resume run", exact=True)).to_have_count(0) + assert not decision_writes(ui) + + +@pytest.mark.parametrize("code", ["deadline_exceeded", "execution_budget_exceeded"]) +def test_returned_cancel_only_budget_gate_overrides_retained_repeat_progress(workflow_repeat_ui, code): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + runtime = ui.workflow_runtimes[runtime_key()] + runtime["gate"] = { + "id": "global-budget-gate", "kind": "pause", "unit_id": REPEAT_ID, "reason_code": code, + "choices": ["cancel"], "reason": "The global run budget is exhausted. Cancel and start a new run.", + } + if code == "deadline_exceeded": + runtime["limits"]["deadline_at"] = "2000-01-01T00:00:00Z" + else: + runtime["limits"]["admitted_count"] = 5000 + open_repeat_run(ui) + expect(page.get_by_role("region", name="Repeat progress", exact=True)).to_contain_text("waiting manual continue") + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_have_count(0) + expect(page.get_by_role("button", name="Resume run", exact=True)).to_have_count(0) + page.get_by_role("button", name="Cancel run", exact=True).click() + expect(page.get_by_role("region", name="Repeat progress", exact=True)).to_contain_text("State: cancelled") + request = decision_writes(ui)[-1] + assert request.body["gate_id"] == "global-budget-gate" + assert request.body["choice"] == "cancel" + assert not ui.repeat_grants + + +@pytest.mark.parametrize("malformed", [ + "hybrid_path", "private_summary", "oversized_batch", "missing_policy", "resume_choice", "missing_summary", + "wrong_kind", "boolean_count", "missing_gate_id", "boolean_version", "lifetime_reset", +]) +def test_unsupported_repeat_gate_shapes_never_enable_decisions(workflow_repeat_ui, malformed): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + runtime = ui.workflow_runtimes[runtime_key()] + if malformed == "hybrid_path": + runtime["gate"]["iteration_path"] = [{"loop_id": REPEAT_ID, "iteration": 1, "item_id": "a" * 64}] + elif malformed == "private_summary": + runtime["gate"]["repeat"]["state_ref"] = "private-state-reference-must-not-render" + elif malformed == "oversized_batch": + runtime["gate"]["repeat"]["batch_size"] = 1001 + elif malformed == "missing_policy": + runtime["limits"].pop("max_repeat_iterations") + elif malformed == "resume_choice": + runtime["gate"]["choices"] = ["continue_repeat", "resume", "cancel"] + elif malformed == "missing_summary": + runtime["gate"].pop("repeat") + elif malformed == "wrong_kind": + runtime["gate"]["kind"] = "approval" + elif malformed == "boolean_count": + runtime["gate"]["repeat"]["completed_count"] = True + elif malformed == "missing_gate_id": + runtime["gate"].pop("id") + elif malformed == "boolean_version": + runtime["version"] = True + else: + runtime["gate"]["repeat"].update(completed_count=1001, completed_iteration=1000, next_iteration=1001) + open_repeat_run(ui) + expect(page.get_by_role("alert").filter(has_text=re.compile("unsupported|invalid frozen Repeat limits"))).to_be_visible() + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_have_count(0) + expect(page.get_by_role("button", name="Resume run", exact=True)).to_have_count(0) + expect(page.get_by_text("private-state-reference-must-not-render", exact=False)).to_have_count(0) + assert not decision_writes(ui) + + +@pytest.mark.parametrize("group", [False, True]) +def test_repeat_round_and_state_paging_reads_only_exact_selected_producers(workflow_repeat_ui, group): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui._seed_repeat_run("group" if group else "user", partial=True) + open_repeat_run(ui, group=group) + rounds = open_rounds(ui) + expect(rounds.get_by_role("list", name="Repeat rounds", exact=True).get_by_role("listitem")).to_have_count(1) + expect(rounds).to_contain_text("false (unmet)") + assert not any(request.path.endswith(("/state", "/records", "/result")) for request in ui.requests) + rounds.get_by_role("button", name="State after round 1", exact=True).click() + state = page.get_by_role("region", name="State after round 1", exact=True) + expect(state).to_contain_text("review-round-0 · attempt 2 · output json") + state.get_by_role("button", name="Load json output excerpt", exact=True).click() + expect(state).to_contain_text("Retained exact review") + result = [request for request in ui.requests if request.path.endswith("/result")][-1] + assert result.path.endswith("/executions/review-round-0/attempts/2/result") + assert result.query["output"] == ["json"] and result.query["limit"] == ["2000"] + assert result.query.get("group_id") == ([GROUP_ID] if group else None) + state.get_by_role("button", name="Next state page", exact=True).click() + expect(state).not_to_contain_text("Retained exact review") + expect(state).to_contain_text("accepted_partial") + expect(state).to_contain_text("original accepted records are retained") + expect(state.get_by_label("Retained prior coverage for state records", exact=True)).to_contain_text("complete: false") + records_slot = state.get_by_role("list", name="Saved state after round 1", exact=True).get_by_role("listitem").filter(has_text="records (records)").first + records_slot.get_by_role("button", name="Load complete records", exact=True).click() + saved = records_slot.get_by_role("list", name="Complete saved records", exact=True) + expect(saved.get_by_role("listitem")).to_have_count(100) + expect(saved).to_contain_text("Original finding 0") + records_slot.get_by_role("button", name="Next records page", exact=True).click() + expect(saved.get_by_role("listitem")).to_have_count(100) + expect(saved).to_contain_text("Original finding 100") + expect(saved).not_to_contain_text('"Original finding 0"') + records_slot.get_by_role("button", name="Inspect contributors", exact=True).click() + expect(records_slot.get_by_role("list", name="Saved collection contributors", exact=True)).to_contain_text("seed-records") + record_reads = [request for request in ui.requests if request.path.endswith("/records")] + assert len(record_reads) == 2 + assert all(request.query["limit"] == ["100"] and request.query["output"] == ["records"] for request in record_reads) + rounds.get_by_role("button", name="Next rounds page", exact=True).click() + expect(page.get_by_role("region", name="State after round 1", exact=True)).to_have_count(0) + expect(rounds.get_by_role("list", name="Repeat rounds", exact=True)).to_contain_text("Round 2") + assert not decision_writes(ui) + + +def test_lifetime_round_1001_remains_distinct_and_uncommitted_after_state_is_not_empty_success(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui._seed_repeat_run("user", completed_count=1000, batch_size=1000, running_round=True, partial=True) + open_repeat_run(ui, width=390, height=844, theme="dark") + progress = page.get_by_role("region", name="Repeat progress", exact=True) + expect(progress).to_contain_text("Lifetime completed rounds: 1000") + expect(progress).to_contain_text("Automatic batch 2: 1 of 1000 rounds admitted") + expect(progress).to_contain_text("Next lifetime round: 1001") + rounds = open_rounds(ui) + for _ in range(20): + rounds.get_by_role("button", name="Next rounds page", exact=True).click() + expect(rounds.get_by_role("list", name="Repeat rounds", exact=True).get_by_role("listitem")).to_have_count(1) + expect(rounds).to_contain_text("Round 1001") + expect(rounds).to_contain_text("Not evaluated") + rounds.get_by_role("button", name="State after round 1001", exact=True).click() + state = page.get_by_role("region", name="State after round 1001", exact=True) + expect(state).to_contain_text("not committed or available yet") + expect(state).to_contain_text("not an empty eligible result") + expect(state.get_by_role("button", name=re.compile("^Load .* output excerpt$"))).to_have_count(0) + decisions = page.get_by_role("list", name="Runtime decisions", exact=True) + expect(decisions).to_contain_text("Manual continuation") + expect(decisions).to_contain_text("workspace-editor-user") + expect(decisions).to_contain_text("fixture-repeat-grant") + assert len([request for request in ui.requests if request.path.endswith("/iterations")]) == 21 + assert all(request.query["limit"] == ["50"] for request in ui.requests if request.path.endswith("/iterations")) + assert not decision_writes(ui) + ui.assert_no_overflow() + + +@pytest.mark.parametrize("malformed", [ + "hybrid_path", "predicate_string", "batch_reset", "private_ref", "too_many", + "summary_state", "state_array", "missing_snapshot_flag", +]) +def test_repeat_iteration_metadata_is_strict_and_never_loads_saved_values_on_failure(workflow_repeat_ui, malformed): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + pages = ui.repeat_iteration_pages[runtime_key()] + rows = pages[""]["items"] + if malformed == "hybrid_path": + rows[0]["iteration_path"][0]["index"] = 0 + elif malformed == "predicate_string": + rows[0]["condition_result"] = "false" + elif malformed == "batch_reset": + rows[0]["iteration"] = 1000 + rows[0]["iteration_path"][0]["iteration"] = 1000 + elif malformed == "private_ref": + rows[0]["after_state_ref"] = "private-state-reference" + elif malformed == "summary_state": + rows[0]["state"] = "waiting_manual_continue" + elif malformed == "state_array": + rows[0]["state"] = ["completed"] + elif malformed == "missing_snapshot_flag": + ui.repeat_iterations_override = { + "repeat_execution_id": REPEAT_EXECUTION_ID, + "repeat": copy.deepcopy(ui.workflow_runtimes[runtime_key()]["repeat_progress"]), + "iterations": copy.deepcopy(rows), "next_cursor": pages[""]["next_cursor"], + "total_count": sum(len(value["items"]) for value in pages.values()), + } + else: + rows.extend(copy.deepcopy(rows[0]) for _ in range(50)) + open_repeat_run(ui) + rounds = open_rounds(ui) + expect(rounds.get_by_role("alert").filter(has_text="unsupported response")).to_be_visible() + expect(rounds.get_by_role("list", name="Repeat rounds", exact=True).get_by_role("listitem")).to_have_count(0) + assert not any(request.path.endswith(("/state", "/result", "/records")) for request in ui.requests) + + +@pytest.mark.parametrize("malformed", [ + "phase", "execution", "private_ref", "any_kind", "validation", "prior_coverage", "duplicate", + "missing_partial_flag", "missing_snapshot_flag", "per_slot_partial", "unavailable_before", +]) +def test_state_slot_metadata_rejects_wrong_selectors_and_unsupported_shapes(workflow_repeat_ui, malformed): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + states = ui._state_slots(0, "before", False)[:2] + response = { + "iteration": 0, "phase": "before", "repeat_execution_id": REPEAT_EXECUTION_ID, + "available": True, "total_count": 4, "states": states, "next_cursor": "offset-2", + "partial": False, "source_snapshot_changed": False, + } + if malformed == "phase": + response["phase"] = "after" + elif malformed == "execution": + response["repeat_execution_id"] = "another-execution" + elif malformed == "private_ref": + states[0]["source"]["result_ref"] = "private-result-reference" + elif malformed == "any_kind": + states[0]["kind"] = "any" + elif malformed == "validation": + states[0]["workflow_validation"]["eligible"] = "true" + elif malformed == "prior_coverage": + states[0]["prior_coverage"] = {"result_ref": {"id": "private-result-reference"}} + elif malformed == "missing_partial_flag": + response.pop("partial") + elif malformed == "missing_snapshot_flag": + response.pop("source_snapshot_changed") + elif malformed == "per_slot_partial": + states[0]["partial"] = True + elif malformed == "unavailable_before": + response.update({"available": False, "states": [], "total_count": 0, "next_cursor": None}) + response.pop("partial") + response.pop("source_snapshot_changed") + else: + states[1] = copy.deepcopy(states[0]) + ui.repeat_state_override = response + open_repeat_run(ui) + rounds = open_rounds(ui) + rounds.get_by_role("button", name="State before round 1", exact=True).click() + state = page.get_by_role("region", name="State before round 1", exact=True) + expect(state.get_by_role("alert").filter(has_text=re.compile("unsupported|conflicting identities"))).to_be_visible() + expect(state.get_by_role("list", name="Saved state before round 1", exact=True).get_by_role("listitem")).to_have_count(0) + assert not any(request.path.endswith(("/result", "/records")) for request in ui.requests) + + +@pytest.mark.parametrize("flag", ["partial", "source_snapshot_changed"]) +def test_uncommitted_after_state_rejects_committed_only_flags(workflow_repeat_ui, flag): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui._seed_repeat_run("user", completed_count=1, batch_size=2, running_round=True) + ui.repeat_state_override = { + "repeat_execution_id": REPEAT_EXECUTION_ID, "iteration": 1, "phase": "after", + "available": False, "states": [], "next_cursor": None, "total_count": 0, flag: False, + } + open_repeat_run(ui) + rounds = open_rounds(ui) + rounds.get_by_role("button", name="Next rounds page", exact=True).click() + rounds.get_by_role("button", name="State after round 2", exact=True).click() + state = page.get_by_role("region", name="State after round 2", exact=True) + expect(state.get_by_role("alert").filter(has_text="unsupported response")).to_be_visible() + expect(state.get_by_role("button", name=re.compile("^Load .* output excerpt$"))).to_have_count(0) + assert not any(request.path.endswith(("/result", "/records")) for request in ui.requests) + + +def test_revoked_repeat_state_refresh_clears_cached_metadata_and_content(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui) + open_rounds(ui).get_by_role("button", name="State before round 1", exact=True).click() + state = page.get_by_role("region", name="State before round 1", exact=True) + state.get_by_role("button", name="Load text output excerpt", exact=True).click() + expect(state).to_contain_text("Retained exact draft") + ui.reject_next( + "GET", f"/api/user/workflows/{REPEAT_WORKFLOW_ID}/runs/{REPEAT_RUN_ID}/executions/{REPEAT_EXECUTION_ID}/iterations/0/state", + status=403, error="Source access could not be confirmed.", + ) + state.get_by_role("button", name="Refresh", exact=True).click() + expect(page.get_by_role("alert").filter(has_text="no longer have access")).to_be_visible() + expect(page.get_by_text("Retained exact draft", exact=False)).to_have_count(0) + expect(page.get_by_role("list", name="Saved state before round 1", exact=True)).to_have_count(0) + + +def test_repeat_final_named_records_use_existing_exact_reader(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.complete_with_records_export() + open_repeat_run(ui) + rounds = open_rounds(ui) + rounds.get_by_role("button", name="Next rounds page", exact=True).click() + expect(rounds).to_contain_text("true (satisfied)") + rounds.get_by_role("button", name="Inspect Repeat final outputs", exact=True).click() + records = rounds.get_by_role("region", name="Complete record inspection", exact=True) + expect(records.get_by_label("Complete records output", exact=True)).to_have_value("findings") + records.get_by_role("button", name="Load complete records", exact=True).click() + expect(records.get_by_role("list", name="Complete saved records", exact=True)).to_contain_text("Original finding 0") + request = [request for request in ui.requests if request.path.endswith("/records")][-1] + assert request.path.endswith(f"/executions/{REPEAT_EXECUTION_ID}/attempts/1/records") + assert request.query["output"] == ["findings"] + assert not ui.writes + + +@pytest.mark.parametrize("kind", ["records", "document_results"]) +def test_current_repeat_collection_can_be_selected_for_saved_record_reporting(workflow_repeat_ui, kind): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.personal_workflows[REPEAT_WORKFLOW_ID] = repeat_workflow_record(collections=True) + open_repeat(ui) + revise = task_block(page, "Revise draft") + details(revise) + revise.get_by_role("button", name="Add revise draft inputs input", exact=True).click() + revise.get_by_label("Revise draft inputs input 3 source", exact=True).select_option("repeat_state") + revise.get_by_label("Revise draft inputs input 3 state", exact=True).select_option(kind) + revise.get_by_label("Large saved inputs", exact=True).select_option("saved_record_report") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + saved = ui.workflow_writes[-1].body["tasks"][2] + assert saved["input_processing"] == "saved_record_report" + assert saved["inputs"][2]["source"] == state_binding("unused", kind, kind)["source"] + assert saved["inputs"][2]["expected_kind"] == kind + assert not any(request.path.endswith("/run") for request in ui.writes) + + +@pytest.mark.parametrize("kind", ["records", "document_results"]) +def test_for_each_freezes_current_repeat_collection_with_typed_item_fields(workflow_repeat_ui, kind): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + record = repeat_workflow_record(collections=True) + loop_id = "review-saved-items" + output = "documents" if kind == "document_results" else "records" + record["flow"]["nodes"][-1]["body"]["nodes"].insert(0, { + "id": loop_id, "kind": "for_each", "item_key": "source_identity", "max_items": 5, + "iterable": {"kind": "input", "name": "rows"}, + "inputs": [flow_binding("rows", f"seed-{kind}", output, kind=kind)], + "body": {"id": "saved-item-body", "outputs": [], "nodes": [{ + "id": "row-filter", "kind": "if", + "inputs": [{ + "name": "item", "source": {"kind": "loop_item", "loop_id": loop_id, "scope": "current"}, + "required": True, "expected_kind": "json", "allow_partial": False, + }], + "condition": {"op": "eq", "left": {"input": "item", "path": "/value/finding"}, "right": {"literal": "review"}}, + "then": {"id": "matching-items", "nodes": []}, + "else": {"id": "other-items", "nodes": []}, + "join": {"id": "row-filter-join", "exports": []}, + }]}, + }) + ui.personal_workflows[REPEAT_WORKFLOW_ID] = record + open_repeat(ui) + loop = page.get_by_role("region", name="For each block", exact=True) + loop.get_by_label("Saved collection output", exact=True).select_option( + label=f"Current Repeat {REPEAT_ID} state {kind} ({kind.replace('_', ' ')})", + ) + expect(loop).to_contain_text("start of this Repeat round") + expect(loop.get_by_label("If / else condition left field", exact=True)).to_have_value("/value/finding") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + saved_loop = ui.workflow_writes[-1].body["flow"]["nodes"][-1]["body"]["nodes"][0] + assert saved_loop["inputs"] == [state_binding("rows", kind, kind)] + assert saved_loop["body"]["nodes"][0]["condition"]["left"]["path"] == "/value/finding" + + +def test_nested_repeat_initial_state_is_explicitly_named_outer_state(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + record = repeat_workflow_record() + outer = record["flow"]["nodes"][-1] + inner = copy.deepcopy(outer) + inner["id"] = "nested-refinement" + inner["body"]["id"] = "nested-refinement-body" + for slot in inner["state"]: + slot["initial"] = state_binding(slot["name"], slot["name"], slot["output_contract"]["kind"])["source"] + for binding in record["tasks"][2]["inputs"]: + binding["source"]["loop_id"] = inner["id"] + outer["body"]["nodes"] = [inner] + outer["body"]["outputs"] = [ + flow_binding("next_draft", inner["id"], "report", kind="text"), + flow_binding("next_review", inner["id"], "review", kind="json"), + ] + ui.personal_workflows[REPEAT_WORKFLOW_ID] = record + open_repeat(ui) + nested = repeat_block(page).last + expect(nested.get_by_label("State 1 initial source", exact=True)).to_have_value("repeat_state") + expect(nested.get_by_label("State 1 initial Repeat", exact=True)).to_have_value(REPEAT_ID) + expect(nested.get_by_label("State 1 initial state", exact=True)).to_have_value("draft") + nested.get_by_label("Maximum rounds before manual continuation", exact=True).fill("3") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + inner["max_iterations"] = 3 + assert ui.workflow_writes[-1].body["flow"] == record["flow"] + + +def test_permission_loss_closes_open_repeat_confirmation_without_a_grant(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui, group=True) + page.get_by_role("button", name="Continue Repeat", exact=True).click() + dialog = page.get_by_role("dialog", name="Continue Repeat?", exact=True) + expect(dialog).to_be_visible() + ui.runtime_can_decide[runtime_key(True)] = False + expect(dialog).to_have_count(0, timeout=10000) + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_have_count(0) + assert not decision_writes(ui) + assert not ui.repeat_grants + + +def test_repeat_body_attempt_inspection_keeps_round_and_retry_identity(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui) + rounds = open_rounds(ui) + rounds.get_by_role("button", name="Inspect round execution review-round-0", exact=True).click() + attempts = rounds.get_by_role("list", name="Attempts for execution review-round-0", exact=True) + expect(attempts).to_contain_text("Attempt 1") + expect(attempts).to_contain_text("Attempt 2") + expect(attempts).to_contain_text(f"{REPEAT_ID} (round 1)") + expect(attempts.get_by_text("No result was committed for this attempt.", exact=True)).to_be_visible() + attempts.get_by_role("button", name="Load authoritative output excerpt", exact=True).click() + expect(attempts).to_contain_text("Retained exact review") + request = [request for request in ui.requests if request.path.endswith("/result")][-1] + assert request.path.endswith("/executions/review-round-0/attempts/2/result") + assert request.query["output"] == ["authoritative"] + assert not decision_writes(ui) + + +@pytest.mark.parametrize("group", [False, True]) +def test_repeat_body_instances_drill_into_nested_rounds_and_frozen_items(workflow_repeat_ui, group): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.add_nested_inspection("group" if group else "user") + open_repeat_run(ui, group=group) + outer = open_rounds(ui) + outer.get_by_role("button", name="Inspect round execution nested-repeat-execution", exact=True).click() + expect(page.get_by_role("list", name="Attempts for execution nested-repeat-execution", exact=True)).to_be_visible() + assert not any("/nested-repeat-execution/iterations" in request.path for request in ui.requests) + outer.get_by_role("button", name="Show Repeat rounds for nested-repeat-execution", exact=True).click() + nested = page.get_by_role("region", name="Repeat round inspection", exact=True).last + expect(nested).to_contain_text(f"{REPEAT_ID} (round 1)") + expect(nested).to_contain_text("nested-refinement (round 1)") + expect(nested).to_contain_text("true (satisfied)") + nested.get_by_role("button", name="State before round 1", exact=True).click() + expect(nested.get_by_role("region", name="State before round 1", exact=True)).to_contain_text("seed-draft") + outer.get_by_role("button", name="Inspect round execution nested-each-execution", exact=True).click() + expect(page.get_by_role("region", name="Repeat round inspection", exact=True)).to_have_count(1) + assert not any("/nested-each-execution/items" in request.path for request in ui.requests) + outer.get_by_role("button", name="Show frozen items for nested-each-execution", exact=True).click() + items = outer.get_by_role("region", name="Frozen item inspection", exact=True) + expect(items).to_contain_text("Exact saved finding") + expect(items).to_contain_text(f"{REPEAT_ID} (round 1)") + reads = [request for request in ui.requests if + "/nested-repeat-execution/iterations" in request.path or "/nested-each-execution/items" in request.path] + assert len(reads) == 3 + assert all(request.query["limit"] == ["50"] for request in reads) + assert all(request.query.get("group_id") == ([GROUP_ID] if group else None) for request in reads) + assert not any(request.path.endswith(("/result", "/records")) for request in ui.requests) + assert not ui.writes + + +def test_invalid_repeat_poll_removes_previous_summary_and_confirmation(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat_run(ui) + page.get_by_role("button", name="Continue Repeat", exact=True).click() + dialog = page.get_by_role("dialog", name="Continue Repeat?", exact=True) + expect(dialog).to_be_visible() + ui.workflow_runtimes[runtime_key()]["gate"]["repeat"]["batch_usage"] = "2" + expect(dialog).to_have_count(0, timeout=10000) + expect(page.get_by_role("alert").filter(has_text="unsupported Repeat")).to_be_visible() + expect(page.get_by_role("region", name="Repeat progress", exact=True)).to_have_count(0) + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_have_count(0) + assert not decision_writes(ui) + + +def test_mixed_repeat_and_item_body_gate_keeps_exact_identity_without_granting_another_batch(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui._seed_repeat_run("user", completed_count=1000, batch_size=1000, running_round=True) + runtime = ui.workflow_runtimes[runtime_key()] + before = copy.deepcopy(runtime["repeat_progress"]) + runtime["state"] = "waiting_approval" + runtime["gate"] = { + "id": "repeat-body-approval", "kind": "approval", "unit_id": "review", "node_id": "review", + "execution_id": "review-round-1000", "attempt": 2, "choices": ["approve", "reject"], + "iteration_path": [ + {"loop_id": REPEAT_ID, "iteration": 1000}, + {"loop_id": "each-finding", "item_id": "d" * 64, "index": 7}, + ], + } + ui.workflow_runs[REPEAT_WORKFLOW_ID][0]["status"] = "waiting_approval" + open_repeat_run(ui) + expect(page.get_by_text(re.compile(r"Execution review-round-1000.*round 1001"))).to_be_visible() + expect(page.get_by_role("button", name="Continue Repeat", exact=True)).to_have_count(0) + page.get_by_role("button", name="Approve task", exact=True).click() + expect(page.get_by_role("button", name="Approve task", exact=True)).to_have_count(0) + request = decision_writes(ui)[-1] + assert request.body["gate_id"] == "repeat-body-approval" and request.body["choice"] == "approve" + assert request.body["expected_version"] == 10 + assert ui.workflow_runtimes[runtime_key()]["repeat_progress"] == before + assert not ui.repeat_grants + + +@pytest.mark.parametrize("invalid", ["initial_kind", "next_kind", "optional_next"]) +def test_repeat_state_contracts_cannot_coerce_or_silently_omit_atomic_next_slots(workflow_repeat_ui, invalid): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + record = ui.personal_workflows[REPEAT_WORKFLOW_ID] + node = record["flow"]["nodes"][-1] + if invalid == "initial_kind": + node["state"][1]["initial"] = copy.deepcopy(node["state"][0]["initial"]) + elif invalid == "next_kind": + node["state"][1]["next"] = "next_draft" + else: + node["body"]["outputs"][1]["required"] = False + original = copy.deepcopy(record) + open_repeat(ui) + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("alert").filter(has_text="required, declared next body output" if invalid == "optional_next" else "exactly kind json")).to_be_visible() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_be_visible() + assert not ui.workflow_writes + assert ui.personal_workflows[REPEAT_WORKFLOW_ID] == original + + +@pytest.mark.parametrize("maximum", [1, 1000]) +def test_explicit_supported_batch_boundaries_save_exactly(workflow_repeat_ui, maximum): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.repeat_ceiling = 1000 + repeat = open_repeat(ui) + repeat.get_by_label("Maximum rounds before manual continuation", exact=True).fill(str(maximum)) + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + assert ui.workflow_writes[-1].body["flow"]["nodes"][-1]["max_iterations"] == maximum + + +def test_partial_current_state_pass_through_requires_explicit_opt_in(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.personal_workflows[REPEAT_WORKFLOW_ID] = repeat_workflow_record(collections=True) + repeat = open_repeat(ui) + expect(repeat.get_by_label("State 3 allow partial", exact=True)).not_to_be_checked() + repeat.get_by_label("State 3 allow partial", exact=True).check() + repeat.get_by_label("Repeat body outputs input 3 allow partial", exact=True).check() + expect(repeat.get_by_text("Partial coverage and limitations stay attached", exact=False)).to_be_visible() + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + node = ui.workflow_writes[-1].body["flow"]["nodes"][-1] + assert node["state"][2]["output_contract"]["allow_partial"] is True + assert node["body"]["outputs"][2]["source"] == { + "kind": "repeat_state", "loop_id": REPEAT_ID, "state_name": "records", "scope": "current", + } + assert node["body"]["outputs"][2]["allow_partial"] is True + assert node["state"][3]["output_contract"]["allow_partial"] is False + + +@pytest.mark.parametrize("unsupported", ["missing_maximum", "literal_initial", "any_state", "future_field", "future_predicate", "state_source_field"]) +def test_unsupported_repeat_definitions_stay_intact_and_read_only(workflow_repeat_ui, unsupported): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + record = ui.personal_workflows[REPEAT_WORKFLOW_ID] + node = record["flow"]["nodes"][-1] + if unsupported == "missing_maximum": + del node["max_iterations"] + elif unsupported == "literal_initial": + node["state"][0]["initial"] = {"kind": "literal", "value": "not an admitted saved output"} + elif unsupported == "any_state": + node["state"][0]["output_contract"]["kind"] = "any" + elif unsupported == "future_field": + node["automatic_continue"] = True + elif unsupported == "future_predicate": + node["until"]["expression"] = "unsupported control language" + else: + record["tasks"][2]["inputs"][0]["source"]["iteration"] = 42 + original = copy.deepcopy(record) + ui.open(f"/workspace/workflows?workflow_id={REPEAT_WORKFLOW_ID}") + expect(page.get_by_role("alert").filter(has_text="cannot safely save")).to_be_visible() + expect(page.get_by_role("button", name="Save workflow", exact=True)).to_have_count(0) + assert ui.personal_workflows[REPEAT_WORKFLOW_ID] == original + assert not ui.workflow_writes + + +@pytest.mark.parametrize("missing", ["all", "policy", "hard_ceiling"]) +def test_missing_repeat_capabilities_preserve_saved_definition_read_only(workflow_repeat_ui, missing): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + original = copy.deepcopy(ui.personal_workflows[REPEAT_WORKFLOW_ID]) + if missing == "all": + ui.repeat_capabilities = False + elif missing == "policy": + ui.repeat_ceiling = None + else: + ui.repeat_hard_ceiling = None + ui.open(f"/workspace/workflows?workflow_id={REPEAT_WORKFLOW_ID}") + expect(page.get_by_role("alert").filter(has_text="cannot safely save")).to_be_visible() + expect(page.get_by_role("button", name="Save workflow", exact=True)).to_have_count(0) + expect(page.get_by_role("button", name="Add Repeat until to Main", exact=True)).to_have_count(0) + assert ui.personal_workflows[REPEAT_WORKFLOW_ID] == original + assert not ui.workflow_writes + + +@pytest.mark.parametrize("hard_ceiling", [0, 999, 1001, "1000", True]) +def test_invalid_advertised_repeat_hard_ceiling_never_enables_authoring(workflow_repeat_ui, hard_ceiling): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + original = copy.deepcopy(ui.personal_workflows[REPEAT_WORKFLOW_ID]) + ui.repeat_hard_ceiling = hard_ceiling + ui.open(f"/workspace/workflows?workflow_id={REPEAT_WORKFLOW_ID}") + expect(page.get_by_role("alert").filter(has_text="invalid loop capabilities or limits")).to_be_visible() + expect(page.get_by_role("button", name="Save workflow", exact=True)).to_have_count(0) + expect(page.get_by_role("button", name="Add Repeat until to Main", exact=True)).to_have_count(0) + assert ui.personal_workflows[REPEAT_WORKFLOW_ID] == original + assert not ui.workflow_writes + + +def test_lowered_repeat_policy_preserves_schema_valid_authored_maximum(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + ui.personal_workflows[REPEAT_WORKFLOW_ID]["flow"]["nodes"][-1]["max_iterations"] = 30 + original = copy.deepcopy(ui.personal_workflows[REPEAT_WORKFLOW_ID]) + repeat = open_repeat(ui) + maximum = repeat.get_by_label("Maximum rounds before manual continuation", exact=True) + expect(maximum).to_have_value("30") + expect(maximum).to_be_enabled() + expect(page.get_by_role("alert").filter(has_text="cannot safely save")).to_have_count(0) + expect(repeat.get_by_role("alert")).to_contain_text("administrator ceiling of 25 for new runs") + expect(repeat.get_by_role("alert")).to_contain_text("authored value is preserved") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(maximum).to_have_value("30") + assert ui.personal_workflows[REPEAT_WORKFLOW_ID] == original + assert not ui.workflow_writes + maximum.fill("25") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("dialog", name="Edit workflow", exact=True)).to_have_count(0) + assert ui.workflow_writes[-1].body["flow"]["nodes"][-1]["max_iterations"] == 25 + + +def test_repeat_inherited_runner_and_body_tasks_disallow_hosted_without_hiding_outside_choices(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + open_repeat(ui) + page.get_by_label("Runner type", exact=True).select_option("agent") + hosted = page.get_by_label("Agent", exact=True).locator("option").filter(has_text="Hosted reviewer") + expect(hosted).to_have_attribute("disabled", "") + body = task_block(page, "Revise draft") + details(body) + body.get_by_label("Task runner", exact=True).select_option("agent") + expect(body.get_by_label("Task agent", exact=True).locator("option").filter(has_text="Hosted reviewer")).to_have_attribute("disabled", "") + outside = task_block(page, "Seed draft") + details(outside) + outside.get_by_label("Task runner", exact=True).select_option("agent") + expect(outside.get_by_label("Task agent", exact=True).locator("option").filter(has_text="Hosted reviewer")).not_to_have_attribute("disabled", "") + assert not ui.workflow_writes + + +def test_state_rename_retains_dependent_bindings_and_reports_missing_slot(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + repeat = open_repeat(ui) + repeat.get_by_label("State 1 name", exact=True).fill("renamed") + revise = task_block(page, "Revise draft") + details(revise) + expect(revise.get_by_label("Revise draft inputs input 1 state", exact=True)).to_have_value("draft") + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("alert").filter(has_text="named current state slot")).to_be_visible() + assert not ui.workflow_writes + + +def test_stale_save_keeps_repeat_draft_and_exact_next_state(workflow_repeat_ui): + ui, page = workflow_repeat_ui, workflow_repeat_ui.page + repeat = open_repeat(ui) + repeat.get_by_label("Maximum rounds before manual continuation", exact=True).fill("7") + ui.mutate_revision(REPEAT_WORKFLOW_ID) + page.get_by_role("button", name="Save workflow", exact=True).click() + expect(page.get_by_role("alert").filter(has_text="draft has been retained")).to_be_visible() + expect(repeat.get_by_label("Maximum rounds before manual continuation", exact=True)).to_have_value("7") + expect(repeat.get_by_label("State 1 next body output", exact=True)).to_have_value("next_draft") + assert not ui.workflow_writes diff --git a/ui_tests/test_workflow_loop_admin_limits.py b/ui_tests/test_workflow_loop_admin_limits.py index e6324b641..15ac7ed8d 100644 --- a/ui_tests/test_workflow_loop_admin_limits.py +++ b/ui_tests/test_workflow_loop_admin_limits.py @@ -1,15 +1,17 @@ # test_workflow_loop_admin_limits.py """ -Source-backed browser tests for Classic/V2 workflow loop admission limits. -Version: 0.261.117 +Source-backed browser tests for Classic/V2 For-each and Repeat policy limits. +Version: 0.261.120 Implemented in: 0.261.117 The actual Classic pane, V2 SPA, field registry and admin patch normalizer run against intercepted APIs. No live settings, Azure resource, or model is used. +Repeat-until coverage was added in 0.261.120. """ import sys from pathlib import Path +from typing import NamedTuple import pytest from playwright.sync_api import expect @@ -28,6 +30,30 @@ pytestmark = pytest.mark.ui +class LimitPolicy(NamedTuple): + key: str + label: str + default: int + maximum: int + help_id: str + help_text: tuple[str, ...] + + +@pytest.fixture(params=[ + pytest.param(LimitPolicy( + "workflow_max_loop_items", "Workflow Loop Item Limit", 500, 5000, + "workflow-max-loop-items-help", ("actual items", "Active runs", "never truncated"), + ), id="for-each"), + pytest.param(LimitPolicy( + "workflow_max_repeat_iterations", "Workflow Repeat Iteration Limit", 25, 1000, + "workflow-max-repeat-iterations-help", + ("automatic Repeat until batch", "per-block maximum", "never shortened", "manual continuation", "admitted limit"), + ), id="repeat-until"), +]) +def workflow_limit(request): + return request.param + + @pytest.fixture def loop_admin_ui(page): fixture = WorkflowAdminLimitsFixture(page) @@ -35,26 +61,27 @@ def loop_admin_ui(page): fixture.assert_clean() -@pytest.mark.parametrize("configured", [None, 5000]) -def test_classic_loop_limit_default_bounds_and_keyboard(loop_admin_ui, configured): +@pytest.mark.parametrize("configured", ["absent", "maximum"]) +def test_classic_workflow_limit_default_bounds_and_keyboard(loop_admin_ui, workflow_limit, configured): ui, page = loop_admin_ui, loop_admin_ui.page - if configured is None: - ui.settings.pop("workflow_max_loop_items", None) + policy = workflow_limit + if configured == "absent": + ui.settings.pop(policy.key, None) else: - ui.settings["workflow_max_loop_items"] = configured + ui.settings[policy.key] = policy.maximum ui.open_workflow(classic=True, width=390) - field = page.get_by_label("Workflow Loop Item Limit", exact=True) - expect(field).to_have_value(str(configured if configured is not None else 500)) + field = page.get_by_label(policy.label, exact=True) + expect(field).to_have_value(str(policy.default if configured == "absent" else policy.maximum)) expect(field).to_have_attribute("type", "number") - expect(field).to_have_attribute("name", "workflow_max_loop_items") + expect(field).to_have_attribute("name", policy.key) expect(field).to_have_attribute("min", "1") - expect(field).to_have_attribute("max", "5000") + expect(field).to_have_attribute("max", str(policy.maximum)) expect(field).to_have_attribute("step", "1") expect(field).to_have_attribute("required", "") - expect(page.locator("#workflow-max-loop-items-help")).to_contain_text("actual items") - expect(page.locator("#workflow-max-loop-items-help")).to_contain_text("Active runs") - expect(page.locator("#workflow-max-loop-items-help")).to_contain_text("never truncated") - for invalid in ("0", "5001", "1.5", ""): + expect(field).to_have_attribute("aria-describedby", policy.help_id) + for text in policy.help_text: + expect(page.locator(f"#{policy.help_id}")).to_contain_text(text) + for invalid in ("0", str(policy.maximum + 1), "1.5", ""): field.fill(invalid) assert not field.evaluate("element => element.checkValidity()") field.fill("1") @@ -66,52 +93,55 @@ def test_classic_loop_limit_default_bounds_and_keyboard(loop_admin_ui, configure assert ui.patches == [] -def test_v2_loop_limit_registry_mobile_keyboard_and_narrow_patch(loop_admin_ui): +def test_v2_workflow_limit_registry_mobile_keyboard_and_narrow_patch(loop_admin_ui, workflow_limit): ui, page = loop_admin_ui, loop_admin_ui.page + policy = workflow_limit ui.open_workflow(width=390) - field = page.get_by_label("Workflow Loop Item Limit", exact=True) - expect(field).to_have_value("500") + field = page.get_by_label(policy.label, exact=True) + expect(field).to_have_value(str(policy.default)) expect(field).to_have_attribute("min", "1") - expect(field).to_have_attribute("max", "5000") + expect(field).to_have_attribute("max", str(policy.maximum)) expect(field).to_have_attribute("step", "1") field.focus() field.press("ArrowDown") - expect(field).to_have_value("499") + expect(field).to_have_value(str(policy.default - 1)) save = page.get_by_role("button", name="Save changes", exact=True) save.focus() save.press("Enter") expect(save).to_have_count(0) - assert ui.patches == [{"workflow_max_loop_items": 499}] - assert ui.settings["workflow_max_loop_items"] == 499 + assert ui.patches == [{policy.key: policy.default - 1}] + assert ui.settings[policy.key] == policy.default - 1 page.reload(wait_until="networkidle") - expect(field).to_have_value("499") + expect(field).to_have_value(str(policy.default - 1)) assert page.evaluate("document.documentElement.scrollWidth <= window.innerWidth") -def test_v2_invalid_loop_limit_retains_draft_and_existing_policy(loop_admin_ui): +def test_v2_invalid_workflow_limit_retains_draft_and_existing_policy(loop_admin_ui, workflow_limit): ui, page = loop_admin_ui, loop_admin_ui.page + policy = workflow_limit ui.open_workflow() - field = page.get_by_label("Workflow Loop Item Limit", exact=True) - field.fill("5001") + field = page.get_by_label(policy.label, exact=True) + field.fill(str(policy.maximum + 1)) page.get_by_role("button", name="Save changes", exact=True).click() - expect(page.get_by_role("alert").filter(has_text="5,000")).to_be_visible() - expect(field).to_have_value("5001") - assert ui.settings["workflow_max_loop_items"] == 500 + expect(page.get_by_role("alert").filter(has_text=f"{policy.maximum:,}")).to_be_visible() + expect(field).to_have_value(str(policy.maximum + 1)) + assert ui.settings[policy.key] == policy.default page.get_by_role("button", name="Discard", exact=True).click() - expect(field).to_have_value("500") - field.fill("5000") + expect(field).to_have_value(str(policy.default)) + field.fill(str(policy.maximum)) page.get_by_role("button", name="Save changes", exact=True).click() expect(page.get_by_role("button", name="Save changes", exact=True)).to_have_count(0) - assert ui.settings["workflow_max_loop_items"] == 5000 + assert ui.settings[policy.key] == policy.maximum -def test_v2_unrelated_patch_preserves_an_absent_loop_limit(loop_admin_ui): +def test_v2_unrelated_patch_preserves_an_absent_workflow_limit(loop_admin_ui, workflow_limit): ui, page = loop_admin_ui, loop_admin_ui.page - ui.settings.pop("workflow_max_loop_items", None) + policy = workflow_limit + ui.settings.pop(policy.key, None) ui.open_workflow() - expect(page.get_by_label("Workflow Loop Item Limit", exact=True)).to_have_value("500") + expect(page.get_by_label(policy.label, exact=True)).to_have_value(str(policy.default)) page.get_by_label("Workflow Task Limit", exact=True).fill("51") page.get_by_role("button", name="Save changes", exact=True).click() expect(page.get_by_role("button", name="Save changes", exact=True)).to_have_count(0) assert ui.patches == [{"workflow_max_tasks": 51}] - assert "workflow_max_loop_items" not in ui.settings + assert policy.key not in ui.settings