From da5b2d4cec50e77f158a29df676204e1a0247562 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 27 Oct 2026 16:27:47 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20ms-swift=E6=A0=BC=E5=BC=8F=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=A2=9E=E5=8A=A0=E6=A0=BC=E5=BC=8F=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 6 +- dingo/data/converter/ms_swift.py | 164 +++------ dingo/data/converter/ms_swift_schema.py | 348 ++++++++++++++++++ docs/agent_trace/ms_swift.md | 15 +- test/__init__.py | 1 + test/scripts/__init__.py | 1 + .../dataset/test_ms_swift_agent_metrics.py | 79 ++-- .../dataset/test_ms_swift_converter.py | 89 +++-- test/scripts/dataset/test_ms_swift_schema.py | 160 ++++++++ test/scripts/exec/test_local.py | 19 +- test/scripts/ms_swift_test_utils.py | 67 ++++ 11 files changed, 756 insertions(+), 193 deletions(-) create mode 100644 dingo/data/converter/ms_swift_schema.py create mode 100644 test/__init__.py create mode 100644 test/scripts/__init__.py create mode 100644 test/scripts/dataset/test_ms_swift_schema.py create mode 100644 test/scripts/ms_swift_test_utils.py diff --git a/AGENTS.md b/AGENTS.md index 6e862bf5..d708ae7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,8 @@ dingo/ │ │ ├── dataset/ ← Dataset implementations per source │ │ ├── searcher.py ← Searcher protocol + SearchResult + registry (mock, ES) │ │ └── converter/ ← Format converters (JSON, JSONL, CSV, Parquet, MinerU, etc.) +│ │ ├── ms_swift.py ← ms-swift evaluator-field preparation +│ │ └── ms_swift_schema.py ← Full standardized-delivery validation │ ├── model/ │ │ ├── model.py ← Model registry (rule_register, llm_register) │ │ ├── rule/ ← Rule-based evaluators (80+ built-in) @@ -171,7 +173,9 @@ input_data = { MinerU formats automatically flatten blocks into `Data` objects, preserving all original fields (type, bbox, page_idx, etc.) via Pydantic `extra="allow"`. Use `dataset.mineru_config.include_types` to filter by block type. -The `ms_swift` converter lives in `dingo/data/converter/ms_swift.py`. It prepares +The `ms_swift` converter lives in `dingo/data/converter/ms_swift.py` and validates +each record against the full standardized delivery contract in +`dingo/data/converter/ms_swift_schema.py` before preparing any derived fields. It prepares independent `_` fields for all 9 deterministic Agent rules and 7 Agent LLM metrics, only when selected in `evaluator[].evals`. Each deterministic rule owns its parsing and field assembly in its `prepare_*` diff --git a/dingo/data/converter/ms_swift.py b/dingo/data/converter/ms_swift.py index a09fcfba..b03e5522 100644 --- a/dingo/data/converter/ms_swift.py +++ b/dingo/data/converter/ms_swift.py @@ -11,6 +11,7 @@ from dingo.config import InputArgs from dingo.data.converter.base import BaseConverter +from dingo.data.converter.ms_swift_schema import validate_ms_swift_record from dingo.io import Data @@ -22,6 +23,19 @@ def _messages_content(record: Dict[str, Any]) -> str: return json.dumps(messages, ensure_ascii=False) +def _tool_call_name_and_arguments(call: Dict[str, Any]): + """Read fields already guaranteed by the standardized-format validator.""" + function = call["function"] + arguments = function["arguments"] + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + # A string is itself a valid JSON value under the standard. + pass + return function["name"], arguments + + def prepare_llm_agent_task_completion(record: Dict[str, Any]) -> Dict[str, str]: """Prepare the initial conversation-only task completion input.""" return { @@ -43,20 +57,7 @@ def prepare_rule_agent_trace_loop_detection(record: Dict[str, Any]) -> Dict[str, raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) return { @@ -87,20 +88,7 @@ def prepare_rule_agent_trace_destructive_action(record: Dict[str, Any]) -> Dict[ raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) return { @@ -131,20 +119,7 @@ def prepare_rule_agent_trace_gateway_bypass(record: Dict[str, Any]) -> Dict[str, raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} call_id = call.get("id") observation = {} @@ -192,20 +167,7 @@ def prepare_rule_agent_trace_integrity(record: Dict[str, Any]) -> Dict[str, str] raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) tool_spans_recorded += 1 call_id = call.get("id") if isinstance(call_id, str) and call_id: @@ -248,20 +210,7 @@ def prepare_rule_agent_trace_latency_anomaly(record: Dict[str, Any]) -> Dict[str raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) step = {"name": name} call_id = call.get("id") observation = {} @@ -273,9 +222,16 @@ def prepare_rule_agent_trace_latency_anomaly(record: Dict[str, Any]) -> Dict[str # Copy recorded timings and groups, preferring matched tool results. for key in ("duration", "duration_seconds", "group", "type"): if key in observation: - step[key] = observation[key] + value = observation[key] elif key in call: - step[key] = call[key] + value = call[key] + else: + continue + # Standard ms-swift reserves call.type="function" for the + # invocation kind; it is not a latency peer-group label. + if key == "type" and value == "function": + continue + step[key] = value steps.append(step) return {"RuleAgentTraceLatencyAnomaly_content": json.dumps({"steps": steps}, ensure_ascii=False)} @@ -293,20 +249,7 @@ def prepare_rule_agent_trace_sandbox_explicitly_disabled(record: Dict[str, Any]) raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) return { @@ -337,20 +280,7 @@ def prepare_rule_agent_trace_secret_exposure(record: Dict[str, Any]) -> Dict[str raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} call_id = call.get("id") observation = {} @@ -389,20 +319,7 @@ def prepare_rule_agent_trace_sensitive_access(record: Dict[str, Any]) -> Dict[st raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") for call_index, call in enumerate(tool_calls): path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" - function = call.get("function") if isinstance(call, dict) else None - if not isinstance(function, dict): - raise ValueError(f"{path} must be an object") - name = function.get("name") - if not isinstance(name, str) or not name: - raise ValueError(f"{path}.name must be a non-empty string") - arguments = function.get("arguments", {}) - if isinstance(arguments, str): - try: - arguments = json.loads(arguments) - except json.JSONDecodeError as error: - raise ValueError(f"{path}.arguments must contain a JSON object") from error - if not isinstance(arguments, dict): - raise ValueError(f"{path}.arguments must be an object") + name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) return { @@ -493,15 +410,16 @@ def convertor(cls, input_args: InputArgs) -> Callable: preparers = [prepare for name, prepare in _PREPARERS.items() if name in selected_names] def _convert(raw: Union[str, Dict[str, Any]]) -> Data: - record = json.loads(raw) if isinstance(raw, str) else raw - if not isinstance(record, dict): - raise ValueError("ms_swift record must be a JSON object") - messages = record.get("messages") - if not isinstance(messages, list): - raise ValueError("ms_swift messages must be a list") - for index, message in enumerate(messages): - if not isinstance(message, dict): - raise ValueError(f"ms_swift messages[{index}] must be an object") + if isinstance(raw, str): + try: + record = json.loads(raw) + except json.JSONDecodeError as error: + raise ValueError( + f"ms_swift record is malformed: invalid JSON ({error.msg})" + ) from error + else: + record = raw + validate_ms_swift_record(record) converted = dict(record) derived_fields = set() diff --git a/dingo/data/converter/ms_swift_schema.py b/dingo/data/converter/ms_swift_schema.py new file mode 100644 index 00000000..a0e78b2c --- /dev/null +++ b/dingo/data/converter/ms_swift_schema.py @@ -0,0 +1,348 @@ +"""Validation for the standardized ms-swift delivery format.""" + +from collections import Counter +from typing import Any, Dict, Iterable, Tuple + + +_TRAJECTORY_FIELDS = ( + "messages", + "tools", + "termination", + "source", + "harness", + "model", + "total_rounds", + "total_tool_calls", + "tool_counts", + "reasoning_total_tokens", + "tool_defs_tag", + "missing_tool_defs", + "tool_call_tag", + "tool_call_check", + "server_tool_calls", + "metadata", +) +_TOP_LEVEL_FIELDS = _TRAJECTORY_FIELDS + ("completeness_tag", "completeness") +_TOOL_CALL_CHECK_FIELDS = ( + "total_calls", + "hallucinated_calls", + "undefined_tool_calls", + "unverifiable_calls", + "checked_calls", + "extra_arg_calls", + "missing_required_calls", + "type_mismatch_calls", + "defaulted_missing_calls", + "mismatch_calls", + "mismatches", + "mismatches_truncated", +) +_COMPLETENESS_FIELDS = ( + "is_subagent", + "spawn_calls", + "mounted_subs", + "subtree_complete", + "unkeyed_mounts", + "relay_mounts", + "trailing_unanswered_call", + "no_final_assistant_turn", +) +_MESSAGE_ROLES = frozenset({"system", "developer", "user", "assistant", "tool"}) +_TOP_TOOL_DEF_TAGS = frozenset({"complete", "incomplete", "complete_with_incomplete_sub"}) +_NESTED_TOOL_DEF_TAGS = frozenset({"complete", "incomplete"}) +_TOP_TOOL_CALL_TAGS = frozenset({"consistent", "inconsistent", "consistent_with_inconsistent_sub"}) +_NESTED_TOOL_CALL_TAGS = frozenset({"consistent", "inconsistent"}) +_COMPLETENESS_TAGS = frozenset({ + "orphan_sub", + "complete_main_no_sub", + "incomplete_main_no_sub", + "complete_main_with_complete_sub", + "complete_main_with_incomplete_sub", + "incomplete_main_with_complete_sub", + "incomplete_main_with_incomplete_sub", +}) +_MISMATCH_REASONS = frozenset({ + "undefined_tool", "extra_args", "missing_required", "type_mismatch", +}) + + +def _fail(path: str, detail: str) -> None: + raise ValueError(f"ms_swift record is malformed: {path} {detail}") + + +def _require_fields(value: Dict[str, Any], fields: Iterable[str], path: str) -> None: + missing = sorted(field for field in fields if field not in value) + if missing: + _fail(path, f"is missing required field(s): {missing}") + + +def _require_object(value: Any, path: str) -> Dict[str, Any]: + if not isinstance(value, dict): + _fail(path, "must be an object") + return value + + +def _require_list(value: Any, path: str) -> list: + if not isinstance(value, list): + _fail(path, "must be a list") + return value + + +def _require_string(value: Any, path: str) -> str: + if not isinstance(value, str): + _fail(path, "must be a string") + return value + + +def _require_integer(value: Any, path: str, *, non_negative: bool = False) -> int: + if isinstance(value, bool) or not isinstance(value, int): + _fail(path, "must be an integer") + if non_negative and value < 0: + _fail(path, "must be a non-negative integer") + return value + + +def _require_boolean(value: Any, path: str) -> bool: + if not isinstance(value, bool): + _fail(path, "must be a boolean") + return value + + +def _validate_string_list(value: Any, path: str) -> None: + for index, item in enumerate(_require_list(value, path)): + _require_string(item, f"{path}[{index}]") + + +def _validate_messages(messages: Any, path: str) -> Tuple[int, list, Counter]: + messages = _require_list(messages, path) + assistant_count = 0 + calls = [] + tool_counts = Counter() + call_ids = {} + result_ids = {} + tool_results = [] + + for message_index, item in enumerate(messages): + message_path = f"{path}[{message_index}]" + message = _require_object(item, message_path) + _require_fields(message, ("role", "content"), message_path) + role = _require_string(message["role"], f"{message_path}.role") + if role not in _MESSAGE_ROLES: + _fail(f"{message_path}.role", f"must be one of {sorted(_MESSAGE_ROLES)}") + _require_string(message["content"], f"{message_path}.content") + + if role == "assistant": + assistant_count += 1 + _require_fields(message, ("reasoning_content",), message_path) + _require_string(message["reasoning_content"], f"{message_path}.reasoning_content") + if "reasoning_details" in message: + details_path = f"{message_path}.reasoning_details" + for detail_index, detail in enumerate(_require_list(message["reasoning_details"], details_path)): + _require_object(detail, f"{details_path}[{detail_index}]") + if "tool_calls" not in message: + continue + tool_calls = _require_list(message["tool_calls"], f"{message_path}.tool_calls") + for call_index, item_call in enumerate(tool_calls): + call_path = f"{message_path}.tool_calls[{call_index}]" + call = _require_object(item_call, call_path) + _require_fields(call, ("type", "id", "function"), call_path) + call_type = _require_string(call["type"], f"{call_path}.type") + if call_type != "function": + _fail(f"{call_path}.type", "must equal 'function'") + call_id = _require_string(call["id"], f"{call_path}.id") + if call_id in call_ids: + _fail(f"{call_path}.id", f"duplicates {call_ids[call_id]}") + call_ids[call_id] = f"{call_path}.id" + function_path = f"{call_path}.function" + function = _require_object(call["function"], function_path) + _require_fields(function, ("name", "arguments"), function_path) + name = _require_string(function["name"], f"{function_path}.name") + if not name: + _fail(f"{function_path}.name", "must be non-empty") + calls.append(call) + tool_counts[name] += 1 + + elif role == "tool": + _require_fields(message, ("tool_call_id", "name"), message_path) + tool_call_id = _require_string(message["tool_call_id"], f"{message_path}.tool_call_id") + if tool_call_id in result_ids: + _fail( + f"{message_path}.tool_call_id", + f"duplicates {result_ids[tool_call_id]}", + ) + result_ids[tool_call_id] = f"{message_path}.tool_call_id" + _require_string(message["name"], f"{message_path}.name") + tool_results.append((f"{message_path}.tool_call_id", tool_call_id)) + + for result_path, tool_call_id in tool_results: + if tool_call_id not in call_ids: + _fail(result_path, f"does not match any assistant tool call id: {tool_call_id!r}") + return assistant_count, calls, tool_counts + + +def _validate_tools(tools: Any, path: str) -> None: + names = set() + for index, item in enumerate(_require_list(tools, path)): + tool_path = f"{path}[{index}]" + tool = _require_object(item, tool_path) + _require_fields(tool, ("type", "name", "description", "parameters"), tool_path) + tool_type = _require_string(tool["type"], f"{tool_path}.type") + if tool_type != "function": + _fail(f"{tool_path}.type", "must equal 'function'") + name = _require_string(tool["name"], f"{tool_path}.name") + if not name: + _fail(f"{tool_path}.name", "must be non-empty") + if name in names: + _fail(f"{tool_path}.name", f"duplicates tool definition {name!r}") + names.add(name) + _require_string(tool["description"], f"{tool_path}.description") + _require_object(tool["parameters"], f"{tool_path}.parameters") + + +def _validate_tool_call_check(value: Any, path: str, actual_calls: int) -> None: + check = _require_object(value, path) + _require_fields(check, _TOOL_CALL_CHECK_FIELDS, path) + for field in _TOOL_CALL_CHECK_FIELDS: + if field == "mismatches": + mismatches = _require_list(check[field], f"{path}.{field}") + if len(mismatches) > 50: + _fail(f"{path}.{field}", "must contain at most 50 entries") + for index, item in enumerate(mismatches): + mismatch_path = f"{path}.{field}[{index}]" + mismatch = _require_object(item, mismatch_path) + _require_fields( + mismatch, + ( + "tool", "tool_call_id", "message_index", "reasons", + "extra_args", "missing_required", "type_mismatch", + ), + mismatch_path, + ) + _require_string(mismatch["tool"], f"{mismatch_path}.tool") + _require_string(mismatch["tool_call_id"], f"{mismatch_path}.tool_call_id") + _require_integer( + mismatch["message_index"], f"{mismatch_path}.message_index", + non_negative=True, + ) + _validate_string_list(mismatch["reasons"], f"{mismatch_path}.reasons") + unknown_reasons = set(mismatch["reasons"]) - _MISMATCH_REASONS + if unknown_reasons: + _fail( + f"{mismatch_path}.reasons", + f"contains unsupported value(s): {sorted(unknown_reasons)}", + ) + _validate_string_list(mismatch["extra_args"], f"{mismatch_path}.extra_args") + _validate_string_list( + mismatch["missing_required"], f"{mismatch_path}.missing_required", + ) + type_mismatches_path = f"{mismatch_path}.type_mismatch" + for type_index, type_item in enumerate( + _require_list(mismatch["type_mismatch"], type_mismatches_path) + ): + type_path = f"{type_mismatches_path}[{type_index}]" + type_mismatch = _require_object(type_item, type_path) + _require_fields(type_mismatch, ("arg", "declared", "actual"), type_path) + _require_string(type_mismatch["arg"], f"{type_path}.arg") + _validate_string_list(type_mismatch["declared"], f"{type_path}.declared") + _require_string(type_mismatch["actual"], f"{type_path}.actual") + else: + _require_integer(check[field], f"{path}.{field}", non_negative=True) + if check["total_calls"] != actual_calls: + _fail(f"{path}.total_calls", f"must equal the actual tool-call count {actual_calls}") + + +def _validate_server_tool_calls(value: Any, path: str) -> None: + for index, item in enumerate(_require_list(value, path)): + call_path = f"{path}[{index}]" + call = _require_object(item, call_path) + _require_fields(call, ("name", "id", "arguments", "origin", "result"), call_path) + _require_string(call["name"], f"{call_path}.name") + _require_string(call["id"], f"{call_path}.id") + origin = _require_string(call["origin"], f"{call_path}.origin") + if origin not in ("history", "response"): + _fail(f"{call_path}.origin", "must equal 'history' or 'response'") + if call["result"] is not None: + _require_object(call["result"], f"{call_path}.result") + + +def _validate_metadata(value: Any, path: str) -> None: + metadata = _require_object(value, path) + _require_fields(metadata, ("source_file", "line_no", "created_at"), path) + _require_string(metadata["source_file"], f"{path}.source_file") + _require_integer(metadata["line_no"], f"{path}.line_no", non_negative=True) + _require_string(metadata["created_at"], f"{path}.created_at") + + +def _validate_completeness(value: Any, path: str) -> None: + completeness = _require_object(value, path) + _require_fields(completeness, _COMPLETENESS_FIELDS, path) + for field in ("is_subagent", "subtree_complete", "trailing_unanswered_call", "no_final_assistant_turn"): + _require_boolean(completeness[field], f"{path}.{field}") + for field in ("spawn_calls", "mounted_subs", "unkeyed_mounts", "relay_mounts"): + _require_integer(completeness[field], f"{path}.{field}", non_negative=True) + + +def _validate_trajectory(record: Any, path: str, *, top_level: bool) -> None: + record = _require_object(record, path) + _require_fields(record, _TOP_LEVEL_FIELDS if top_level else _TRAJECTORY_FIELDS, path) + + for field in ("termination", "source", "harness", "model"): + _require_string(record[field], f"{path}.{field}") + assistant_count, calls, actual_tool_counts = _validate_messages( + record["messages"], f"{path}.messages", + ) + _validate_tools(record["tools"], f"{path}.tools") + + total_rounds = _require_integer(record["total_rounds"], f"{path}.total_rounds", non_negative=True) + total_tool_calls = _require_integer( + record["total_tool_calls"], f"{path}.total_tool_calls", non_negative=True, + ) + if total_rounds != assistant_count: + _fail(f"{path}.total_rounds", f"must equal the assistant-message count {assistant_count}") + if total_tool_calls != len(calls): + _fail(f"{path}.total_tool_calls", f"must equal the actual tool-call count {len(calls)}") + + tool_counts = _require_object(record["tool_counts"], f"{path}.tool_counts") + for name, count in tool_counts.items(): + _require_string(name, f"{path}.tool_counts key") + _require_integer(count, f"{path}.tool_counts[{name!r}]", non_negative=True) + if tool_counts != dict(actual_tool_counts): + _fail(f"{path}.tool_counts", f"must equal the actual counts {dict(actual_tool_counts)}") + + _require_integer(record["reasoning_total_tokens"], f"{path}.reasoning_total_tokens", non_negative=True) + tool_defs_tag = _require_string(record["tool_defs_tag"], f"{path}.tool_defs_tag") + tool_call_tag = _require_string(record["tool_call_tag"], f"{path}.tool_call_tag") + valid_def_tags = _TOP_TOOL_DEF_TAGS if top_level else _NESTED_TOOL_DEF_TAGS + valid_call_tags = _TOP_TOOL_CALL_TAGS if top_level else _NESTED_TOOL_CALL_TAGS + if tool_defs_tag not in valid_def_tags: + _fail(f"{path}.tool_defs_tag", f"must be one of {sorted(valid_def_tags)}") + if tool_call_tag not in valid_call_tags: + _fail(f"{path}.tool_call_tag", f"must be one of {sorted(valid_call_tags)}") + _validate_string_list(record["missing_tool_defs"], f"{path}.missing_tool_defs") + _validate_tool_call_check(record["tool_call_check"], f"{path}.tool_call_check", len(calls)) + _validate_server_tool_calls(record["server_tool_calls"], f"{path}.server_tool_calls") + _validate_metadata(record["metadata"], f"{path}.metadata") + + if top_level: + completeness_tag = _require_string(record["completeness_tag"], f"{path}.completeness_tag") + if completeness_tag not in _COMPLETENESS_TAGS: + _fail(f"{path}.completeness_tag", f"must be one of {sorted(_COMPLETENESS_TAGS)}") + _validate_completeness(record["completeness"], f"{path}.completeness") + + if "sub_agent_relay_mounts" in record: + mounts = _require_object(record["sub_agent_relay_mounts"], f"{path}.sub_agent_relay_mounts") + for spawn_id, relay_id in mounts.items(): + _require_string(spawn_id, f"{path}.sub_agent_relay_mounts key") + _require_string(relay_id, f"{path}.sub_agent_relay_mounts[{spawn_id!r}]") + if "sub_agent_trajectory" in record: + trajectories = _require_object(record["sub_agent_trajectory"], f"{path}.sub_agent_trajectory") + for call_id, trajectory in trajectories.items(): + _require_string(call_id, f"{path}.sub_agent_trajectory key") + _validate_trajectory( + trajectory, f"{path}.sub_agent_trajectory[{call_id!r}]", top_level=False, + ) + + +def validate_ms_swift_record(record: Any) -> None: + """Raise ``ValueError`` unless *record* follows the full delivery schema.""" + _validate_trajectory(record, "ms_swift", top_level=True) diff --git a/docs/agent_trace/ms_swift.md b/docs/agent_trace/ms_swift.md index 287fd7a3..ecae25c3 100644 --- a/docs/agent_trace/ms_swift.md +++ b/docs/agent_trace/ms_swift.md @@ -15,10 +15,17 @@ converter 只为 `evaluator[].evals` 中选中的指标生成辅助字段，并 ## 一、输入约定 -- `messages` 必须是数组，每条消息必须是对象。LLM 指标读取全部消息；确定性规则按各自需要读取工具调用或顶层统计。 -- 读取工具调用时，仅读取 `role="assistant"` 的 `tool_calls`；该键缺失或为 `null` 时按空列表处理，其他非数组值会报错。 -- 每次调用必须有对象类型的 `function` 和非空字符串 `function.name`。`arguments` 缺失时为 `{}`；对象原样使用，JSON 字符串解析为对象，显式 `null`、数组或无效 JSON 均会报错。 -- 需要配对工具结果的指标只使用非空字符串 ID。数字、空字符串或缺失 ID 不参与配对；有效调用 ID 或有效工具返回 ID 重复时会报错。 +- converter 按《轨迹数据的标准化格式》的完整交付口径校验每条记录，校验与所选指标无关，并且发生在生成任何辅助字段之前。 +- 顶层必选字段为：`messages`、`tools`、`termination`、`source`、`harness`、`model`、`total_rounds`、`total_tool_calls`、`tool_counts`、`reasoning_total_tokens`、`tool_defs_tag`、`missing_tool_defs`、`tool_call_tag`、`tool_call_check`、`server_tool_calls`、`metadata`、`completeness_tag`、`completeness`。字段必须存在且类型正确，允许规范明确支持的空字符串、空数组或空对象。 +- `sub_agent_trajectory` 和 `sub_agent_relay_mounts` 是可选字段；存在 `sub_agent_trajectory` 时递归校验每个子轨迹。子轨迹不要求顶层专属的 `completeness_tag` 和 `completeness`。 +- 所有消息必须包含字符串型 `role` 和 `content`；允许的角色为 `system`、`developer`、`user`、`assistant`、`tool`。assistant 消息还必须包含字符串型 `reasoning_content`；tool 消息还必须包含字符串型 `tool_call_id` 和 `name`。 +- `tool_calls` 仅在至少有一次调用时出现。每个调用必须包含 `type="function"`、字符串型 `id`，以及含 `name`、`arguments` 的 `function` 对象。 +- `tools` 中每个定义必须包含 `type="function"`、`name`、`description` 和对象类型的 `parameters`。 +- `total_rounds`、`total_tool_calls`、`tool_counts` 和 `tool_call_check.total_calls` 必须与当前节点的 `messages` 一致；tool 消息的 `tool_call_id` 必须能匹配当前节点中的调用。 +- 工具调用没有返回、调用缺少工具定义、参数与 schema 不一致或轨迹没有最终回复属于质量或完整性问题，不属于格式错误；这些情况由质量标签、`tool_call_check` 和 `completeness` 表达。 +- 读取工具调用时，仅读取 `role="assistant"` 的 `tool_calls`；该键可以缺失，存在时必须是数组，空数组按规范保留，不能用 `null` 代替。 +- 每次调用必须有 `type="function"`、字符串型唯一 `id`、对象类型的 `function`、非空字符串 `function.name` 和 `function.arguments`。`arguments` 可以是任意 JSON 值；为兼容旧数据，其中可解析的 JSON 字符串仍会在构造规则输入时解析，但不会修改原始记录。 +- tool 消息的 `tool_call_id` 必须匹配当前节点的调用；调用不一定有工具返回。重复调用 ID 会按格式错误处理；preparer 需要建立返回索引时也会拒绝重复的工具返回 ID。 ## 二、确定性规则 diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..0dcdbd82 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ +"""Test support package.""" diff --git a/test/scripts/__init__.py b/test/scripts/__init__.py new file mode 100644 index 00000000..c30a3e26 --- /dev/null +++ b/test/scripts/__init__.py @@ -0,0 +1 @@ +"""Shared helpers for the test suite.""" diff --git a/test/scripts/dataset/test_ms_swift_agent_metrics.py b/test/scripts/dataset/test_ms_swift_agent_metrics.py index 929e1aa9..126af68c 100644 --- a/test/scripts/dataset/test_ms_swift_agent_metrics.py +++ b/test/scripts/dataset/test_ms_swift_agent_metrics.py @@ -13,6 +13,7 @@ from dingo.model import Model from dingo.model.llm.base_openai import BaseOpenAI from dingo.model.rule import rule_agent +from test.scripts.ms_swift_test_utils import refresh_ms_swift_statistics, standard_ms_swift_record RULE_NAMES = ( "RuleAgentTraceDestructiveAction", "RuleAgentTraceGatewayBypass", "RuleAgentTraceIntegrity", @@ -60,20 +61,30 @@ def evaluate_rule(record, name): @pytest.fixture def record(): - return { - "data_id": "trace-1", "total_tool_calls": 2, "total_tokens": 600001, - "tools": [{"name": "search"}], - "messages": [ + return standard_ms_swift_record( + data_id="trace-1", total_tokens=600001, + tools=[{ + "type": "function", "name": "search", "description": "Search", + "parameters": {}, + }, { + "type": "function", "name": "read", "description": "Read", + "parameters": {}, + }], + messages=[ {"role": "user", "content": "查找文档并总结结果"}, {"role": "assistant", "content": "", "reasoning_content": "这是推理，不一定是计划", "tool_calls": [ - {"id": "a", "function": {"name": "search", "arguments": {"query": "documents"}}}, - {"id": "b", "function": {"name": "read", "arguments": {"path": "notes.txt"}}}, + {"type": "function", "id": "a", "function": { + "name": "search", "arguments": {"query": "documents"}, + }}, + {"type": "function", "id": "b", "function": { + "name": "read", "arguments": {"path": "notes.txt"}, + }}, ]}, - {"role": "tool", "tool_call_id": "b", "content": "document"}, - {"role": "tool", "tool_call_id": "a", "content": "found"}, - {"role": "assistant", "content": "已完成"}, + {"role": "tool", "name": "read", "tool_call_id": "b", "content": "document"}, + {"role": "tool", "name": "search", "tool_call_id": "a", "content": "found"}, + {"role": "assistant", "content": "已完成", "reasoning_content": ""}, ], - } + ) @pytest.mark.parametrize("name", ALL_NAMES) @@ -137,7 +148,7 @@ def test_secret_results_are_paired_by_id_even_when_return_order_differs(record): def test_missing_result_is_not_guessed_from_an_unrelated_tool_message(record): - record["messages"][2]["tool_call_id"] = "unrelated" + record["messages"].pop(2) payload = json.loads(convert(record).RuleAgentTraceSecretExposure_content) assert "result" not in payload["tool_calls"][1] integrity = json.loads(convert(record).RuleAgentTraceIntegrity_content) @@ -157,11 +168,10 @@ def test_tool_call_preparers_parse_json_arguments_without_changing_source(record @pytest.mark.parametrize("name", RULE_NAMES[:-1]) -def test_rule_preparers_reject_invalid_arguments_without_changing_source(record, name): +def test_rule_preparers_accept_non_object_arguments_allowed_by_standard(record, name): record["messages"][1]["tool_calls"][1]["function"]["arguments"] = "invalid JSON" original = copy.deepcopy(record) - with pytest.raises(ValueError, match=r"messages\[1\].tool_calls\[1\].function.arguments"): - convert(record, (name,)) + convert(record, (name,)) assert record == original @@ -180,15 +190,22 @@ def test_ambiguous_call_ids_fail_instead_of_using_the_wrong_result(record, name, def test_gateway_maps_explicit_status_without_changing_rule_behavior(): - record = {"messages": [ - {"role": "assistant", "tool_calls": [ - {"id": "a", "function": {"name": "WebFetch", "arguments": {"url": "https://example.org"}}}, + record = standard_ms_swift_record(messages=[ + {"role": "assistant", "content": "", "reasoning_content": "", "tool_calls": [ + {"type": "function", "id": "a", "function": { + "name": "WebFetch", "arguments": {"url": "https://example.org"}, + }}, ]}, - {"role": "tool", "tool_call_id": "a", "status": "error", "content": "GATEWAY BLOCKED"}, - {"role": "assistant", "tool_calls": [ - {"id": "b", "function": {"name": "shell", "arguments": {"command": "curl https://example.org"}}}, + { + "role": "tool", "name": "WebFetch", "tool_call_id": "a", + "status": "error", "content": "GATEWAY BLOCKED", + }, + {"role": "assistant", "content": "", "reasoning_content": "", "tool_calls": [ + {"type": "function", "id": "b", "function": { + "name": "shell", "arguments": {"command": "curl https://example.org"}, + }}, ]}, - ]} + ]) name = "RuleAgentTraceGatewayBypass" assert evaluate_rule(record, name).status record["messages"][1].pop("status") @@ -196,22 +213,20 @@ def test_gateway_maps_explicit_status_without_changing_rule_behavior(): record["messages"][2]["tool_calls"][0]["function"] = { "name": "exec_command", "arguments": {"cmd": "curl https://example.org"}, } + refresh_ms_swift_statistics(record) payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) assert "status" not in payload["tool_calls"][0] assert payload["tool_calls"][1]["tool_name"] == "exec_command" assert payload["tool_calls"][1]["args"] == {"cmd": "curl https://example.org"} -def test_integrity_compares_actual_count_with_source_counter(record): +def test_integrity_uses_valid_source_counter_and_schema_rejects_mismatch(record): name = "RuleAgentTraceIntegrity" payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) assert payload == {"tool_calls_expected": 2, "tool_spans_recorded": 2, "open_observation_count": 0} record["total_tool_calls"] = 3 - assert evaluate_rule(record, name).status - record.pop("total_tool_calls") - payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) - assert "tool_calls_expected" not in payload - assert "trace_truncated" not in payload + with pytest.raises(ValueError, match=r"total_tool_calls.*actual tool-call count 2"): + convert(record, (name,)) def test_latency_is_na_without_explicit_timings_and_uses_recorded_extensions(record): @@ -219,9 +234,15 @@ def test_latency_is_na_without_explicit_timings_and_uses_recorded_extensions(rec assert not evaluate_rule(record, name).applicable calls = record["messages"][1]["tool_calls"] record["messages"][1]["tool_calls"] = [ - {"function": calls[0]["function"], "duration": 1 if index < 10 else 100, "group": "tool"} + { + "type": "function", "id": f"latency-{index}", + "function": calls[0]["function"], + "duration": 1 if index < 10 else 100, "group": "tool", + } for index in range(11) ] + record["messages"] = record["messages"][:2] + record["messages"][4:] + refresh_ms_swift_statistics(record) payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) assert payload["steps"][-1] == {"name": "search", "duration": 100, "group": "tool"} assert evaluate_rule(record, name).status @@ -229,7 +250,7 @@ def test_latency_is_na_without_explicit_timings_and_uses_recorded_extensions(rec def test_latency_pairs_timings_by_id_and_prefers_tool_message_extensions(record): name = "RuleAgentTraceLatencyAnomaly" - record["messages"][1]["tool_calls"][0].update(duration=10, group="call", type="source") + record["messages"][1]["tool_calls"][0].update(duration=10, group="call") record["messages"][1]["tool_calls"][1].update(duration_seconds=20, group="fallback") record["messages"][3].update(duration=1, group="observation", type="tool") payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) diff --git a/test/scripts/dataset/test_ms_swift_converter.py b/test/scripts/dataset/test_ms_swift_converter.py index 35f9728c..925e7ef3 100644 --- a/test/scripts/dataset/test_ms_swift_converter.py +++ b/test/scripts/dataset/test_ms_swift_converter.py @@ -15,6 +15,7 @@ from dingo.model.llm.agent_eval.llm_agent_task_completion import LLMAgentTaskCompletion from dingo.model.llm.agent_eval.llm_agent_tool_correctness import LLMAgentToolCorrectness from dingo.model.rule.rule_agent import RuleAgentTraceDestructiveAction, RuleAgentTraceLoopDetection +from test.scripts.ms_swift_test_utils import refresh_ms_swift_statistics, standard_ms_swift_record EVALUATORS = ( "LLMAgentTaskCompletion", @@ -34,25 +35,34 @@ def make_args(names=EVALUATORS, **kwargs): @pytest.fixture def record(): - return { - "data_id": "trace-1", - "tools": [{"name": "search", "description": "Search documents"}], - "metadata": {"source": "example"}, - "messages": [ - {"role": "developer", "content": "遵循用户要求"}, + return standard_ms_swift_record( + tools=[{ + "type": "function", "name": "search", "description": "Search documents", + "parameters": {}, + }, { + "type": "function", "name": "read", "description": "Read documents", + "parameters": {}, + }], + messages=[ + {"role": "system", "content": "遵循用户要求"}, {"role": "user", "content": "查找资料"}, { "role": "assistant", "content": "", "reasoning_content": "先搜索", "tool_calls": [ - {"id": "call-1", "function": {"name": "search", "arguments": {"query": "资料"}}}, - {"id": "call-2", "function": {"name": "read", "arguments": '{"path": "notes.txt"}'}}, + {"type": "function", "id": "call-1", "function": { + "name": "search", "arguments": {"query": "资料"}, + }}, + {"type": "function", "id": "call-2", "function": { + "name": "read", "arguments": {"path": "notes.txt"}, + }}, ], }, {"role": "tool", "name": "search", "tool_call_id": "call-1", "content": "找到了"}, {"role": "tool", "name": "read", "tool_call_id": "call-2", "content": "资料正文"}, - {"role": "assistant", "content": "已完成"}, + {"role": "assistant", "content": "已完成", "reasoning_content": ""}, ], - } + data_id="trace-1", + ) @pytest.mark.parametrize("as_json", [False, True]) @@ -75,7 +85,7 @@ def test_independent_fields_and_source_preservation(record, as_json): assert converted[f"{name}_prompt"] == "" assert "资料" in converted[f"{name}_content"] assert json.loads(converted[f"{name}_content"]) == [ - {"role": "developer", "content": "遵循用户要求"}, + {"role": "system", "content": "遵循用户要求"}, {"role": "user", "content": "查找资料"}, {"role": "assistant", "content": ""}, {"role": "tool", "content": "找到了"}, @@ -101,10 +111,10 @@ def test_generates_only_selected_evaluator_fields(record, name): assert data._derived_fields == expected -def test_llm_conversion_does_not_parse_omitted_tool_calls(record): +def test_format_validation_precedes_llm_projection(record): record["messages"][2]["tool_calls"] = "not a valid tool call list" - converted = converters["ms_swift"].convertor(make_args(("LLMAgentTaskCompletion",)))(record) - assert json.loads(converted.LLMAgentTaskCompletion_content)[2] == {"role": "assistant", "content": ""} + with pytest.raises(ValueError, match="tool_calls must be a list"): + converters["ms_swift"].convertor(make_args(("LLMAgentTaskCompletion",)))(record) @pytest.mark.parametrize("names", [(), ("RuleAbnormalChar",)]) @@ -122,8 +132,8 @@ def prepare(record): monkeypatch.setitem(module._PREPARERS, "RuleAgentTraceLoopDetection", prepare) convert = converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",))) - first = convert({"messages": [], "data_id": "first"}) - second = convert({"messages": [], "data_id": "second"}) + first = convert(standard_ms_swift_record(data_id="first")) + second = convert(standard_ms_swift_record(data_id="second")) assert first._derived_fields == {"custom_first"} assert second._derived_fields == {"custom_second"} assert pickle.loads(pickle.dumps(first))._derived_fields == {"custom_first"} @@ -140,7 +150,13 @@ def test_rejects_derived_field_collision_without_overwriting_source(record): def test_converted_sequence_can_detect_loops(record): calls = record["messages"][2]["tool_calls"] - record["messages"][2]["tool_calls"] = calls * 3 + record["messages"] = record["messages"][:3] + record["messages"][2]["tool_calls"] = [ + dict(copy.deepcopy(call), id=f"loop-{repeat}-{index}") + for repeat in range(3) + for index, call in enumerate(calls) + ] + refresh_ms_swift_statistics(record) converted = converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(record) result = RuleAgentTraceLoopDetection.eval(Data(content=converted.RuleAgentTraceLoopDetection_content)) assert result.applicable @@ -148,9 +164,12 @@ def test_converted_sequence_can_detect_loops(record): assert "Loop detected" in result.reason[0] # Six calls with different arguments must not turn into a name-only loop. record["messages"][2]["tool_calls"] = [ - {"function": {"name": "search", "arguments": {"query": str(index)}}} + {"type": "function", "id": f"search-{index}", "function": { + "name": "search", "arguments": {"query": str(index)}, + }} for index in range(6) ] + refresh_ms_swift_statistics(record) converted = converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(record) result = RuleAgentTraceLoopDetection.eval(Data(content=converted.RuleAgentTraceLoopDetection_content)) assert result.applicable @@ -159,9 +178,15 @@ def test_converted_sequence_can_detect_loops(record): def test_destructive_rule_input_and_unchanged_cmd(record): record["messages"][2]["tool_calls"] = [ - {"function": {"name": "shell", "arguments": {"command": "rm -rf /important"}}}, - {"function": {"name": "exec_command", "arguments": {"cmd": "rm -rf /other"}}}, + {"type": "function", "id": "shell-1", "function": { + "name": "shell", "arguments": {"command": "rm -rf /important"}, + }}, + {"type": "function", "id": "shell-2", "function": { + "name": "exec_command", "arguments": {"cmd": "rm -rf /other"}, + }}, ] + record["messages"] = record["messages"][:3] + refresh_ms_swift_statistics(record) converted = converters["ms_swift"].convertor(make_args(("RuleAgentTraceDestructiveAction",)))(record) content = converted.RuleAgentTraceDestructiveAction_content assert json.loads(content)["tool_calls"][1]["args"] == {"cmd": "rm -rf /other"} @@ -202,14 +227,20 @@ def test_local_dataset_reads_jsonl_records_and_filters_directory(tmp_path, recor assert all(json.loads(item.RuleAgentTraceLoopDetection_content)["tool_calls"] for item in data) -@pytest.mark.parametrize("raw, message", [ - ([], "record must be a JSON object"), - ({}, "messages must be a list"), - ({"messages": [None]}, r"messages\[0\] must be an object"), - ({"messages": [{"role": "assistant", "tool_calls": {}}]}, "tool_calls must be a list"), - ({"messages": [{"role": "assistant", "tool_calls": [{"function": {"name": "x", "arguments": "bad"}}]}]}, - "arguments must contain a JSON object"), +def test_missing_required_delivery_fields_has_actionable_error(): + with pytest.raises(ValueError, match=r"missing required field.*tools"): + converters["ms_swift"].convertor(make_args(()))({"messages": []}) + + +@pytest.mark.parametrize("mutate, message", [ + (lambda item: item.update(messages=[None]), r"messages\[0\] must be an object"), + (lambda item: item.update(messages=[{ + "role": "assistant", "content": "", "reasoning_content": "", "tool_calls": {}, + }]), "tool_calls must be a list"), + (lambda item: item["messages"][0].pop("content"), r"messages\[0\].*missing required field.*content"), ]) -def test_invalid_structure_has_actionable_error(raw, message): +def test_invalid_structure_has_actionable_error(mutate, message): + record = standard_ms_swift_record(messages=[{"role": "user", "content": "x"}]) + mutate(record) with pytest.raises(ValueError, match=message): - converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(raw) + converters["ms_swift"].convertor(make_args(()))(record) diff --git a/test/scripts/dataset/test_ms_swift_schema.py b/test/scripts/dataset/test_ms_swift_schema.py new file mode 100644 index 00000000..9b94e6cb --- /dev/null +++ b/test/scripts/dataset/test_ms_swift_schema.py @@ -0,0 +1,160 @@ +import copy + +import pytest + +from dingo.config import InputArgs +from dingo.data.converter import converters +from dingo.data.converter.ms_swift_schema import validate_ms_swift_record +from test.scripts.ms_swift_test_utils import standard_ms_swift_record + + +REQUIRED_TOP_LEVEL_FIELDS = ( + "messages", "tools", "termination", "source", "harness", "model", + "total_rounds", "total_tool_calls", "tool_counts", "reasoning_total_tokens", + "tool_defs_tag", "missing_tool_defs", "tool_call_tag", "tool_call_check", + "server_tool_calls", "metadata", "completeness_tag", "completeness", +) + + +def convert(record, names=()): + args = InputArgs( + dataset={"source": "local", "format": "ms_swift"}, + evaluator=[{"evals": [{"name": name}]} for name in names], + ) + return converters["ms_swift"].convertor(args)(record) + + +@pytest.mark.parametrize("field", REQUIRED_TOP_LEVEL_FIELDS) +def test_every_required_top_level_field_must_exist(field): + record = standard_ms_swift_record() + record.pop(field) + with pytest.raises(ValueError, match=rf"missing required field.*{field}"): + validate_ms_swift_record(record) + + +def test_optional_top_level_fields_may_be_absent(): + record = standard_ms_swift_record() + assert "sub_agent_trajectory" not in record + assert "sub_agent_relay_mounts" not in record + validate_ms_swift_record(record) + + +def test_explicit_empty_values_are_preserved_and_valid(): + record = standard_ms_swift_record(messages=[{ + "role": "assistant", "content": "", "reasoning_content": "", "tool_calls": [], + }]) + validate_ms_swift_record(record) + + +def test_developer_message_role_is_valid(): + record = standard_ms_swift_record(messages=[{ + "role": "developer", "content": "Follow the application policy.", + }]) + converted = convert(record, ("LLMAgentTaskCompletion",)) + assert '"role": "developer"' in converted.LLMAgentTaskCompletion_content + + +def test_validation_runs_before_any_derived_field_preparer(monkeypatch): + import dingo.data.converter.ms_swift as module + + calls = [] + + def prepare(_record): + calls.append("called") + return {"RuleAgentTraceLoopDetection_content": "{}"} + + monkeypatch.setitem(module._PREPARERS, "RuleAgentTraceLoopDetection", prepare) + record = standard_ms_swift_record() + record.pop("tools") + with pytest.raises(ValueError, match=r"missing required field.*tools"): + convert(record, ("RuleAgentTraceLoopDetection",)) + assert calls == [] + + +@pytest.mark.parametrize("message, missing", [ + ({"content": "x"}, "role"), + ({"role": "user"}, "content"), + ({"role": "assistant", "content": "x"}, "reasoning_content"), + ({"role": "tool", "content": "x", "name": "read"}, "tool_call_id"), + ({"role": "tool", "content": "x", "tool_call_id": "a"}, "name"), +]) +def test_required_message_fields_must_exist(message, missing): + record = standard_ms_swift_record(messages=[message]) + with pytest.raises(ValueError, match=rf"missing required field.*{missing}"): + validate_ms_swift_record(record) + + +def test_unanswered_tool_call_is_valid_format_and_quality_metadata_carries_the_gap(): + record = standard_ms_swift_record(messages=[{ + "role": "assistant", "content": "", "reasoning_content": "", + "tool_calls": [{ + "type": "function", "id": "call-1", + "function": {"name": "read", "arguments": {"path": "a.txt"}}, + }], + }]) + record["completeness_tag"] = "incomplete_main_no_sub" + record["completeness"]["trailing_unanswered_call"] = True + validate_ms_swift_record(record) + + +def test_duplicate_tool_results_are_not_standard_format(): + record = standard_ms_swift_record(messages=[ + { + "role": "assistant", "content": "", "reasoning_content": "", + "tool_calls": [{ + "type": "function", "id": "call-1", + "function": {"name": "read", "arguments": {}}, + }], + }, + {"role": "tool", "tool_call_id": "call-1", "name": "read", "content": "a"}, + {"role": "tool", "tool_call_id": "call-1", "name": "read", "content": "b"}, + ]) + with pytest.raises(ValueError, match=r"tool_call_id duplicates"): + validate_ms_swift_record(record) + + +def test_missing_tool_definition_is_quality_issue_not_format_error(): + record = standard_ms_swift_record(messages=[{ + "role": "assistant", "content": "", "reasoning_content": "", + "tool_calls": [{ + "type": "function", "id": "call-1", + "function": {"name": "read", "arguments": {}}, + }], + }]) + record["tool_defs_tag"] = "incomplete" + record["missing_tool_defs"] = ["read"] + record["tool_call_tag"] = "inconsistent" + record["tool_call_check"]["undefined_tool_calls"] = 1 + record["tool_call_check"]["mismatch_calls"] = 1 + validate_ms_swift_record(record) + + +def test_nested_trajectory_is_validated_recursively(): + child = standard_ms_swift_record() + child.pop("completeness_tag") + child.pop("completeness") + parent = standard_ms_swift_record(sub_agent_trajectory={"spawn-1": child}) + validate_ms_swift_record(parent) + invalid = copy.deepcopy(parent) + invalid["sub_agent_trajectory"]["spawn-1"].pop("tools") + with pytest.raises(ValueError, match=r"sub_agent_trajectory.*missing required field.*tools"): + validate_ms_swift_record(invalid) + + +def test_declared_statistics_must_match_messages(): + record = standard_ms_swift_record() + record["total_tool_calls"] = 1 + with pytest.raises(ValueError, match=r"total_tool_calls.*actual tool-call count 0"): + validate_ms_swift_record(record) + + +def test_mismatch_detail_requires_its_standard_fields(): + record = standard_ms_swift_record() + record["tool_call_check"]["mismatches"] = [{"tool": "read"}] + with pytest.raises(ValueError, match=r"mismatches\[0\].*missing required field.*reasons"): + validate_ms_swift_record(record) + + +def test_invalid_json_has_format_error(): + with pytest.raises(ValueError, match=r"malformed: invalid JSON"): + convert("{not json") diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index 605240d7..c046dfa8 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -7,6 +7,7 @@ from dingo.io import ResultInfo from dingo.io.output.eval_detail import EvalDetail, TokenUsage from dingo.model import Model +from test.scripts.ms_swift_test_utils import standard_ms_swift_record class TestLocal: @@ -19,16 +20,20 @@ def test_ms_swift_evaluates_derived_fields_and_cleans_saved_records( field = "RuleAgentTraceLoopDetection_content" records = [] for index in range(2): - records.append({ - "data_id": f"trace-{index}", - "metadata": {field: "nested source value"}, - "LLMAgentToolCorrectness_content": "unselected source field", - "messages": [{"role": "assistant", "tool_calls": [ - {"function": {"name": "search", "arguments": { + record = standard_ms_swift_record( + data_id=f"trace-{index}", + LLMAgentToolCorrectness_content="unselected source field", + messages=[{ + "role": "assistant", "content": "", "reasoning_content": "", + "tool_calls": [ + {"type": "function", "id": f"call-{call}", "function": { + "name": "search", "arguments": { "query": "same" if index == 0 else str(call), }}} for call in range(6) ]}], - }) + ) + record["metadata"][field] = "nested source value" + records.append(record) input_path = tmp_path / "traces.jsonl" input_path.write_text( "\n".join(json.dumps(record) for record in records), encoding="utf-8", diff --git a/test/scripts/ms_swift_test_utils.py b/test/scripts/ms_swift_test_utils.py new file mode 100644 index 00000000..0d57d283 --- /dev/null +++ b/test/scripts/ms_swift_test_utils.py @@ -0,0 +1,67 @@ +from collections import Counter +from copy import deepcopy + + +def refresh_ms_swift_statistics(record): + """Refresh standard counters after a test mutates ``messages``.""" + calls = [ + call + for message in record["messages"] + if message.get("role") == "assistant" + for call in message.get("tool_calls", []) + ] + record["total_rounds"] = sum( + message.get("role") == "assistant" for message in record["messages"] + ) + record["total_tool_calls"] = len(calls) + record["tool_counts"] = dict(Counter(call["function"]["name"] for call in calls)) + record["tool_call_check"]["total_calls"] = len(calls) + return record + + +def standard_ms_swift_record(*, messages=None, tools=None, **extra): + """Build a complete standardized record for converter tests.""" + record = { + "messages": deepcopy(messages or []), + "tools": deepcopy(tools or []), + "termination": "", + "source": "test.jsonl", + "harness": "test", + "model": "test-model", + "total_rounds": 0, + "total_tool_calls": 0, + "tool_counts": {}, + "reasoning_total_tokens": 0, + "tool_defs_tag": "complete", + "missing_tool_defs": [], + "tool_call_tag": "consistent", + "tool_call_check": { + "total_calls": 0, + "hallucinated_calls": 0, + "undefined_tool_calls": 0, + "unverifiable_calls": 0, + "checked_calls": 0, + "extra_arg_calls": 0, + "missing_required_calls": 0, + "type_mismatch_calls": 0, + "defaulted_missing_calls": 0, + "mismatch_calls": 0, + "mismatches": [], + "mismatches_truncated": 0, + }, + "server_tool_calls": [], + "metadata": {"source_file": "test.jsonl", "line_no": 0, "created_at": ""}, + "completeness_tag": "complete_main_no_sub", + "completeness": { + "is_subagent": False, + "spawn_calls": 0, + "mounted_subs": 0, + "subtree_complete": True, + "unkeyed_mounts": 0, + "relay_mounts": 0, + "trailing_unanswered_call": False, + "no_final_assistant_turn": False, + }, + } + record.update(extra) + return refresh_ms_swift_statistics(record) From e070ff50fa6d07afece7f32482ecea79e3b7347b Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 27 Oct 2026 16:38:48 +0800 Subject: [PATCH 2/2] feat: lint --- dingo/data/converter/ms_swift.py | 12 +++----- dingo/data/converter/ms_swift_schema.py | 1 - .../dataset/test_ms_swift_agent_metrics.py | 2 +- .../dataset/test_ms_swift_converter.py | 2 +- test/scripts/dataset/test_ms_swift_schema.py | 3 +- test/scripts/exec/test_local.py | 30 ++++++++++++------- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/dingo/data/converter/ms_swift.py b/dingo/data/converter/ms_swift.py index b03e5522..85af83bc 100644 --- a/dingo/data/converter/ms_swift.py +++ b/dingo/data/converter/ms_swift.py @@ -55,8 +55,7 @@ def prepare_rule_agent_trace_loop_detection(record: Dict[str, Any]) -> Dict[str, tool_calls = [] if not isinstance(tool_calls, list): raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") - for call_index, call in enumerate(tool_calls): - path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + for call in tool_calls: name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) @@ -86,8 +85,7 @@ def prepare_rule_agent_trace_destructive_action(record: Dict[str, Any]) -> Dict[ tool_calls = [] if not isinstance(tool_calls, list): raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") - for call_index, call in enumerate(tool_calls): - path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + for call in tool_calls: name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) @@ -247,8 +245,7 @@ def prepare_rule_agent_trace_sandbox_explicitly_disabled(record: Dict[str, Any]) tool_calls = [] if not isinstance(tool_calls, list): raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") - for call_index, call in enumerate(tool_calls): - path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + for call in tool_calls: name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) @@ -317,8 +314,7 @@ def prepare_rule_agent_trace_sensitive_access(record: Dict[str, Any]) -> Dict[st tool_calls = [] if not isinstance(tool_calls, list): raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") - for call_index, call in enumerate(tool_calls): - path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + for call in tool_calls: name, arguments = _tool_call_name_and_arguments(call) item = {"tool_name": name, "args": arguments} calls.append(item) diff --git a/dingo/data/converter/ms_swift_schema.py b/dingo/data/converter/ms_swift_schema.py index a0e78b2c..a388ebcd 100644 --- a/dingo/data/converter/ms_swift_schema.py +++ b/dingo/data/converter/ms_swift_schema.py @@ -3,7 +3,6 @@ from collections import Counter from typing import Any, Dict, Iterable, Tuple - _TRAJECTORY_FIELDS = ( "messages", "tools", diff --git a/test/scripts/dataset/test_ms_swift_agent_metrics.py b/test/scripts/dataset/test_ms_swift_agent_metrics.py index 126af68c..304e0bd6 100644 --- a/test/scripts/dataset/test_ms_swift_agent_metrics.py +++ b/test/scripts/dataset/test_ms_swift_agent_metrics.py @@ -3,6 +3,7 @@ import copy import importlib import json +from test.scripts.ms_swift_test_utils import refresh_ms_swift_statistics, standard_ms_swift_record import pytest @@ -13,7 +14,6 @@ from dingo.model import Model from dingo.model.llm.base_openai import BaseOpenAI from dingo.model.rule import rule_agent -from test.scripts.ms_swift_test_utils import refresh_ms_swift_statistics, standard_ms_swift_record RULE_NAMES = ( "RuleAgentTraceDestructiveAction", "RuleAgentTraceGatewayBypass", "RuleAgentTraceIntegrity", diff --git a/test/scripts/dataset/test_ms_swift_converter.py b/test/scripts/dataset/test_ms_swift_converter.py index 925e7ef3..27e055bc 100644 --- a/test/scripts/dataset/test_ms_swift_converter.py +++ b/test/scripts/dataset/test_ms_swift_converter.py @@ -4,6 +4,7 @@ import importlib import json import pickle +from test.scripts.ms_swift_test_utils import refresh_ms_swift_statistics, standard_ms_swift_record import pytest @@ -15,7 +16,6 @@ from dingo.model.llm.agent_eval.llm_agent_task_completion import LLMAgentTaskCompletion from dingo.model.llm.agent_eval.llm_agent_tool_correctness import LLMAgentToolCorrectness from dingo.model.rule.rule_agent import RuleAgentTraceDestructiveAction, RuleAgentTraceLoopDetection -from test.scripts.ms_swift_test_utils import refresh_ms_swift_statistics, standard_ms_swift_record EVALUATORS = ( "LLMAgentTaskCompletion", diff --git a/test/scripts/dataset/test_ms_swift_schema.py b/test/scripts/dataset/test_ms_swift_schema.py index 9b94e6cb..0540b09f 100644 --- a/test/scripts/dataset/test_ms_swift_schema.py +++ b/test/scripts/dataset/test_ms_swift_schema.py @@ -1,12 +1,11 @@ import copy +from test.scripts.ms_swift_test_utils import standard_ms_swift_record import pytest from dingo.config import InputArgs from dingo.data.converter import converters from dingo.data.converter.ms_swift_schema import validate_ms_swift_record -from test.scripts.ms_swift_test_utils import standard_ms_swift_record - REQUIRED_TOP_LEVEL_FIELDS = ( "messages", "tools", "termination", "source", "harness", "model", diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index c046dfa8..172d5e34 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -1,4 +1,5 @@ import json +from test.scripts.ms_swift_test_utils import standard_ms_swift_record import pytest @@ -7,7 +8,6 @@ from dingo.io import ResultInfo from dingo.io.output.eval_detail import EvalDetail, TokenUsage from dingo.model import Model -from test.scripts.ms_swift_test_utils import standard_ms_swift_record class TestLocal: @@ -26,11 +26,11 @@ def test_ms_swift_evaluates_derived_fields_and_cleans_saved_records( messages=[{ "role": "assistant", "content": "", "reasoning_content": "", "tool_calls": [ - {"type": "function", "id": f"call-{call}", "function": { - "name": "search", "arguments": { - "query": "same" if index == 0 else str(call), - }}} for call in range(6) - ]}], + {"type": "function", "id": f"call-{call}", "function": { + "name": "search", "arguments": { + "query": "same" if index == 0 else str(call), + }}} for call in range(6) + ]}], ) record["metadata"][field] = "nested source value" records.append(record) @@ -507,9 +507,13 @@ def test_all_labels_config(self): executor = Executor.exec_map["local"](input_args) result = executor.execute() print(result) - assert all([item in result.type_ratio.get('content') for item in ["QUALITY_BAD_EFFECTIVENESS.RuleColonEnd", - "QUALITY_BAD_EFFECTIVENESS.RuleSpecialCharacter", - "QUALITY_GOOD"]]) + assert all([ + item in result.type_ratio.get('content') for item in [ + "QUALITY_BAD_EFFECTIVENESS.RuleColonEnd", + "QUALITY_BAD_EFFECTIVENESS.RuleSpecialCharacter", + "QUALITY_GOOD", + ] + ]) label_stats = { item.label: item for item in result.label_stats["content"] @@ -523,8 +527,12 @@ def test_all_labels_config(self): input_args = InputArgs(**input_data) executor = Executor.exec_map["local"](input_args) result = executor.execute() - assert all([item in result.type_ratio.get('content') for item in ["QUALITY_BAD_EFFECTIVENESS.RuleColonEnd", - "QUALITY_BAD_EFFECTIVENESS.RuleSpecialCharacter"]]) + assert all([ + item in result.type_ratio.get('content') for item in [ + "QUALITY_BAD_EFFECTIVENESS.RuleColonEnd", + "QUALITY_BAD_EFFECTIVENESS.RuleSpecialCharacter", + ] + ]) def test_metrics_score_collection_with_scores(self): """测试带有分数的指标评估时，summary 正确收集和计算分数"""