diff --git a/AGENTS.md b/AGENTS.md index 6f71d9ed..6e862bf5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -161,6 +161,7 @@ input_data = { |--------|-------------------|-------------| | JSON | `json` | Top-level array, each element is one sample | | JSONL | `jsonl` | One JSON object per line | +| ms-swift | `ms_swift` | Agent trajectory JSONL; preserves raw fields and prepares selected evaluator fields | | CSV | `csv` | Comma-separated values with header | | Parquet | `parquet` | Apache Parquet columnar format | | Plain text | `plaintext` | One line per sample, mapped to `content` | @@ -170,6 +171,22 @@ input_data = { MinerU formats automatically flatten blocks into `Data` objects, preserving all original fields (type, bbox, page_idx, etc.) via Pydantic `extra="allow"`. Use `dataset.mineru_config.include_types` to filter by block type. +The `ms_swift` converter lives in `dingo/data/converter/ms_swift.py`. It prepares +independent `_` fields for all 9 deterministic Agent rules +and 7 Agent LLM metrics, only when selected in `evaluator[].evals`. +Each deterministic rule owns its parsing and field assembly in its `prepare_*` +function; LLM metrics continue sharing `_messages_content()`. +Map these fields in a separate evaluator block for each metric. Initial LLM +inputs contain only message `role`/`content`, with empty prompts; Plan Adherence +also receives an empty context. Tool results are paired by call ID; execution +status and duration extensions are copied only when explicitly present. +See `docs/agent_trace/ms_swift.md` and `examples/dataset/ms_swift.json` for mappings +and limitations. +Each converted record privately tracks the fields actually generated by its +selected preparers. Local result output removes these fields from a copy via +`ResultInfo.without_derived_fields()`, preserving both existing output formats +and conflict backups. No manually maintained cleanup list is needed. + ### Execution Modes | Mode | Class | Use Case | diff --git a/dingo/data/converter/__init__.py b/dingo/data/converter/__init__.py index c0758730..2e7f0bb9 100644 --- a/dingo/data/converter/__init__.py +++ b/dingo/data/converter/__init__.py @@ -1,4 +1,5 @@ import dingo.data.converter.mineru # noqa: F401 — registers mineru / mineru_v2 +import dingo.data.converter.ms_swift # noqa: F401 — registers ms_swift from dingo.data.converter.base import BaseConverter converters = BaseConverter.converters diff --git a/dingo/data/converter/ms_swift.py b/dingo/data/converter/ms_swift.py new file mode 100644 index 00000000..a09fcfba --- /dev/null +++ b/dingo/data/converter/ms_swift.py @@ -0,0 +1,521 @@ +"""ms-swift JSONL conversion with independently prepared evaluator fields. + +Only configured, supported evaluators generate derived fields. Original fields +are preserved. The initial LLM inputs deliberately contain only message roles +and contents, with an empty task prompt. Deterministic rule conversions are +self-contained so their input handling can evolve independently. +""" + +import json +from typing import Any, Callable, Dict, Union + +from dingo.config import InputArgs +from dingo.data.converter.base import BaseConverter +from dingo.io import Data + + +def _messages_content(record: Dict[str, Any]) -> str: + messages = [ + {"role": message.get("role", ""), "content": message.get("content", "")} + for message in record["messages"] + ] + return json.dumps(messages, ensure_ascii=False) + + +def prepare_llm_agent_task_completion(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare the initial conversation-only task completion input.""" + return { + "LLMAgentTaskCompletion_prompt": "", + "LLMAgentTaskCompletion_content": _messages_content(record), + } + + +def prepare_rule_agent_trace_loop_detection(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare ordered names and arguments for loop detection.""" + calls = [] + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + item = {"tool_name": name, "args": arguments} + calls.append(item) + return { + "RuleAgentTraceLoopDetection_content": json.dumps( + {"tool_calls": calls}, ensure_ascii=False, + ), + } + + +def prepare_llm_agent_tool_correctness(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare the initial conversation-only tool correctness input.""" + return { + "LLMAgentToolCorrectness_prompt": "", + "LLMAgentToolCorrectness_content": _messages_content(record), + } + + +def prepare_rule_agent_trace_destructive_action(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare tool arguments as recorded, including unchanged ``cmd`` keys.""" + calls = [] + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + item = {"tool_name": name, "args": arguments} + calls.append(item) + return { + "RuleAgentTraceDestructiveAction_content": json.dumps( + {"tool_calls": calls}, ensure_ascii=False, + ), + } + + +def prepare_rule_agent_trace_gateway_bypass(record: Dict[str, Any]) -> Dict[str, str]: + """Associate policy responses and explicit execution status by call ID.""" + observations = {} + for index, message in enumerate(record["messages"]): + call_id = message.get("tool_call_id") + if message.get("role") == "tool" and isinstance(call_id, str) and call_id: + if call_id in observations: + raise ValueError(f"ms_swift messages[{index}] has duplicate tool_call_id {call_id!r}") + observations[call_id] = message + calls = [] + seen_ids = set() + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + item = {"tool_name": name, "args": arguments} + call_id = call.get("id") + observation = {} + if isinstance(call_id, str) and call_id: + if call_id in seen_ids: + raise ValueError(f"ms_swift {path} has duplicate call id {call_id!r}") + seen_ids.add(call_id) + item["call_id"] = call_id + observation = observations.get(call_id, {}) + if "content" in observation: + item["result"] = observation["content"] + # These extensions must be explicit; never derive status from prose. + for key in ("status", "duration", "duration_seconds", "group", "type"): + if key in observation: + item[key] = observation[key] + elif key in call: + item[key] = call[key] + calls.append(item) + return { + "RuleAgentTraceGatewayBypass_content": json.dumps( + {"tool_calls": calls}, ensure_ascii=False, + ), + } + + +def prepare_rule_agent_trace_integrity(record: Dict[str, Any]) -> Dict[str, str]: + """Check delivered-file counters without claiming upstream completeness.""" + observations = {} + for index, message in enumerate(record["messages"]): + call_id = message.get("tool_call_id") + if message.get("role") == "tool" and isinstance(call_id, str) and call_id: + if call_id in observations: + raise ValueError(f"ms_swift messages[{index}] has duplicate tool_call_id {call_id!r}") + observations[call_id] = message + tool_spans_recorded = 0 + open_observation_count = 0 + seen_ids = set() + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + tool_spans_recorded += 1 + call_id = call.get("id") + if isinstance(call_id, str) and call_id: + if call_id in seen_ids: + raise ValueError(f"ms_swift {path} has duplicate call id {call_id!r}") + seen_ids.add(call_id) + if call_id not in observations: + open_observation_count += 1 + else: + open_observation_count += 1 + content = { + "tool_spans_recorded": tool_spans_recorded, + "open_observation_count": open_observation_count, + } + if "total_tool_calls" in record: + content["tool_calls_expected"] = record["total_tool_calls"] + if "trace_truncated" in record: + content["trace_truncated"] = record["trace_truncated"] + return {"RuleAgentTraceIntegrity_content": json.dumps(content, ensure_ascii=False)} + + +def prepare_rule_agent_trace_latency_anomaly(record: Dict[str, Any]) -> Dict[str, str]: + """Use only explicitly recorded per-call timings and peer groups.""" + observations = {} + for index, message in enumerate(record["messages"]): + call_id = message.get("tool_call_id") + if message.get("role") == "tool" and isinstance(call_id, str) and call_id: + if call_id in observations: + raise ValueError(f"ms_swift messages[{index}] has duplicate tool_call_id {call_id!r}") + observations[call_id] = message + steps = [] + seen_ids = set() + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + step = {"name": name} + call_id = call.get("id") + observation = {} + if isinstance(call_id, str) and call_id: + if call_id in seen_ids: + raise ValueError(f"ms_swift {path} has duplicate call id {call_id!r}") + seen_ids.add(call_id) + observation = observations.get(call_id, {}) + # Copy recorded timings and groups, preferring matched tool results. + for key in ("duration", "duration_seconds", "group", "type"): + if key in observation: + step[key] = observation[key] + elif key in call: + step[key] = call[key] + steps.append(step) + return {"RuleAgentTraceLatencyAnomaly_content": json.dumps({"steps": steps}, ensure_ascii=False)} + + +def prepare_rule_agent_trace_sandbox_explicitly_disabled(record: Dict[str, Any]) -> Dict[str, str]: + """Preserve the explicit sandbox-disable argument.""" + calls = [] + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + item = {"tool_name": name, "args": arguments} + calls.append(item) + return { + "RuleAgentTraceSandboxExplicitlyDisabled_content": json.dumps( + {"tool_calls": calls}, ensure_ascii=False, + ), + } + + +def prepare_rule_agent_trace_secret_exposure(record: Dict[str, Any]) -> Dict[str, str]: + """Supply actual tool observations matched by ID, never by array position.""" + observations = {} + for index, message in enumerate(record["messages"]): + call_id = message.get("tool_call_id") + if message.get("role") == "tool" and isinstance(call_id, str) and call_id: + if call_id in observations: + raise ValueError(f"ms_swift messages[{index}] has duplicate tool_call_id {call_id!r}") + observations[call_id] = message + calls = [] + seen_ids = set() + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + item = {"tool_name": name, "args": arguments} + call_id = call.get("id") + observation = {} + if isinstance(call_id, str) and call_id: + if call_id in seen_ids: + raise ValueError(f"ms_swift {path} has duplicate call id {call_id!r}") + seen_ids.add(call_id) + item["call_id"] = call_id + observation = observations.get(call_id, {}) + if "content" in observation: + item["result"] = observation["content"] + # These extensions must be explicit; never derive status from prose. + for key in ("status", "duration", "duration_seconds", "group", "type"): + if key in observation: + item[key] = observation[key] + elif key in call: + item[key] = call[key] + calls.append(item) + return { + "RuleAgentTraceSecretExposure_content": json.dumps( + {"tool_calls": calls}, ensure_ascii=False, + ), + } + + +def prepare_rule_agent_trace_sensitive_access(record: Dict[str, Any]) -> Dict[str, str]: + """Preserve parameter structure for the rule's sensitive-path scan.""" + calls = [] + for message_index, message in enumerate(record["messages"]): + if message.get("role") != "assistant": + continue + tool_calls = message.get("tool_calls", []) + if tool_calls is None: + tool_calls = [] + if not isinstance(tool_calls, list): + raise ValueError(f"ms_swift messages[{message_index}].tool_calls must be a list") + for call_index, call in enumerate(tool_calls): + path = f"ms_swift messages[{message_index}].tool_calls[{call_index}].function" + function = call.get("function") if isinstance(call, dict) else None + if not isinstance(function, dict): + raise ValueError(f"{path} must be an object") + name = function.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"{path}.name must be a non-empty string") + arguments = function.get("arguments", {}) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError as error: + raise ValueError(f"{path}.arguments must contain a JSON object") from error + if not isinstance(arguments, dict): + raise ValueError(f"{path}.arguments must be an object") + item = {"tool_name": name, "args": arguments} + calls.append(item) + return { + "RuleAgentTraceSensitiveAccess_content": json.dumps( + {"tool_calls": calls}, ensure_ascii=False, + ), + } + + +def prepare_rule_agent_trace_token_budget(record: Dict[str, Any]) -> Dict[str, str]: + """Do not substitute reasoning_total_tokens for total token usage.""" + content = {"total_tokens": record["total_tokens"]} if "total_tokens" in record else {} + return {"RuleAgentTraceTokenBudget_content": json.dumps(content, ensure_ascii=False)} + + +def prepare_llm_agent_argument_correctness(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare the initial conversation-only argument correctness input.""" + return { + "LLMAgentArgumentCorrectness_prompt": "", + "LLMAgentArgumentCorrectness_content": _messages_content(record), + } + + +def prepare_llm_agent_error_recovery(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare conversation evidence without inventing structured error events.""" + return { + "LLMAgentErrorRecovery_prompt": "", + "LLMAgentErrorRecovery_content": _messages_content(record), + } + + +def prepare_llm_agent_plan_adherence(record: Dict[str, Any]) -> Dict[str, str]: + """Leave both the original plan and the task goal explicitly unset.""" + return { + "LLMAgentPlanAdherence_prompt": "", + "LLMAgentPlanAdherence_content": _messages_content(record), + "LLMAgentPlanAdherence_context": "", + } + + +def prepare_llm_agent_plan_quality(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare the initial conversation-only plan quality input.""" + return { + "LLMAgentPlanQuality_prompt": "", + "LLMAgentPlanQuality_content": _messages_content(record), + } + + +def prepare_llm_agent_step_efficiency(record: Dict[str, Any]) -> Dict[str, str]: + """Prepare the initial conversation-only step efficiency input.""" + return { + "LLMAgentStepEfficiency_prompt": "", + "LLMAgentStepEfficiency_content": _messages_content(record), + } + + +_PREPARERS = { + "LLMAgentTaskCompletion": prepare_llm_agent_task_completion, + "RuleAgentTraceLoopDetection": prepare_rule_agent_trace_loop_detection, + "LLMAgentToolCorrectness": prepare_llm_agent_tool_correctness, + "RuleAgentTraceDestructiveAction": prepare_rule_agent_trace_destructive_action, + "RuleAgentTraceGatewayBypass": prepare_rule_agent_trace_gateway_bypass, + "RuleAgentTraceIntegrity": prepare_rule_agent_trace_integrity, + "RuleAgentTraceLatencyAnomaly": prepare_rule_agent_trace_latency_anomaly, + "RuleAgentTraceSandboxExplicitlyDisabled": prepare_rule_agent_trace_sandbox_explicitly_disabled, + "RuleAgentTraceSecretExposure": prepare_rule_agent_trace_secret_exposure, + "RuleAgentTraceSensitiveAccess": prepare_rule_agent_trace_sensitive_access, + "RuleAgentTraceTokenBudget": prepare_rule_agent_trace_token_budget, + "LLMAgentArgumentCorrectness": prepare_llm_agent_argument_correctness, + "LLMAgentErrorRecovery": prepare_llm_agent_error_recovery, + "LLMAgentPlanAdherence": prepare_llm_agent_plan_adherence, + "LLMAgentPlanQuality": prepare_llm_agent_plan_quality, + "LLMAgentStepEfficiency": prepare_llm_agent_step_efficiency, +} + + +@BaseConverter.register("ms_swift") +class MsSwiftConverter(BaseConverter): + """Convert one ms-swift record into one Data with evaluator-specific fields.""" + + @classmethod + def convertor(cls, input_args: InputArgs) -> Callable: + selected_names = { + evaluator.name + for pipeline in input_args.evaluator + for evaluator in pipeline.evals + } + preparers = [prepare for name, prepare in _PREPARERS.items() if name in selected_names] + + def _convert(raw: Union[str, Dict[str, Any]]) -> Data: + record = json.loads(raw) if isinstance(raw, str) else raw + if not isinstance(record, dict): + raise ValueError("ms_swift record must be a JSON object") + messages = record.get("messages") + if not isinstance(messages, list): + raise ValueError("ms_swift messages must be a list") + for index, message in enumerate(messages): + if not isinstance(message, dict): + raise ValueError(f"ms_swift messages[{index}] must be an object") + + converted = dict(record) + derived_fields = set() + for prepare in preparers: + extra_fields = prepare(record) + conflicts = extra_fields.keys() & converted.keys() + if conflicts: + raise ValueError( + f"ms_swift derived fields conflict with existing fields: {sorted(conflicts)}" + ) + converted.update(extra_fields) + derived_fields.update(extra_fields) + data = Data(**converted) + data._derived_fields = derived_fields + return data + + return _convert diff --git a/dingo/data/datasource/local.py b/dingo/data/datasource/local.py index a8176b7a..5b6d00f1 100644 --- a/dingo/data/datasource/local.py +++ b/dingo/data/datasource/local.py @@ -51,7 +51,7 @@ def _find_all_files(self, path: str, file_list: List[str]): for _f in os.listdir(path): f = os.path.join(path, _f) if os.path.isfile(f): - if getattr(self.input_args.dataset, "format", None) == "jsonl": + if getattr(self.input_args.dataset, "format", None) in ("jsonl", "ms_swift"): if f.lower().endswith(".jsonl"): file_list.append(f) elif getattr(self.input_args.dataset, "format", None) == "json": diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 5db2d284..bb395e2b 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -90,7 +90,11 @@ def execute(self) -> SummaryModel: futures_results = [] for data in batch: dingo_id += 1 - r_i = ResultInfo(dingo_id = str(dingo_id), raw_data = data.to_dict()) + r_i = ResultInfo( + dingo_id=str(dingo_id), + raw_data=data.to_dict(), + derived_fields=data._derived_fields, + ) futures_results.append(r_i) for e_p in self.input_args.evaluator: @@ -419,6 +423,8 @@ def _should_write_to_file(self, input_args: InputArgs, file_path: str) -> bool: def _build_output_json(self, input_args: InputArgs, result_info: ResultInfo) -> str: field_list = self._resolve_field_list(input_args) + if input_args.dataset.format == "ms_swift": + result_info = result_info.without_derived_fields() if input_args.executor.result_save.raw: output_data = result_info.to_raw_dict(field_list=field_list) else: diff --git a/dingo/io/input/data.py b/dingo/io/input/data.py index 921b5a16..18258da6 100644 --- a/dingo/io/input/data.py +++ b/dingo/io/input/data.py @@ -1,6 +1,6 @@ -from typing import Any, Dict +from typing import Any, Dict, Set -from pydantic import BaseModel +from pydantic import BaseModel, PrivateAttr class Data(BaseModel): @@ -9,6 +9,9 @@ class Data(BaseModel): Flexible data structure that allows any fields to be configured. """ + # Per-record converter metadata; never included in evaluator field mappings. + _derived_fields: Set[str] = PrivateAttr(default_factory=set) + class Config: extra = "allow" diff --git a/dingo/io/output/result_info.py b/dingo/io/output/result_info.py index 8a03a4ee..3e1d2c8d 100644 --- a/dingo/io/output/result_info.py +++ b/dingo/io/output/result_info.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from datetime import date, datetime from decimal import Decimal -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set from pydantic import BaseModel, Field @@ -23,6 +23,15 @@ class ResultInfo(BaseModel): default_factory=dict, exclude=True, ) + derived_fields: Set[str] = Field(default_factory=set, exclude=True) + + def without_derived_fields(self) -> "ResultInfo": + """Return an output copy without converter-generated top-level raw fields.""" + cleaned_raw_data = { + key: value for key, value in self.raw_data.items() + if key not in self.derived_fields + } + return self.model_copy(update={"raw_data": cleaned_raw_data}) @staticmethod def _eval_detail_to_dict(model_res: EvalDetail) -> Dict[str, Any]: diff --git a/docs/agent_trace/ms_swift.md b/docs/agent_trace/ms_swift.md new file mode 100644 index 00000000..287fd7a3 --- /dev/null +++ b/docs/agent_trace/ms_swift.md @@ -0,0 +1,1114 @@ +# ms-swift converter 用户说明 + +使用 `dataset.format="ms_swift"` 读取 Agent 轨迹 JSONL：每行一个 JSON 对象， +包含 `messages` 数组，一行转换成一个评估样本。本地目录输入只读取 `.jsonl` 文件。 +当前支持以下 16 个指标，只处理主轨迹，不展开 `sub_agent_trajectory`。 + +converter 只为 `evaluator[].evals` 中选中的指标生成辅助字段，并保留原始记录。 +每个指标单独配置 `evaluator[].fields`，映射到自己的 `<指标名>_<字段名>`。 +全部指标同时选择时共生成 24 个辅助字段；同一指标重复配置时，每条记录只转换一次。 + +下文每节都给出 **fields 配置、源字段映射、转换步骤、输入示例和转换结果**。 +`content` 辅助字段的实际类型是 **JSON 字符串**；为便于阅读，示例展示其 +**解析后的 JSON 值**。表格中的 `tool_calls`、`steps` 等路径也指解析后的值。 +`prompt` 和 `context` 的实际值则在各节中单独说明。示例只展示输入适配，不代表指标已判定通过或失败。 + +## 一、输入约定 + +- `messages` 必须是数组，每条消息必须是对象。LLM 指标读取全部消息；确定性规则按各自需要读取工具调用或顶层统计。 +- 读取工具调用时，仅读取 `role="assistant"` 的 `tool_calls`；该键缺失或为 `null` 时按空列表处理，其他非数组值会报错。 +- 每次调用必须有对象类型的 `function` 和非空字符串 `function.name`。`arguments` 缺失时为 `{}`；对象原样使用，JSON 字符串解析为对象，显式 `null`、数组或无效 JSON 均会报错。 +- 需要配对工具结果的指标只使用非空字符串 ID。数字、空字符串或缺失 ID 不参与配对；有效调用 ID 或有效工具返回 ID 重复时会报错。 + +## 二、确定性规则 + +### 1. RuleAgentTraceDestructiveAction（破坏性操作） + +将每次工具调用的名称和参数交给规则，供其扫描删除、数据库破坏或提权命令。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceDestructiveAction_content" +} +``` + +生成 `RuleAgentTraceDestructiveAction_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `messages[role=assistant].tool_calls[].function.name` | `tool_calls[].tool_name` | 原样复制 | +| `messages[role=assistant].tool_calls[].function.arguments` | `tool_calls[].args` | 对象直接使用；JSON 字符串解析为对象 | + +**转换步骤** + +1. 按消息顺序遍历所有 `assistant` 消息，再按各消息中的 `tool_calls` 顺序遍历调用。 +2. 每个调用生成一个对象，`function.name` 写入 `tool_name`，解析后的 `function.arguments` 写入 `args`；缺少参数时使用 `{}`。 +3. 参数键和值全部保留。`cmd` 不改名为 `command`，也不从命令内容推断工具类型。 +4. 将这些对象装入 `{"tool_calls": [...]}`，序列化后写入 `RuleAgentTraceDestructiveAction_content`。不加入调用 ID、工具返回或执行状态。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + {"id":"delete-1","function":{"name":"exec_command","arguments":"{\"cmd\":\"rm -rf /srv/demo\"}"}}, + {"id":"delete-2","function":{"name":"shell","arguments":{"command":"DROP TABLE demo","timeout":30}}} + ] + } + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_calls": [ + {"tool_name":"exec_command","args":{"cmd":"rm -rf /srv/demo"}}, + {"tool_name":"shell","args":{"command":"DROP TABLE demo","timeout":30}} + ] +} +``` + +**缺失情况与使用边界** + +没有工具调用时输出 `{"tool_calls": []}`。现有规则读取 `args.command/source/script`，不读取 `args.cmd`；因此第一条调用的 `cmd` 虽然成功转换，仍可能漏检。 + +### 2. RuleAgentTraceGatewayBypass（网关绕过） + +将工具调用、网关返回和明确的执行状态放到同一个调用对象中，供规则识别受阻后使用代码直接联网的行为。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceGatewayBypass_content" +} +``` + +生成 `RuleAgentTraceGatewayBypass_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `assistant.tool_calls[].function.name` | `tool_calls[].tool_name` | 原样复制 | +| `assistant.tool_calls[].function.arguments` | `tool_calls[].args` | 解析为对象，保留原始参数名 | +| `assistant.tool_calls[].id` | `tool_calls[].call_id` | 仅保留非空字符串 | +| 匹配的 `tool` 消息的 `content` | `tool_calls[].result` | 按 ID 匹配；字段存在时原样复制 | +| 调用或匹配工具消息的 `status/duration/duration_seconds/group/type` | 调用对象上的同名字段 | 逐字段取值，工具消息优先 | + +**转换步骤** + +1. 先收集所有 `role="tool"` 消息，以非空字符串 `tool_call_id` 建立索引。 +2. 按原始顺序遍历 `assistant.tool_calls`，解析名称和参数。调用的非空字符串 `id` 写入 `call_id`，再用该 ID 查找工具消息。 +3. 匹配消息有 `content` 键时写入 `result`；没有匹配消息或没有该键时，省略 `result`。显式 `content=null` 会保留为 `result=null`。 +4. 对 `status`、`duration`、`duration_seconds`、`group`、`type` 分别检查：匹配工具消息有该键时使用其值，否则使用调用上的值；两处都没有则省略。即使工具消息的值为 `null`，也不会回退到调用值。 +5. 输出 `{"tool_calls": [...]}`。返回顺序不改变调用顺序，也不从 `"GATEWAY BLOCKED"` 等文本推断 `status`。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "web-1", + "type": "function", + "status": "timeout", + "duration": 9, + "function": {"name":"WebFetch","arguments":{"url":"https://example.org"}} + }, + { + "id": "exec-1", + "status": "success", + "function": {"name":"shell","arguments":{"command":"curl https://example.org"}} + } + ] + }, + {"role":"tool","tool_call_id":"web-1","content":"GATEWAY BLOCKED","status":"error","duration":0.1} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_calls": [ + { + "tool_name": "WebFetch", + "args": {"url":"https://example.org"}, + "call_id": "web-1", + "result": "GATEWAY BLOCKED", + "status": "error", + "duration": 0.1, + "type": "function" + }, + {"tool_name":"shell","args":{"command":"curl https://example.org"},"call_id":"exec-1","status":"success"} + ] +} +``` + +**缺失情况与使用边界** + +示例中 `web-1` 的工具消息覆盖了调用上的 `status` 和 `duration`；`exec-1` 没有工具返回，仍保留调用及其显式状态。重复有效调用 ID 或工具返回 ID 会报错。现有规则不识别 `exec_command` 或 `args.cmd`；缺少状态时沿用原有判定，不自动转成 N/A。 + +### 3. RuleAgentTraceIntegrity（轨迹完整性） + +将当前文件中的工具调用计数和配对情况转换成完整性统计字段。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceIntegrity_content" +} +``` + +生成 `RuleAgentTraceIntegrity_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| 顶层 `total_tool_calls` | `tool_calls_expected` | 仅在源键存在时复制，不重新估算或转换类型 | +| 所有 `assistant.tool_calls` 中的实际调用数量 | `tool_spans_recorded` | 逐次统计，始终输出整数 | +| 没有有效 ID 或没有同 ID 工具消息的调用数量 | `open_observation_count` | 逐次统计，始终输出整数 | +| 顶层 `trace_truncated` | `trace_truncated` | 仅在源键存在时原样复制 | + +**转换步骤** + +1. 索引所有带非空字符串 `tool_call_id` 的工具消息，并遍历全部 `assistant.tool_calls`。当前实现仍校验调用名称和参数结构，但不把名称和参数写入统计结果。 +2. 每遇到一个有效结构的调用，`tool_spans_recorded` 加 1。 +3. 调用的 `id` 缺失、为空或不是字符串，或者没有相同 `tool_call_id` 的工具消息时，`open_observation_count` 加 1。配对只检查 ID，匹配工具消息不必有 `content`。 +4. 源记录有 `total_tool_calls` 时复制为 `tool_calls_expected`；有 `trace_truncated` 时复制同名值。缺少时省略这两个字段，不推断截断状态。 +5. 序列化统计对象。仅有工具返回而没有对应调用，不会增加调用数或未配对调用数。 + +**输入示例** + +```json +{ + "total_tool_calls": 3, + "trace_truncated": true, + "messages": [ + { + "role": "assistant", + "tool_calls": [ + {"id":"a","function":{"name":"search","arguments":{"query":"Dingo"}}}, + {"id":"b","function":{"name":"read","arguments":{"path":"notes.txt"}}} + ] + }, + {"role":"tool","tool_call_id":"b"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_spans_recorded": 2, + "open_observation_count": 1, + "tool_calls_expected": 3, + "trace_truncated": true +} +``` + +**缺失情况与使用边界** + +示例中实际有 2 次调用，只有 `b` 配对成功，因此未配对调用数为 1。没有调用且无顶层统计字段时输出 `{"tool_spans_recorded": 0, "open_observation_count": 0}`。重复有效 ID 会报错。这只能检查交付文件内部一致性；现有规则不一定将未配对或截断信息本身判为缺陷。 + +### 4. RuleAgentTraceLatencyAnomaly（耗时异常） + +将工具调用转换成带名称、显式耗时和分组信息的步骤列表。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceLatencyAnomaly_content" +} +``` + +生成 `RuleAgentTraceLatencyAnomaly_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `assistant.tool_calls[].function.name` | `steps[].name` | 原样复制，目标键是 `name` | +| 调用或匹配工具消息的 `duration` | `steps[].duration` | 工具消息有该键时优先 | +| 调用或匹配工具消息的 `duration_seconds` | `steps[].duration_seconds` | 独立复制，不换算单位 | +| 调用或匹配工具消息的 `group/type` | `steps[].group/type` | 逐字段复制，不从工具名称推断 | + +**转换步骤** + +1. 索引工具消息，按调用的非空字符串 `id` 与 `tool_call_id` 配对，保持调用顺序。仍校验调用参数，即使它们不会出现在输出中。 +2. 每个调用先生成 `{"name": function.name}`。 +3. 对 `duration`、`duration_seconds`、`group`、`type` 分别取值：匹配工具消息有该键时使用其值，否则用调用上的值，两处都没有则省略。两种耗时字段同时存在时会同时保留。 +4. 没有耗时的调用也保留一个只有 `name` 的步骤。不根据相邻消息、开始结束时间或返回文本计算耗时。 +5. 输出 `{"steps": [...]}`，不加入参数、调用 ID、返回内容或 `status`。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + {"id":"a","duration":2,"group":"source","function":{"name":"search","arguments":{"query":"Dingo"}}}, + {"id":"b","duration_seconds":4,"type":"source","function":{"name":"summarize","arguments":{}}}, + {"function":{"name":"save","arguments":{"path":"summary.txt"}}} + ] + }, + {"role":"tool","tool_call_id":"b","content":"done","duration_seconds":3,"type":"inference"}, + {"role":"tool","tool_call_id":"a","content":"found","duration":1,"group":"retrieval"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "steps": [ + {"name":"search","duration":1,"group":"retrieval"}, + {"name":"summarize","duration_seconds":3,"type":"inference"}, + {"name":"save"} + ] +} +``` + +**缺失情况与使用边界** + +工具返回顺序为 `b`、`a`，输出仍按 `a`、`b`、`save` 排列。没有调用时输出 `{"steps": []}`。重复有效 ID 会报错。耗时有效性及异常阈值由规则判断；某个分组至少有 3 个有效正耗时才参与检测。规则优先读取 `duration` 和 `group`，缺少对应键时再读取 `duration_seconds` 和 `type`。 + +### 5. RuleAgentTraceLoopDetection（重复调用循环） + +保留完整的工具名、参数及调用顺序，供规则比较重复调用序列。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceLoopDetection_content" +} +``` + +生成 `RuleAgentTraceLoopDetection_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `assistant.tool_calls[].function.name` | `tool_calls[].tool_name` | 原样复制 | +| `assistant.tool_calls[].function.arguments` | `tool_calls[].args` | 解析为对象，保留全部参数 | + +**转换步骤** + +1. 按消息顺序扫描 `assistant` 消息，再按每条消息内的数组顺序扫描工具调用。 +2. 将名称和解析后的参数组装成 `{"tool_name": ..., "args": ...}`。参数缺失时使用 `{}`。 +3. 跨消息拼接所有调用，不排序、不去重、不只保留工具名称。相同工具名但参数不同的调用仍然不同。 +4. 输出 `{"tool_calls": [...]}`。不关联工具结果，也不输出调用 ID、状态或耗时；重复 ID 不影响这个转换。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + {"id":"a","function":{"name":"search","arguments":"{\"query\":\"Dingo\"}"}}, + {"id":"b","function":{"name":"read","arguments":{"path":"notes.txt"}}} + ] + }, + {"role":"tool","tool_call_id":"b","content":"document"}, + {"role":"assistant","tool_calls":[{"id":"a","function":{"name":"search","arguments":{"query":"Dingo"}}}]} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_calls": [ + {"tool_name":"search","args":{"query":"Dingo"}}, + {"tool_name":"read","args":{"path":"notes.txt"}}, + {"tool_name":"search","args":{"query":"Dingo"}} + ] +} +``` + +**缺失情况与使用边界** + +没有调用时输出 `{"tool_calls": []}`。转换不预先判定循环；现有规则少于 6 个有效调用时返回 N/A，因此本例仅展示如何保留序列。 + +### 6. RuleAgentTraceSandboxExplicitlyDisabled（显式关闭沙箱） + +把工具参数中的沙箱关闭标志交给规则，同时保留其原始值。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceSandboxExplicitlyDisabled_content" +} +``` + +生成 `RuleAgentTraceSandboxExplicitlyDisabled_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `assistant.tool_calls[].function.name` | `tool_calls[].tool_name` | 原样复制 | +| `assistant.tool_calls[].function.arguments` | `tool_calls[].args` | 解析后保留全部键和值 | +| 参数中的 `dangerouslyDisableSandbox` | `tool_calls[].args.dangerouslyDisableSandbox` | 保持原始类型，不转换为布尔值 | + +**转换步骤** + +1. 按顺序读取每个 `assistant` 工具调用，将 `function.name` 写入 `tool_name`。 +2. 参数对象原样保留；JSON 字符串解析为对象；缺少参数时使用 `{}`。 +3. 不只提取 `dangerouslyDisableSandbox`，其他参数同样保留；该标志缺失时不补 `false`，存在时不修改值。 +4. 输出 `{"tool_calls": [...]}`，不关联工具返回或执行状态。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "sandbox-1", + "function": { + "name": "shell", + "arguments": "{\"command\":\"echo demo\",\"dangerouslyDisableSandbox\":\"true\",\"timeout\":30}" + } + } + ] + } + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_calls": [{"tool_name":"shell","args":{"command":"echo demo","dangerouslyDisableSandbox":"true","timeout":30}}] +} +``` + +**缺失情况与使用边界** + +示例中的 `"true"` 在输出中仍是字符串，其含义由规则判断。没有调用时输出 `{"tool_calls": []}`；没有标志时，converter 不推断沙箱实际是否开启。 + +### 7. RuleAgentTraceSecretExposure（工具返回中的敏感凭证） + +将真实工具返回放入对应调用的 `result`，供规则扫描敏感凭证。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceSecretExposure_content" +} +``` + +生成 `RuleAgentTraceSecretExposure_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `assistant.tool_calls[].function.name/arguments` | `tool_calls[].tool_name/args` | 名称复制，参数解析为对象 | +| 调用的非空字符串 `id` | `tool_calls[].call_id` | 保留并用于结果配对 | +| 同 ID 的工具消息 `content` | `tool_calls[].result` | 原样复制，保留字符串、对象或 `null` | +| 调用或匹配工具消息的 `status/duration/duration_seconds/group/type` | 调用对象上的同名字段 | 逐字段复制，工具消息优先 | + +**转换步骤** + +1. 先按非空字符串 `tool_call_id` 索引工具消息，再按原始顺序遍历工具调用。 +2. 每次调用保留名称和完整参数；有效 ID 写入 `call_id`。 +3. 只把同 ID 工具消息的 `content` 写入 `result`。不按返回顺序配对，不扫描其他消息来补齐结果，也不读取调用自身的 `result`。 +4. 找不到工具消息，或匹配消息没有 `content` 键时，省略 `result`；`content=null` 则保留 `result=null`。 +5. 现有转换还保留显式 `status`、`duration`、`duration_seconds`、`group`、`type`，逐字段优先使用匹配工具消息，再回退到调用。 +6. 输出 `{"tool_calls": [...]}`。敏感凭证识别由规则完成，converter 不先替换或脱敏返回内容。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + {"id":"a","function":{"name":"search","arguments":{"query":"Dingo"}}}, + {"id":"b","status":"pending","function":{"name":"read","arguments":{"path":"demo.pem"}}}, + {"id":"c","function":{"name":"save","arguments":{"path":"summary.txt"}}} + ] + }, + { + "role": "tool", + "tool_call_id": "b", + "content": "-----BEGIN PRIVATE KEY-----\nEXAMPLE\n-----END PRIVATE KEY-----", + "status": "success" + }, + {"role":"tool","tool_call_id":"a","content":"found"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_calls": [ + {"tool_name":"search","args":{"query":"Dingo"},"call_id":"a","result":"found"}, + { + "tool_name": "read", + "args": {"path":"demo.pem"}, + "call_id": "b", + "result": "-----BEGIN PRIVATE KEY-----\nEXAMPLE\n-----END PRIVATE KEY-----", + "status": "success" + }, + {"tool_name":"save","args":{"path":"summary.txt"},"call_id":"c"} + ] +} +``` + +**缺失情况与使用边界** + +示例使用虚构凭证文本。`b` 的返回虽然先出现，仍只写入 `b`；`c` 没有返回，因此没有 `result`。没有调用时输出 `{"tool_calls": []}`，重复有效 ID 会报错。没有匹配结果的调用不会提供可供规则扫描的返回证据。 + +### 8. RuleAgentTraceSensitiveAccess（敏感路径访问） + +把工具参数按原始层级保留下来，供规则检查敏感文件或目录路径。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceSensitiveAccess_content" +} +``` + +生成 `RuleAgentTraceSensitiveAccess_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| `assistant.tool_calls[].function.name` | `tool_calls[].tool_name` | 原样复制 | +| `assistant.tool_calls[].function.arguments` | `tool_calls[].args` | 解析为对象，保留嵌套字典和数组 | + +**转换步骤** + +1. 按顺序读取 `assistant` 工具调用，将工具名称写入 `tool_name`。 +2. 解析参数，缺少参数时使用 `{}`。不把嵌套字典展开为一级字段，也不把参数拼接成一段文本。 +3. 保留 `path`、`cmd`、`files` 等所有真实参数名和值，不根据名称选择或改写路径。 +4. 输出 `{"tool_calls": [...]}`，不关联工具返回、调用 ID 或状态。 + +**输入示例** + +```json +{ + "messages": [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "read-1", + "function": {"name":"read","arguments":{"path":".env","options":{"path":"~/.ssh/id_rsa"},"files":["notes.txt"]}} + } + ] + } + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "tool_calls": [{"tool_name":"read","args":{"path":".env","options":{"path":"~/.ssh/id_rsa"},"files":["notes.txt"]}}] +} +``` + +**缺失情况与使用边界** + +示例中的一级 `path` 和嵌套 `options.path` 均被保留，但现有规则只扫描一级简单参数值，不递归扫描嵌套字典或数组；嵌套路径仍可能漏检。没有调用时输出 `{"tool_calls": []}`。 + +### 9. RuleAgentTraceTokenBudget（总 Token 预算） + +把源记录明确提供的总 Token 用量交给规则。 + +**fields 配置** + +```json +{ + "content": "RuleAgentTraceTokenBudget_content" +} +``` + +生成 `RuleAgentTraceTokenBudget_content`，类型为 JSON 字符串。 + +**读取与映射** + +| 源字段或统计方式 | 转换后的路径 | 处理方式 | +|---|---|---| +| 顶层 `total_tokens` | `total_tokens` | 源键存在时原样复制，不进行数值转换 | +| 顶层 `reasoning_total_tokens` | 无对应输出 | 不作为总用量的替代值 | + +**转换步骤** + +1. 检查顶层是否存在 `total_tokens` 键。 +2. 存在时生成 `{"total_tokens": record["total_tokens"]}`，保留源值及类型；即使值为 `0` 或 `null`，也不因其真假而省略。 +3. 不存在时生成空对象 `{}`，不读取 `reasoning_total_tokens` 或 `metadata.total_tokens`，也不累加消息中的用量。 +4. 序列化对象，写入 `RuleAgentTraceTokenBudget_content`；本转换不解析工具调用。 + +**输入示例** + +```json +{ + "total_tokens": 560000, + "reasoning_total_tokens": 900000, + "messages": [] +} +``` + +**转换结果**（`content` 解析后） + +```json +{ + "total_tokens": 560000 +} +``` + +**缺失情况与使用边界** + +本例只复制 560000。若删除顶层 `total_tokens`，即使 `reasoning_total_tokens=900000`，输出仍为 `{}`。总用量是否有效以及是否超出预算由规则判断；没有可用用量时返回 N/A。 + +## 三、LLM 指标 + +本版 7 个 LLM 指标统一使用 `_messages_content()`，但分别生成自己的辅助字段。 +下面每节独立说明该指标的输入适配方式；它们暂未分别实现工具、计划或错误事件提取。 + +### 10. LLMAgentArgumentCorrectness（工具参数正确性） + +为参数正确性评估准备会话文本；本版没有单独提取结构化工具参数。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentArgumentCorrectness_prompt", + "content": "LLMAgentArgumentCorrectness_content" +} +``` + +生成 `LLMAgentArgumentCorrectness_content`（JSON 字符串）和 `LLMAgentArgumentCorrectness_prompt`（空字符串 `""`）。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentArgumentCorrectness_content`；`LLMAgentArgumentCorrectness_prompt` 固定为空字符串。 + +**输入示例** + +```json +{ + "prompt": "检索 Dingo 的安装文档", + "tools": [{"name":"search","description":"搜索文档"}], + "messages": [ + {"role":"user","content":"请搜索 Dingo 的安装文档"}, + { + "role": "assistant", + "content": "开始搜索", + "tool_calls": [{"id":"a","function":{"name":"search","arguments":{"query":"Dingo installation"}}}] + }, + {"role":"tool","tool_call_id":"a","content":"找到安装文档"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"user","content":"请搜索 Dingo 的安装文档"}, + {"role":"assistant","content":"开始搜索"}, + {"role":"tool","content":"找到安装文档"} +] +``` + +示例中的 `function.arguments.query` 和工具定义不会进入辅助 `content`。用户请求保留在会话里，但不会被抽取为 `prompt`。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`。如果参数只存在于 `tool_calls`，LLM 在这版输入中看不到具体参数，参数正确性评分的证据可能不足。 + +### 11. LLMAgentToolCorrectness（工具选择正确性） + +为工具选择评估准备会话文本；本版不额外传入可用工具定义或调用名称。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentToolCorrectness_prompt", + "content": "LLMAgentToolCorrectness_content" +} +``` + +生成 `LLMAgentToolCorrectness_content`（JSON 字符串）和 `LLMAgentToolCorrectness_prompt`（空字符串 `""`）。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentToolCorrectness_content`；`LLMAgentToolCorrectness_prompt` 固定为空字符串。 + +**输入示例** + +```json +{ + "tools": [{"name":"search","description":"检索资料"},{"name":"read","description":"读取文件"}], + "messages": [ + {"role":"developer","content":"需要查资料时使用搜索工具"}, + {"role":"user","content":"查找 Dingo 文档"}, + { + "role": "assistant", + "content": "我来检索", + "tool_calls": [{"id":"a","function":{"name":"search","arguments":{"query":"Dingo"}}}] + }, + {"role":"tool","tool_call_id":"a","name":"search"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"developer","content":"需要查资料时使用搜索工具"}, + {"role":"user","content":"查找 Dingo 文档"}, + {"role":"assistant","content":"我来检索"}, + {"role":"tool","content":""} +] +``` + +`developer` 消息也会保留；最后一条工具消息缺少 `content`，因此补为空字符串。工具定义、`tool_calls` 及工具消息的 `name` 均省略。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`。如果工具名称和职责只存在于结构化调用或工具定义中，LLM 无法从本版输入直接获得这些证据。 + +### 12. LLMAgentErrorRecovery（错误恢复） + +为错误恢复评估准备完整顺序的消息文本；本版不构建独立错误事件或重试记录。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentErrorRecovery_prompt", + "content": "LLMAgentErrorRecovery_content" +} +``` + +生成 `LLMAgentErrorRecovery_content`（JSON 字符串）和 `LLMAgentErrorRecovery_prompt`（空字符串 `""`）。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentErrorRecovery_content`；`LLMAgentErrorRecovery_prompt` 固定为空字符串。 + +**输入示例** + +```json +{ + "messages": [ + {"role":"user","content":"检索资料并总结"}, + { + "role": "assistant", + "content": "正在检索", + "tool_calls": [{"id":"a","function":{"name":"search","arguments":{"query":"Dingo"}}}] + }, + { + "role": "tool", + "tool_call_id": "a", + "status": "error", + "content": "{\"status\":\"error\",\"message\":\"timeout\"}" + }, + {"role":"assistant","content":"检索超时，我将重试"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"user","content":"检索资料并总结"}, + {"role":"assistant","content":"正在检索"}, + {"role":"tool","content":"{\"status\":\"error\",\"message\":\"timeout\"}"}, + {"role":"assistant","content":"检索超时，我将重试"} +] +``` + +工具消息的顶层 `status` 不进入辅助 `content`；它的 `content` 本身是一段 JSON 字符串，仍作为字符串保留，不解析为错误对象。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`。converter 不检查是否发生错误、不识别重试链路。非空精简会话可能触发 LLM 评估，即使会话里没有已确认的错误事件。 + +### 13. LLMAgentPlanAdherence（计划遵循程度） + +准备会话消息，并为计划遵循指标提供空的任务目标与计划上下文字段。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentPlanAdherence_prompt", + "content": "LLMAgentPlanAdherence_content", + "context": "LLMAgentPlanAdherence_context" +} +``` + +生成 `LLMAgentPlanAdherence_content`（JSON 字符串）和 `LLMAgentPlanAdherence_prompt`（空字符串 `""`）。 +另生成 `LLMAgentPlanAdherence_context`，值同样为 `""`。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| 源记录的计划或 `context` | 辅助 `context` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentPlanAdherence_content`；`LLMAgentPlanAdherence_prompt` 固定为空字符串。 +4. 将 `LLMAgentPlanAdherence_context` 固定设为空字符串，独立于源记录的上下文。 + +**输入示例** + +```json +{ + "prompt": "比较 A 和 B", + "context": "原始计划：先检索 A，再检索 B，最后比较", + "messages": [ + {"role":"user","content":"请比较 A 和 B"}, + {"role":"assistant","content":"先检索 A","reasoning_content":"之后再检索 B"}, + {"role":"assistant","content":"已经完成比较"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"user","content":"请比较 A 和 B"}, + {"role":"assistant","content":"先检索 A"}, + {"role":"assistant","content":"已经完成比较"} +] +``` + +即使原始记录有 `prompt` 和 `context`，本转换仍生成空的辅助 `prompt` 和 `context`；原始字段继续留在原始记录中。`reasoning_content` 也不会进入精简会话。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`，`prompt/context` 仍为空。当前没有独立的原始计划或任务目标输入，无法把成功转换直接理解为计划遵循证据已完整。 + +### 14. LLMAgentPlanQuality（计划质量） + +为计划质量评估准备会话文本；本版不从推理字段中提取计划。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentPlanQuality_prompt", + "content": "LLMAgentPlanQuality_content" +} +``` + +生成 `LLMAgentPlanQuality_content`（JSON 字符串）和 `LLMAgentPlanQuality_prompt`（空字符串 `""`）。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentPlanQuality_content`；`LLMAgentPlanQuality_prompt` 固定为空字符串。 + +**输入示例** + +```json +{ + "messages": [ + {"role":"user","content":"查资料后写一份总结"}, + {"role":"assistant","content":"","reasoning_content":"先检索资料，再核对来源，最后总结"}, + {"role":"assistant","content":"总结已完成"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"user","content":"查资料后写一份总结"}, + {"role":"assistant","content":""}, + {"role":"assistant","content":"总结已完成"} +] +``` + +示例的计划只出现在 `reasoning_content` 中，转换后该计划文本不在辅助 `content` 里；空的 assistant `content` 按原值保留。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`。只有消息 `content` 中明确写出的计划会随文本保留；本版不识别、整理或推断计划，计划质量评分可能缺少计划证据。 + +### 15. LLMAgentStepEfficiency（执行步骤效率） + +为步骤效率评估准备会话文本；本版不把工具调用展开成独立的执行步骤。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentStepEfficiency_prompt", + "content": "LLMAgentStepEfficiency_content" +} +``` + +生成 `LLMAgentStepEfficiency_content`（JSON 字符串）和 `LLMAgentStepEfficiency_prompt`（空字符串 `""`）。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentStepEfficiency_content`；`LLMAgentStepEfficiency_prompt` 固定为空字符串。 + +**输入示例** + +```json +{ + "total_tool_calls": 2, + "messages": [ + {"role":"user","content":"查找并阅读 Dingo 文档"}, + { + "role": "assistant", + "content": "我来查找并阅读", + "tool_calls": [ + {"id":"a","function":{"name":"search","arguments":{"query":"Dingo"}}}, + {"id":"b","function":{"name":"read","arguments":{"path":"notes.txt"}}} + ] + }, + {"role":"tool","tool_call_id":"a","content":"已找到文档"}, + {"role":"tool","tool_call_id":"b","content":"文档内容"}, + {"role":"assistant","content":"阅读完成"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"user","content":"查找并阅读 Dingo 文档"}, + {"role":"assistant","content":"我来查找并阅读"}, + {"role":"tool","content":"已找到文档"}, + {"role":"tool","content":"文档内容"}, + {"role":"assistant","content":"阅读完成"} +] +``` + +一条 assistant 消息里有两次工具调用，转换后仍只是一条 assistant 消息，不展开成两个步骤。顶层 `total_tool_calls` 和工具消息的 `tool_call_id` 也不进入辅助 `content`。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`。本版不统计执行步骤、不建立调用与返回的结构化关系，也不划分多轮任务边界；这些信息不足可能影响效率评分。 + +### 16. LLMAgentTaskCompletion（任务完成情况） + +为任务完成情况评估准备完整顺序的会话文本；本版不单独抽取任务目标。 + +**fields 配置** + +```json +{ + "prompt": "LLMAgentTaskCompletion_prompt", + "content": "LLMAgentTaskCompletion_content" +} +``` + +生成 `LLMAgentTaskCompletion_content`（JSON 字符串）和 `LLMAgentTaskCompletion_prompt`（空字符串 `""`）。 + +**读取与映射** + +| 源字段 | 转换后的路径或字段 | 处理方式 | +|---|---|---| +| `messages[].role` | `content[].role` | 原样保留，键缺失时使用 `""` | +| `messages[].content` | `content[].content` | 原样保留，键缺失时使用 `""`，不解析其中的 JSON 文本 | +| 源记录的任务描述或 `prompt` | 辅助 `prompt` | 不提取或复制，固定为 `""` | +| `tool_calls/reasoning_content/tools` 及其他消息、顶层字段 | 无对应输出 | 不进入精简会话 | + +**转换步骤** + +1. 调用 `_messages_content()`，按原始数组顺序遍历所有消息，保留所有角色，不筛选 assistant 或 tool。 +2. 每条消息生成 `{"role": message.get("role", ""), "content": message.get("content", "")}`，其他键省略；字段显式为 `null` 时保留 `null`，不会替换为空字符串。 +3. 将消息列表序列化为 JSON 字符串，写入 `LLMAgentTaskCompletion_content`；`LLMAgentTaskCompletion_prompt` 固定为空字符串。 + +**输入示例** + +```json +{ + "prompt": "列出 Dingo 的安装步骤", + "total_tool_calls": 1, + "reasoning_total_tokens": 100, + "messages": [ + {"role":"user","content":"请列出 Dingo 的安装步骤"}, + {"role":"assistant","content":"先运行 pip install dingo-python，再准备配置并执行评估","reasoning_content":"已经覆盖全部步骤"} + ] +} +``` + +**转换结果**（`content` 解析后） + +```json +[ + {"role":"user","content":"请列出 Dingo 的安装步骤"}, + {"role":"assistant","content":"先运行 pip install dingo-python，再准备配置并执行评估"} +] +``` + +用户要求和最终回答保留在精简会话中，但源记录的 `prompt` 不会复制到辅助 `prompt`。调用计数、Token 用量和 `reasoning_content` 不进入辅助 `content`。 + +**缺失情况与使用边界** + +没有消息时 `content` 为字符串 `"[]"`。本版保留会话里的任务文本，但不将其提取为独立目标，也不划分多任务轨迹；任务完成评分需考虑这些输入边界。 + +## 四、配置与运行 + +每节的 **fields 配置** 放入该指标自己的 `evaluator` 配置块， +`evals[].name` 使用章节中的完整指标名。例如： + +```json +{ + "input_path": "data/agent_traces.jsonl", + "dataset": {"source":"local","format":"ms_swift"}, + "executor": {"result_save":{"good":true,"raw":true}}, + "evaluator": [ + { + "fields": {"content":"RuleAgentTraceLoopDetection_content"}, + "evals": [{"name":"RuleAgentTraceLoopDetection"}] + }, + { + "fields": { + "prompt": "LLMAgentPlanAdherence_prompt", + "content": "LLMAgentPlanAdherence_content", + "context": "LLMAgentPlanAdherence_context" + }, + "evals": [ + { + "name": "LLMAgentPlanAdherence", + "config": {"model":"YOUR_MODEL","api_url":"https://your-endpoint/v1","key":"YOUR_API_KEY"} + } + ] + } + ] +} +``` + +完整 16 个指标的配置见 [ms_swift.json](../../examples/dataset/ms_swift.json)。 +修改输入路径，并为选中的 LLM 指标填写模型配置后运行： + +```bash +dingo eval --input examples/dataset/ms_swift.json +``` + +## 五、结果输出 + +这些辅助字段只用于质检输入。本地执行器写出结果时，自动删除本条记录实际生成的辅助字段， +无需维护删除清单；原始记录及未生成的同名前缀字段仍按现有输出规则保留。 + +| 配置 | 输出形态 | 辅助字段清理位置 | +|---|---|---| +| `result_save.raw=true` | 原始字段与 `dingo_id`、`dingo_result` 一起输出 | 输出的原始字段中 | +| `result_save.raw=false` | 评估结果与 `raw_data` 一起输出 | `raw_data` 中 | + +`field_list` 筛选发生在辅助字段清理之后；已有 `dingo_id_old_v1` 等字段冲突备份沿用原逻辑。 +`eval_details` 中用于追溯输入的映射名称仍然保留，辅助字段值不再写入原始数据输出。 + +若某个辅助字段与原始记录字段同名，转换会报错，避免覆盖原始数据。 +未选中指标的同名原始字段不会被清理。错误定位会包含消息或调用索引，便于修正输入。 diff --git a/docs/config.md b/docs/config.md index c96891a2..f403a060 100644 --- a/docs/config.md +++ b/docs/config.md @@ -33,7 +33,7 @@ | Parameter | Type | Default | Required | Description | |-----------|------|---------|----------|-------------| | source | str | "hugging_face" | Yes | 数据源类型，可选值：['hugging_face', 'local'] | -| format | str | "json" | Yes | 数据格式，可选值：['json', 'jsonl', 'plaintext', 'listjson', 'csv', 'parquet', 'md', 'mineru', 'mineru_v2'] | +| format | str | "json" | Yes | 数据格式，可选值：['json', 'jsonl', 'ms_swift', 'plaintext', 'listjson', 'csv', 'parquet', 'md', 'mineru', 'mineru_v2'] | | field | object | - | Yes | 字段映射配置 | | hf_config | object | - | No | HuggingFace 特定配置 | | mineru_config | object | - | No | MinerU 格式特定配置（仅 mineru / mineru_v2 格式使用） | @@ -55,6 +55,13 @@ MinerU 支持的 block 类型包括：`text`, `title`, `image`, `table`, `equati 每个 block 会被转换为一条 `Data` 记录。block 的 `type`、`bbox`、`page_idx` 等原始字段会通过 Pydantic `extra="allow"` 保留在 Data 对象上。 +#### ms-swift 格式 (dataset.format = "ms_swift") + +读取 Agent 轨迹 JSONL，每行一个包含 `messages` 数组的对象。保留原始字段， +只为 `evaluator[].evals` 中选中的已支持指标生成 `<指标名>_<字段名>` 字段。 +每个指标使用独立的 `fields` 映射配置块。当前支持 9 个确定性规则和 7 个 LLM 指标， +映射及配置示例见 [ms-swift converter 用户说明](agent_trace/ms_swift.md)。 + #### DatasetField 配置 (dataset.field) 字段映射配置： diff --git a/examples/agent/ms-swift.py b/examples/agent/ms-swift.py new file mode 100644 index 00000000..458e1b99 --- /dev/null +++ b/examples/agent/ms-swift.py @@ -0,0 +1,46 @@ +from pathlib import Path + +from dingo.config import InputArgs +from dingo.exec import Executor + +config = { + "model": "", + "key": "", + "api_url": "" +} + +input_data = { + "input_path": r"", + "dataset": {"source": "local", "format": "ms_swift"}, + "executor": {"result_save":{"good":True, "raw":True}}, + "evaluator": [ + { + "fields": { + "prompt": "LLMAgentTaskCompletion_prompt", + "content": "LLMAgentTaskCompletion_content" + }, + "evals": [{"name": "LLMAgentTaskCompletion","config": config}] + }, + { + "fields": {"content": "RuleAgentTraceLoopDetection_content"}, + "evals": [{"name": "RuleAgentTraceLoopDetection"}] + }, + { + "fields": { + "prompt": "LLMAgentToolCorrectness_prompt", + "content": "LLMAgentToolCorrectness_content" + }, + "evals": [{"name": "LLMAgentToolCorrectness","config": config}] + }, + { + "fields": {"content": "RuleAgentTraceDestructiveAction_content"}, + "evals": [{"name": "RuleAgentTraceDestructiveAction"}] + } + ] +} + +if __name__ == '__main__': + input_args = InputArgs(**input_data) + executor = Executor.exec_map["local"](input_args) + result = executor.execute() + print(result) diff --git a/test/scripts/dataset/test_ms_swift_agent_metrics.py b/test/scripts/dataset/test_ms_swift_agent_metrics.py new file mode 100644 index 00000000..929e1aa9 --- /dev/null +++ b/test/scripts/dataset/test_ms_swift_agent_metrics.py @@ -0,0 +1,292 @@ +"""Remaining ms-swift metrics, including evidence gaps and output cleanup.""" + +import copy +import importlib +import json + +import pytest + +from dingo.config import InputArgs +from dingo.data.converter import converters +from dingo.exec import LocalExecutor +from dingo.io import Data, ResultInfo +from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI +from dingo.model.rule import rule_agent + +RULE_NAMES = ( + "RuleAgentTraceDestructiveAction", "RuleAgentTraceGatewayBypass", "RuleAgentTraceIntegrity", + "RuleAgentTraceLatencyAnomaly", "RuleAgentTraceLoopDetection", "RuleAgentTraceSandboxExplicitlyDisabled", + "RuleAgentTraceSecretExposure", "RuleAgentTraceSensitiveAccess", "RuleAgentTraceTokenBudget", +) +LLM_MODULES = { + "LLMAgentArgumentCorrectness": "llm_agent_argument_correctness", + "LLMAgentToolCorrectness": "llm_agent_tool_correctness", + "LLMAgentErrorRecovery": "llm_agent_error_recovery", + "LLMAgentPlanAdherence": "llm_agent_plan_adherence", + "LLMAgentPlanQuality": "llm_agent_plan_quality", + "LLMAgentStepEfficiency": "llm_agent_step_efficiency", + "LLMAgentTaskCompletion": "llm_agent_task_completion", +} +LLM_MODELS = { + name: getattr(importlib.import_module(f"dingo.model.llm.agent_eval.{module}"), name) + for name, module in LLM_MODULES.items() +} +ALL_NAMES = RULE_NAMES + tuple(LLM_MODELS) + + +def make_args(names=ALL_NAMES, **kwargs): + pipelines = [] + for name in names: + fields = {"content": f"{name}_content"} + if name.startswith("LLM"): + fields["prompt"] = f"{name}_prompt" + if name == "LLMAgentPlanAdherence": + fields["context"] = f"{name}_context" + pipelines.append({"fields": fields, "evals": [{"name": name}]}) + return InputArgs( + dataset={"source": "local", "format": "ms_swift"}, evaluator=pipelines, **kwargs, + ) + + +def convert(record, names=ALL_NAMES): + return converters["ms_swift"].convertor(make_args(names))(record) + + +def evaluate_rule(record, name): + data = convert(record, (name,)) + return getattr(rule_agent, name).eval(Data(content=getattr(data, f"{name}_content"))) + + +@pytest.fixture +def record(): + return { + "data_id": "trace-1", "total_tool_calls": 2, "total_tokens": 600001, + "tools": [{"name": "search"}], + "messages": [ + {"role": "user", "content": "查找文档并总结结果"}, + {"role": "assistant", "content": "", "reasoning_content": "这是推理，不一定是计划", "tool_calls": [ + {"id": "a", "function": {"name": "search", "arguments": {"query": "documents"}}}, + {"id": "b", "function": {"name": "read", "arguments": {"path": "notes.txt"}}}, + ]}, + {"role": "tool", "tool_call_id": "b", "content": "document"}, + {"role": "tool", "tool_call_id": "a", "content": "found"}, + {"role": "assistant", "content": "已完成"}, + ], + } + + +@pytest.mark.parametrize("name", ALL_NAMES) +def test_each_metric_generates_only_its_actual_fields(record, name): + original = copy.deepcopy(record) + args = make_args((name,)) + data = converters["ms_swift"].convertor(args)(record) + expected = set(args.evaluator[0].fields.values()) + assert data._derived_fields == expected + assert set(data.to_dict()) - set(record) == expected + assert record == original + + +@pytest.mark.parametrize("raw", [False, True]) +def test_all_16_metrics_keep_independent_inputs_and_cleanup(record, raw): + args = make_args(executor={"result_save": {"raw": raw}}) + data = converters["ms_swift"].convertor(args)(record) + assert len(data._derived_fields) == 24 + result = ResultInfo(dingo_id="1", raw_data=data.to_dict(), derived_fields=data._derived_fields) + output = json.loads(LocalExecutor(args)._build_output_json(args, result)) + source = output if raw else output["raw_data"] + assert {key: source[key] for key in record} == record + assert not data._derived_fields.intersection(source) + assert result.raw_data.keys() >= data._derived_fields + + +@pytest.mark.parametrize("name", tuple(LLM_MODELS)) +def test_all_llm_inputs_follow_the_selected_simplified_contract(record, monkeypatch, name): + args = make_args((name,)) + data = convert(record, (name,)).to_dict() + mapped = Data(**{key: data[source] for key, source in args.evaluator[0].fields.items()}) + assert mapped.prompt == "" + assert json.loads(mapped.content) == [ + {"role": message["role"], "content": message["content"]} for message in record["messages"] + ] + if name == "LLMAgentPlanAdherence": + assert mapped.context == "" + model = LLM_MODELS[name] + requests = [] + + def send(cls, messages): + requests.append(messages) + return '{"score": 8, "reason": "mock evaluation"}' + + monkeypatch.setattr(BaseOpenAI, "create_client", classmethod(lambda cls: None)) + monkeypatch.setattr(BaseOpenAI, "send_messages", classmethod(send)) + result = model.eval(mapped) + assert result.applicable + assert result.score == 0.8 + assert mapped.content in requests[0][0]["content"] + + +def test_secret_results_are_paired_by_id_even_when_return_order_differs(record): + record["messages"][2]["content"] = "-----BEGIN PRIVATE KEY-----" + name = "RuleAgentTraceSecretExposure" + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert [call["result"] for call in payload["tool_calls"]] == ["found", "-----BEGIN PRIVATE KEY-----"] + result = evaluate_rule(record, name) + assert result.status + assert "call 2" in result.reason[0] + + +def test_missing_result_is_not_guessed_from_an_unrelated_tool_message(record): + record["messages"][2]["tool_call_id"] = "unrelated" + payload = json.loads(convert(record).RuleAgentTraceSecretExposure_content) + assert "result" not in payload["tool_calls"][1] + integrity = json.loads(convert(record).RuleAgentTraceIntegrity_content) + assert integrity["open_observation_count"] == 1 + + +@pytest.mark.parametrize("name", ( + "RuleAgentTraceDestructiveAction", "RuleAgentTraceGatewayBypass", "RuleAgentTraceLoopDetection", + "RuleAgentTraceSandboxExplicitlyDisabled", "RuleAgentTraceSecretExposure", "RuleAgentTraceSensitiveAccess", +)) +def test_tool_call_preparers_parse_json_arguments_without_changing_source(record, name): + record["messages"][1]["tool_calls"][1]["function"]["arguments"] = '{"path": "notes.txt"}' + original = copy.deepcopy(record) + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert payload["tool_calls"][1]["args"] == {"path": "notes.txt"} + assert record == original + + +@pytest.mark.parametrize("name", RULE_NAMES[:-1]) +def test_rule_preparers_reject_invalid_arguments_without_changing_source(record, name): + record["messages"][1]["tool_calls"][1]["function"]["arguments"] = "invalid JSON" + original = copy.deepcopy(record) + with pytest.raises(ValueError, match=r"messages\[1\].tool_calls\[1\].function.arguments"): + convert(record, (name,)) + assert record == original + + +@pytest.mark.parametrize("name", ( + "RuleAgentTraceGatewayBypass", "RuleAgentTraceIntegrity", + "RuleAgentTraceLatencyAnomaly", "RuleAgentTraceSecretExposure", +)) +@pytest.mark.parametrize("duplicate_call", [False, True]) +def test_ambiguous_call_ids_fail_instead_of_using_the_wrong_result(record, name, duplicate_call): + if duplicate_call: + record["messages"][1]["tool_calls"][1]["id"] = "a" + else: + record["messages"][2]["tool_call_id"] = "a" + with pytest.raises(ValueError, match="duplicate"): + convert(record, (name,)) + + +def test_gateway_maps_explicit_status_without_changing_rule_behavior(): + record = {"messages": [ + {"role": "assistant", "tool_calls": [ + {"id": "a", "function": {"name": "WebFetch", "arguments": {"url": "https://example.org"}}}, + ]}, + {"role": "tool", "tool_call_id": "a", "status": "error", "content": "GATEWAY BLOCKED"}, + {"role": "assistant", "tool_calls": [ + {"id": "b", "function": {"name": "shell", "arguments": {"command": "curl https://example.org"}}}, + ]}, + ]} + name = "RuleAgentTraceGatewayBypass" + assert evaluate_rule(record, name).status + record["messages"][1].pop("status") + assert not evaluate_rule(record, name).status + record["messages"][2]["tool_calls"][0]["function"] = { + "name": "exec_command", "arguments": {"cmd": "curl https://example.org"}, + } + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert "status" not in payload["tool_calls"][0] + assert payload["tool_calls"][1]["tool_name"] == "exec_command" + assert payload["tool_calls"][1]["args"] == {"cmd": "curl https://example.org"} + + +def test_integrity_compares_actual_count_with_source_counter(record): + name = "RuleAgentTraceIntegrity" + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert payload == {"tool_calls_expected": 2, "tool_spans_recorded": 2, "open_observation_count": 0} + record["total_tool_calls"] = 3 + assert evaluate_rule(record, name).status + record.pop("total_tool_calls") + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert "tool_calls_expected" not in payload + assert "trace_truncated" not in payload + + +def test_latency_is_na_without_explicit_timings_and_uses_recorded_extensions(record): + name = "RuleAgentTraceLatencyAnomaly" + assert not evaluate_rule(record, name).applicable + calls = record["messages"][1]["tool_calls"] + record["messages"][1]["tool_calls"] = [ + {"function": calls[0]["function"], "duration": 1 if index < 10 else 100, "group": "tool"} + for index in range(11) + ] + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert payload["steps"][-1] == {"name": "search", "duration": 100, "group": "tool"} + assert evaluate_rule(record, name).status + + +def test_latency_pairs_timings_by_id_and_prefers_tool_message_extensions(record): + name = "RuleAgentTraceLatencyAnomaly" + record["messages"][1]["tool_calls"][0].update(duration=10, group="call", type="source") + record["messages"][1]["tool_calls"][1].update(duration_seconds=20, group="fallback") + record["messages"][3].update(duration=1, group="observation", type="tool") + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert payload["steps"] == [ + {"name": "search", "duration": 1, "group": "observation", "type": "tool"}, + {"name": "read", "duration_seconds": 20, "group": "fallback"}, + ] + + +@pytest.mark.parametrize("name,args", [ + ("RuleAgentTraceSandboxExplicitlyDisabled", {"dangerouslyDisableSandbox": True}), + ("RuleAgentTraceSensitiveAccess", {"path": "~/.ssh/id_rsa"}), + ("RuleAgentTraceDestructiveAction", {"command": "rm -rf /srv/production"}), +]) +def test_parameter_rules_use_original_argument_names(record, name, args): + record["messages"][1]["tool_calls"][0]["function"]["arguments"] = args + assert evaluate_rule(record, name).status + + +def test_cmd_is_preserved_without_extending_the_existing_rule(record): + name = "RuleAgentTraceDestructiveAction" + args = {"cmd": "rm -rf /srv/production"} + record["messages"][1]["tool_calls"][0]["function"]["arguments"] = args + payload = json.loads(getattr(convert(record, (name,)), f"{name}_content")) + assert payload["tool_calls"][0]["args"] == args + assert not evaluate_rule(record, name).status + + +def test_token_budget_never_substitutes_reasoning_tokens(record): + name = "RuleAgentTraceTokenBudget" + assert evaluate_rule(record, name).status + record.pop("total_tokens") + record["reasoning_total_tokens"] = 900000 + assert json.loads(getattr(convert(record, (name,)), f"{name}_content")) == {} + assert not evaluate_rule(record, name).applicable + + +@pytest.mark.parametrize("raw", [False, True]) +def test_local_executor_runs_all_16_metrics_and_cleans_outputs(tmp_path, monkeypatch, record, raw): + monkeypatch.setenv("LOCAL_DEPLOYMENT_MODE", "true") + monkeypatch.setattr(BaseOpenAI, "create_client", classmethod(lambda cls: None)) + monkeypatch.setattr( + BaseOpenAI, "send_messages", classmethod(lambda cls, messages: '{"score": 8, "reason": "mock"}'), + ) + path = tmp_path / "trace.jsonl" + path.write_text(json.dumps(record, ensure_ascii=False), encoding="utf-8") + args = make_args( + input_path=str(path), output_path=str(tmp_path / "outputs"), + executor={"max_workers": 2, "result_save": {"raw": raw, "merge": True, "all_labels": True}}, + ) + executor = LocalExecutor(args) + summary = executor.execute() + assert summary.total == 1 + output = executor.get_bad_info_list()[0] + source = output if raw else output["raw_data"] + assert {key: source[key] for key in record} == record + assert set(source) == set(record) | ({"dingo_id", "dingo_result"} if raw else set()) + details = output["dingo_result"]["eval_details"] if raw else output["eval_details"] + assert {detail["metric"] for group in details.values() for detail in group} == set(ALL_NAMES) + assert set(ALL_NAMES) <= set(Model.rule_name_map) | set(Model.llm_name_map) diff --git a/test/scripts/dataset/test_ms_swift_converter.py b/test/scripts/dataset/test_ms_swift_converter.py new file mode 100644 index 00000000..35f9728c --- /dev/null +++ b/test/scripts/dataset/test_ms_swift_converter.py @@ -0,0 +1,215 @@ +"""Verify evaluator isolation, source preservation and usable rule inputs.""" + +import copy +import importlib +import json +import pickle + +import pytest + +from dingo.config import InputArgs +from dingo.data.converter import converters +from dingo.data.dataset.local import LocalDataset +from dingo.data.datasource.local import LocalDataSource +from dingo.io import Data +from dingo.model.llm.agent_eval.llm_agent_task_completion import LLMAgentTaskCompletion +from dingo.model.llm.agent_eval.llm_agent_tool_correctness import LLMAgentToolCorrectness +from dingo.model.rule.rule_agent import RuleAgentTraceDestructiveAction, RuleAgentTraceLoopDetection + +EVALUATORS = ( + "LLMAgentTaskCompletion", + "RuleAgentTraceLoopDetection", + "LLMAgentToolCorrectness", + "RuleAgentTraceDestructiveAction", +) + + +def make_args(names=EVALUATORS, **kwargs): + return InputArgs( + dataset={"source": "local", "format": "ms_swift"}, + evaluator=[{"evals": [{"name": name}]} for name in names], + **kwargs, + ) + + +@pytest.fixture +def record(): + return { + "data_id": "trace-1", + "tools": [{"name": "search", "description": "Search documents"}], + "metadata": {"source": "example"}, + "messages": [ + {"role": "developer", "content": "遵循用户要求"}, + {"role": "user", "content": "查找资料"}, + { + "role": "assistant", "content": "", "reasoning_content": "先搜索", + "tool_calls": [ + {"id": "call-1", "function": {"name": "search", "arguments": {"query": "资料"}}}, + {"id": "call-2", "function": {"name": "read", "arguments": '{"path": "notes.txt"}'}}, + ], + }, + {"role": "tool", "name": "search", "tool_call_id": "call-1", "content": "找到了"}, + {"role": "tool", "name": "read", "tool_call_id": "call-2", "content": "资料正文"}, + {"role": "assistant", "content": "已完成"}, + ], + } + + +@pytest.mark.parametrize("as_json", [False, True]) +def test_independent_fields_and_source_preservation(record, as_json): + original = copy.deepcopy(record) + raw = json.dumps(record, ensure_ascii=False) if as_json else record + data = converters["ms_swift"].convertor(make_args())(raw) + converted = data.to_dict() + assert record == original + assert {key: converted[key] for key in original} == original + assert set(converted) - set(original) == { + "LLMAgentTaskCompletion_prompt", "LLMAgentTaskCompletion_content", + "RuleAgentTraceLoopDetection_content", + "LLMAgentToolCorrectness_prompt", "LLMAgentToolCorrectness_content", + "RuleAgentTraceDestructiveAction_content", + } + assert data._derived_fields == set(converted) - set(original) + assert "_derived_fields" not in data.model_dump() + for name in ("LLMAgentTaskCompletion", "LLMAgentToolCorrectness"): + assert converted[f"{name}_prompt"] == "" + assert "资料" in converted[f"{name}_content"] + assert json.loads(converted[f"{name}_content"]) == [ + {"role": "developer", "content": "遵循用户要求"}, + {"role": "user", "content": "查找资料"}, + {"role": "assistant", "content": ""}, + {"role": "tool", "content": "找到了"}, + {"role": "tool", "content": "资料正文"}, + {"role": "assistant", "content": "已完成"}, + ] + for name in ("RuleAgentTraceLoopDetection", "RuleAgentTraceDestructiveAction"): + assert json.loads(converted[f"{name}_content"]) == {"tool_calls": [ + {"tool_name": "search", "args": {"query": "资料"}}, + {"tool_name": "read", "args": {"path": "notes.txt"}}, + ]} + + +@pytest.mark.parametrize("name", EVALUATORS) +def test_generates_only_selected_evaluator_fields(record, name): + data = converters["ms_swift"].convertor(make_args((name,)))(record) + converted = data.to_dict() + added = set(converted) - set(record) + expected = {f"{name}_content"} + if name.startswith("LLM"): + expected.add(f"{name}_prompt") + assert added == expected + assert data._derived_fields == expected + + +def test_llm_conversion_does_not_parse_omitted_tool_calls(record): + record["messages"][2]["tool_calls"] = "not a valid tool call list" + converted = converters["ms_swift"].convertor(make_args(("LLMAgentTaskCompletion",)))(record) + assert json.loads(converted.LLMAgentTaskCompletion_content)[2] == {"role": "assistant", "content": ""} + + +@pytest.mark.parametrize("names", [(), ("RuleAbnormalChar",)]) +def test_no_supported_evaluator_preserves_original_only(record, names): + data = converters["ms_swift"].convertor(make_args(names))(record) + assert data.to_dict() == record + assert data._derived_fields == set() + + +def test_tracks_actual_fields_per_record_without_a_static_list(monkeypatch): + module = importlib.import_module("dingo.data.converter.ms_swift") + + def prepare(record): + return {f"custom_{record['data_id']}": "derived"} + + monkeypatch.setitem(module._PREPARERS, "RuleAgentTraceLoopDetection", prepare) + convert = converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",))) + first = convert({"messages": [], "data_id": "first"}) + second = convert({"messages": [], "data_id": "second"}) + assert first._derived_fields == {"custom_first"} + assert second._derived_fields == {"custom_second"} + assert pickle.loads(pickle.dumps(first))._derived_fields == {"custom_first"} + + +def test_rejects_derived_field_collision_without_overwriting_source(record): + field = "RuleAgentTraceLoopDetection_content" + record[field] = "user value" + original = copy.deepcopy(record) + with pytest.raises(ValueError, match=field): + converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(record) + assert record == original + + +def test_converted_sequence_can_detect_loops(record): + calls = record["messages"][2]["tool_calls"] + record["messages"][2]["tool_calls"] = calls * 3 + converted = converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(record) + result = RuleAgentTraceLoopDetection.eval(Data(content=converted.RuleAgentTraceLoopDetection_content)) + assert result.applicable + assert result.status + assert "Loop detected" in result.reason[0] + # Six calls with different arguments must not turn into a name-only loop. + record["messages"][2]["tool_calls"] = [ + {"function": {"name": "search", "arguments": {"query": str(index)}}} + for index in range(6) + ] + converted = converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(record) + result = RuleAgentTraceLoopDetection.eval(Data(content=converted.RuleAgentTraceLoopDetection_content)) + assert result.applicable + assert not result.status + + +def test_destructive_rule_input_and_unchanged_cmd(record): + record["messages"][2]["tool_calls"] = [ + {"function": {"name": "shell", "arguments": {"command": "rm -rf /important"}}}, + {"function": {"name": "exec_command", "arguments": {"cmd": "rm -rf /other"}}}, + ] + converted = converters["ms_swift"].convertor(make_args(("RuleAgentTraceDestructiveAction",)))(record) + content = converted.RuleAgentTraceDestructiveAction_content + assert json.loads(content)["tool_calls"][1]["args"] == {"cmd": "rm -rf /other"} + result = RuleAgentTraceDestructiveAction.eval(Data(content=content)) + assert result.status + assert "call 1" in result.reason[0] + + +def test_field_mapping_builds_llm_messages_with_empty_prompts(record): + args = make_args() + for pipeline in args.evaluator: + name = pipeline.evals[0].name + pipeline.fields = {"content": f"{name}_content"} + if name.startswith("LLM"): + pipeline.fields["prompt"] = f"{name}_prompt" + converted = converters["ms_swift"].convertor(args)(record).to_dict() + models = {"LLMAgentTaskCompletion": LLMAgentTaskCompletion, "LLMAgentToolCorrectness": LLMAgentToolCorrectness} + for pipeline in args.evaluator: + name = pipeline.evals[0].name + mapped = Data(**{key: converted[source] for key, source in pipeline.fields.items()}) + if name in models: + assert mapped.prompt == "" + messages = models[name].build_messages(mapped) + assert mapped.content in messages[0]["content"] + else: + assert isinstance(json.loads(mapped.content)["tool_calls"], list) + + +def test_local_dataset_reads_jsonl_records_and_filters_directory(tmp_path, record): + records = [record, dict(record, data_id="trace-2")] + (tmp_path / "traces.jsonl").write_text( + "\n".join(json.dumps(item, ensure_ascii=False) for item in records), encoding="utf-8", + ) + (tmp_path / "unrelated.json").write_text("[]", encoding="utf-8") + args = make_args(("RuleAgentTraceLoopDetection",), input_path=str(tmp_path)) + data = list(LocalDataset(LocalDataSource(args)).get_data()) + assert [item.data_id for item in data] == ["trace-1", "trace-2"] + assert all(json.loads(item.RuleAgentTraceLoopDetection_content)["tool_calls"] for item in data) + + +@pytest.mark.parametrize("raw, message", [ + ([], "record must be a JSON object"), + ({}, "messages must be a list"), + ({"messages": [None]}, r"messages\[0\] must be an object"), + ({"messages": [{"role": "assistant", "tool_calls": {}}]}, "tool_calls must be a list"), + ({"messages": [{"role": "assistant", "tool_calls": [{"function": {"name": "x", "arguments": "bad"}}]}]}, + "arguments must contain a JSON object"), +]) +def test_invalid_structure_has_actionable_error(raw, message): + with pytest.raises(ValueError, match=message): + converters["ms_swift"].convertor(make_args(("RuleAgentTraceLoopDetection",)))(raw) diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index 48f870ab..605240d7 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -1,3 +1,5 @@ +import json + import pytest from dingo.config import InputArgs @@ -8,6 +10,89 @@ class TestLocal: + @pytest.mark.parametrize("raw", [False, True]) + @pytest.mark.parametrize("merge", [False, True]) + def test_ms_swift_evaluates_derived_fields_and_cleans_saved_records( + self, tmp_path, monkeypatch, raw, merge, + ): + monkeypatch.setenv("LOCAL_DEPLOYMENT_MODE", "true") + field = "RuleAgentTraceLoopDetection_content" + records = [] + for index in range(2): + records.append({ + "data_id": f"trace-{index}", + "metadata": {field: "nested source value"}, + "LLMAgentToolCorrectness_content": "unselected source field", + "messages": [{"role": "assistant", "tool_calls": [ + {"function": {"name": "search", "arguments": { + "query": "same" if index == 0 else str(call), + }}} for call in range(6) + ]}], + }) + input_path = tmp_path / "traces.jsonl" + input_path.write_text( + "\n".join(json.dumps(record) for record in records), encoding="utf-8", + ) + args = InputArgs( + input_path=str(input_path), output_path=str(tmp_path / "outputs"), + dataset={"source": "local", "format": "ms_swift"}, + executor={"batch_size": 2, "max_workers": 2, "result_save": { + "raw": raw, "merge": merge, "good": True, "all_labels": True, + }}, + evaluator=[{"fields": {"content": field}, "evals": [ + {"name": "RuleAgentTraceLoopDetection"}, + ]}], + ) + executor = LocalExecutor(args) + summary = executor.execute() + assert (summary.total, summary.num_bad, summary.num_good) == (2, 1, 1) + good = executor.get_good_info_list() + bad = executor.get_bad_info_list() + assert len(good) == len(bad) == 1 + for output in bad + good: + source = output if raw else output["raw_data"] + assert field not in source + assert "derived_fields" not in output + assert "_derived_fields" not in source + index = int(source["data_id"].split("-")[1]) + assert {key: source[key] for key in records[index]} == records[index] + details = output["dingo_result"]["eval_details"] if raw else output["eval_details"] + assert details[field][0]["metric"] == "RuleAgentTraceLoopDetection" + + @pytest.mark.parametrize("raw", [False, True]) + def test_ms_swift_cleanup_precedes_field_filter_and_full_field_samples(self, raw): + args = InputArgs( + dataset={"source": "local", "format": "ms_swift"}, + executor={"result_save": { + "raw": raw, "full_field_sample_count": 1, + "field_list": ["dingo_id"] if raw else ["dingo_id", "raw_data"], + }}, + ) + executor = LocalExecutor(args) + result = ResultInfo( + dingo_id="1", raw_data={"data_id": "source", "derived": "temporary"}, + derived_fields={"derived"}, + ) + first = json.loads(executor._build_output_json(args, result)) + second = json.loads(executor._build_output_json(args, result)) + assert "derived" not in (first if raw else first["raw_data"]) + if raw: + assert first["data_id"] == "source" + assert second == {"dingo_id": "1"} + else: + assert second == {"dingo_id": "1", "raw_data": {"data_id": "source"}} + assert result.raw_data["derived"] == "temporary" + + @pytest.mark.parametrize("raw", [False, True]) + def test_non_ms_swift_output_does_not_remove_tracked_fields(self, raw): + args = InputArgs( + dataset={"source": "local", "format": "jsonl"}, + executor={"result_save": {"raw": raw}}, + ) + result = ResultInfo(raw_data={"content": "original"}, derived_fields={"content"}) + output = json.loads(LocalExecutor(args)._build_output_json(args, result)) + assert (output if raw else output["raw_data"])["content"] == "original" + def test_execute_preserves_user_provided_task_id(self, tmp_path): input_path = tmp_path / "input.jsonl" input_path.write_text('{"content": "test"}\n', encoding="utf-8") diff --git a/test/scripts/io/output/test_result_info.py b/test/scripts/io/output/test_result_info.py index c0345053..504eb752 100644 --- a/test/scripts/io/output/test_result_info.py +++ b/test/scripts/io/output/test_result_info.py @@ -7,6 +7,40 @@ class TestResultInfo: + @pytest.mark.parametrize("raw", [False, True]) + def test_without_derived_fields_preserves_source_and_results(self, raw): + field = "RuleAgentTraceLoopDetection_content" + original = { + "messages": [{"role": "user", "content": "test"}], + "metadata": {field: "nested source value"}, + "LLMAgentTaskCompletion_prompt": "untracked source value", + field: "derived value", + "dingo_id": "source-id", + "dingo_result": {"source": True}, + "dingo_id_old_v1": "earlier-id", + } + result = ResultInfo( + dingo_id="new-id", raw_data=original, eval_status=True, + derived_fields={field}, + ) + cleaned = result.without_derived_fields() + assert cleaned.raw_data == {key: value for key, value in original.items() if key != field} + assert result.raw_data == original + assert cleaned.without_derived_fields().raw_data == cleaned.raw_data + assert "derived_fields" not in cleaned.model_dump() + output = cleaned.to_raw_dict() if raw else cleaned.to_dict() + source = output if raw else output["raw_data"] + assert field not in source + assert source["metadata"][field] == "nested source value" + assert source["LLMAgentTaskCompletion_prompt"] == "untracked source value" + if raw: + assert output["dingo_id_old_v1"] == "earlier-id" + assert output["dingo_id_old_v2"] == "source-id" + assert output["dingo_result_old_v1"] == {"source": True} + else: + assert source["dingo_id"] == "source-id" + assert output["eval_status"] is True + def test_to_raw_dict_field_list_none_no_filter(self): result_info = ResultInfo( dingo_id="dingo-1",