From 550f1b113f867afb2fdf28ff6cb9092616b7d7e0 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 19 Oct 2026 17:36:10 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20InputArgs=E5=B1=8F=E8=94=BD?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=96=B0=E5=A2=9E5=E4=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/config/input_args.py | 11 +++++++++-- test/scripts/io/test_summary_model.py | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/dingo/config/input_args.py b/dingo/config/input_args.py index df7bebf8..18b7a05f 100644 --- a/dingo/config/input_args.py +++ b/dingo/config/input_args.py @@ -204,13 +204,20 @@ def to_dict(self) -> Dict[str, Any]: else: data = self.dict() - excluded = {"key"} + excluded = { + "api_key", + "api_token", + "key", + "password", + "s3_ak", + "s3_sk", + } pending = [data] while pending: value = pending.pop() if isinstance(value, dict): for key in list(value): - if key in excluded: + if isinstance(key, str) and key.lower() in excluded: del value[key] else: pending.append(value[key]) diff --git a/test/scripts/io/test_summary_model.py b/test/scripts/io/test_summary_model.py index bdbe6dc9..d2806b59 100644 --- a/test/scripts/io/test_summary_model.py +++ b/test/scripts/io/test_summary_model.py @@ -83,6 +83,22 @@ def test_input_args_are_serialized(self): def test_input_args_to_dict_recursively_excludes_fields(self): input_args = InputArgs(**{ + "dataset": { + "s3_config": { + "s3_ak": "access-key", + "s3_sk": "secret-key", + "s3_bucket": "test-bucket", + }, + "sql_config": { + "username": "test-user", + "password": "secret-password", + }, + }, + "executor": { + "retrieval": { + "api_token": "secret-token", + }, + }, "evaluator": [{ "fields": {"content": "content"}, "evals": [{ @@ -90,6 +106,9 @@ def test_input_args_to_dict_recursively_excludes_fields(self): "config": { "model": "test-model", "key": "secret-key", + "tool_config": { + "API_KEY": "secret-tool-key", + }, }, }], }], @@ -98,7 +117,14 @@ def test_input_args_to_dict_recursively_excludes_fields(self): serialized = input_args.to_dict() config = serialized["evaluator"][0]["evals"][0]["config"] + assert "s3_ak" not in serialized["dataset"]["s3_config"] + assert "s3_sk" not in serialized["dataset"]["s3_config"] + assert serialized["dataset"]["s3_config"]["s3_bucket"] == "test-bucket" + assert "password" not in serialized["dataset"]["sql_config"] + assert serialized["dataset"]["sql_config"]["username"] == "test-user" + assert "api_token" not in serialized["executor"]["retrieval"] assert "key" not in config + assert "API_KEY" not in config["tool_config"] assert config["model"] == "test-model" def test_add_metric_score_single(self): From 0467c83535b675edd9acdad8d36e4773d8cb8fdb Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 19 Oct 2026 18:53:56 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat:=20=E6=94=B9=E6=88=90add=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/config/input_args.py | 148 +++++++++++++++++++++----- test/scripts/io/test_summary_model.py | 4 +- 2 files changed, 125 insertions(+), 27 deletions(-) diff --git a/dingo/config/input_args.py b/dingo/config/input_args.py index 18b7a05f..8d312b56 100644 --- a/dingo/config/input_args.py +++ b/dingo/config/input_args.py @@ -197,31 +197,127 @@ class InputArgs(BaseModel): evaluator: List[EvalPipline] = [] def to_dict(self) -> Dict[str, Any]: - """Return a config snapshot, recursively excluding sensitive fields.""" + """Return a config snapshot containing only explicitly approved fields.""" + # Security allowlist: fields omitted here are excluded by default. + safe_serialization_fields: Dict[str, Any] = { + "task_id": ..., + "task_name": ..., + "input_path": ..., + "output_path": ..., + "log_level": ..., + "dataset": { + "source": ..., + "format": ..., + "hf_config": { + "huggingface_split", + "huggingface_config_name", + }, + "s3_config": { + "s3_endpoint_url", + "s3_bucket", + "s3_addressing_style", + }, + "sql_config": { + "dialect", + "driver", + "username", + "host", + "port", + "database", + "connect_args", + "engine_args", + }, + "excel_config": {"sheet_name", "has_header"}, + "csv_config": { + "has_header", + "encoding", + "dialect", + "delimiter", + "quotechar", + }, + "parquet_config": {"batch_size", "columns"}, + "mineru_config": {"include_types"}, + }, + "executor": { + "start_index": ..., + "end_index": ..., + "max_workers": ..., + "batch_size": ..., + "multi_turn_mode": ..., + "result_save": { + "bad", + "good", + "all_labels", + "raw", + "merge", + "limit", + "field_list", + "full_field_sample_count", + }, + "retrieval": { + "backend": ..., + "api_url": ..., + "limit": ..., + "retrieval_mode": ..., + "sub_queries": ..., + "search_type": ..., + "sort_by": ..., + "freshness_boost": ..., + "filters": ..., + "max_queries": ..., + "title_fuzzy_enabled": ..., + "title_fuzzy_threshold": ..., + "title_fuzzy_margin": ..., + "title_fuzzy_min_len": ..., + "title_fuzzy_max_candidates": ..., + "timeout": ..., + "rate_limit": ..., + "max_retries": ..., + "max_workers": ..., + "open_eval": { + "enabled", + "model", + "api_url", + "top_k", + "aggregate", + "max_workers", + "prompt_mode", + "expected_criteria", + }, + "input_queries": ..., + }, + }, + "evaluator": { + "__all__": { + "fields": ..., + "evals": { + "__all__": { + "name": ..., + "config": { + "threshold": ..., + "pattern": ..., + "key_list": ..., + "refer_path": ..., + "model": ..., + "api_url": ..., + "embedding_config": {"model", "api_url"}, + "custom_metric": { + "metric", + "description", + "criteria", + "input_fields", + }, + }, + }, + }, + }, + }, + } + model_dump = getattr(self, "model_dump", None) if callable(model_dump): - data = model_dump() - else: - data = self.dict() - - excluded = { - "api_key", - "api_token", - "key", - "password", - "s3_ak", - "s3_sk", - } - pending = [data] - while pending: - value = pending.pop() - if isinstance(value, dict): - for key in list(value): - if isinstance(key, str) and key.lower() in excluded: - del value[key] - else: - pending.append(value[key]) - elif isinstance(value, list): - pending.extend(value) - - return data + return model_dump( + include=safe_serialization_fields, + serialize_as_any=True, + ) + return self.dict(include=safe_serialization_fields) diff --git a/test/scripts/io/test_summary_model.py b/test/scripts/io/test_summary_model.py index d2806b59..69e0be9e 100644 --- a/test/scripts/io/test_summary_model.py +++ b/test/scripts/io/test_summary_model.py @@ -109,6 +109,7 @@ def test_input_args_to_dict_recursively_excludes_fields(self): "tool_config": { "API_KEY": "secret-tool-key", }, + "new_config_field": "not-approved-yet", }, }], }], @@ -124,7 +125,8 @@ def test_input_args_to_dict_recursively_excludes_fields(self): assert serialized["dataset"]["sql_config"]["username"] == "test-user" assert "api_token" not in serialized["executor"]["retrieval"] assert "key" not in config - assert "API_KEY" not in config["tool_config"] + assert "tool_config" not in config + assert "new_config_field" not in config assert config["model"] == "test-model" def test_add_metric_score_single(self):