diff --git a/dingo/config/input_args.py b/dingo/config/input_args.py index df7bebf8..8d312b56 100644 --- a/dingo/config/input_args.py +++ b/dingo/config/input_args.py @@ -197,24 +197,127 @@ class InputArgs(BaseModel): evaluator: List[EvalPipline] = [] def to_dict(self) -> Dict[str, Any]: - """Return a config snapshot, recursively excluding sensitive fields.""" + """Return a config snapshot containing only explicitly approved fields.""" + # Security allowlist: fields omitted here are excluded by default. + safe_serialization_fields: Dict[str, Any] = { + "task_id": ..., + "task_name": ..., + "input_path": ..., + "output_path": ..., + "log_level": ..., + "dataset": { + "source": ..., + "format": ..., + "hf_config": { + "huggingface_split", + "huggingface_config_name", + }, + "s3_config": { + "s3_endpoint_url", + "s3_bucket", + "s3_addressing_style", + }, + "sql_config": { + "dialect", + "driver", + "username", + "host", + "port", + "database", + "connect_args", + "engine_args", + }, + "excel_config": {"sheet_name", "has_header"}, + "csv_config": { + "has_header", + "encoding", + "dialect", + "delimiter", + "quotechar", + }, + "parquet_config": {"batch_size", "columns"}, + "mineru_config": {"include_types"}, + }, + "executor": { + "start_index": ..., + "end_index": ..., + "max_workers": ..., + "batch_size": ..., + "multi_turn_mode": ..., + "result_save": { + "bad", + "good", + "all_labels", + "raw", + "merge", + "limit", + "field_list", + "full_field_sample_count", + }, + "retrieval": { + "backend": ..., + "api_url": ..., + "limit": ..., + "retrieval_mode": ..., + "sub_queries": ..., + "search_type": ..., + "sort_by": ..., + "freshness_boost": ..., + "filters": ..., + "max_queries": ..., + "title_fuzzy_enabled": ..., + "title_fuzzy_threshold": ..., + "title_fuzzy_margin": ..., + "title_fuzzy_min_len": ..., + "title_fuzzy_max_candidates": ..., + "timeout": ..., + "rate_limit": ..., + "max_retries": ..., + "max_workers": ..., + "open_eval": { + "enabled", + "model", + "api_url", + "top_k", + "aggregate", + "max_workers", + "prompt_mode", + "expected_criteria", + }, + "input_queries": ..., + }, + }, + "evaluator": { + "__all__": { + "fields": ..., + "evals": { + "__all__": { + "name": ..., + "config": { + "threshold": ..., + "pattern": ..., + "key_list": ..., + "refer_path": ..., + "model": ..., + "api_url": ..., + "embedding_config": {"model", "api_url"}, + "custom_metric": { + "metric", + "description", + "criteria", + "input_fields", + }, + }, + }, + }, + }, + }, + } + model_dump = getattr(self, "model_dump", None) if callable(model_dump): - data = model_dump() - else: - data = self.dict() - - excluded = {"key"} - pending = [data] - while pending: - value = pending.pop() - if isinstance(value, dict): - for key in list(value): - if key in excluded: - del value[key] - else: - pending.append(value[key]) - elif isinstance(value, list): - pending.extend(value) - - return data + return model_dump( + include=safe_serialization_fields, + serialize_as_any=True, + ) + return self.dict(include=safe_serialization_fields) diff --git a/test/scripts/io/test_summary_model.py b/test/scripts/io/test_summary_model.py index bdbe6dc9..69e0be9e 100644 --- a/test/scripts/io/test_summary_model.py +++ b/test/scripts/io/test_summary_model.py @@ -83,6 +83,22 @@ def test_input_args_are_serialized(self): def test_input_args_to_dict_recursively_excludes_fields(self): input_args = InputArgs(**{ + "dataset": { + "s3_config": { + "s3_ak": "access-key", + "s3_sk": "secret-key", + "s3_bucket": "test-bucket", + }, + "sql_config": { + "username": "test-user", + "password": "secret-password", + }, + }, + "executor": { + "retrieval": { + "api_token": "secret-token", + }, + }, "evaluator": [{ "fields": {"content": "content"}, "evals": [{ @@ -90,6 +106,10 @@ def test_input_args_to_dict_recursively_excludes_fields(self): "config": { "model": "test-model", "key": "secret-key", + "tool_config": { + "API_KEY": "secret-tool-key", + }, + "new_config_field": "not-approved-yet", }, }], }], @@ -98,7 +118,15 @@ def test_input_args_to_dict_recursively_excludes_fields(self): serialized = input_args.to_dict() config = serialized["evaluator"][0]["evals"][0]["config"] + assert "s3_ak" not in serialized["dataset"]["s3_config"] + assert "s3_sk" not in serialized["dataset"]["s3_config"] + assert serialized["dataset"]["s3_config"]["s3_bucket"] == "test-bucket" + assert "password" not in serialized["dataset"]["sql_config"] + assert serialized["dataset"]["sql_config"]["username"] == "test-user" + assert "api_token" not in serialized["executor"]["retrieval"] assert "key" not in config + assert "tool_config" not in config + assert "new_config_field" not in config assert config["model"] == "test-model" def test_add_metric_score_single(self):