diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 702d6df..bcf648c 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "deeppapernote", "description": "Academic paper reading and Obsidian note generation skills for building a research WIKI.", - "version": "2.2.0", + "version": "2.3.0", "author": { "name": "917Dhj" }, diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 6081f92..01860fa 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "deeppapernote", - "version": "2.2.0", + "version": "2.3.0", "description": "Academic paper reading and Obsidian note generation skills for building a research WIKI.", "author": { "name": "917Dhj", diff --git a/CHANGELOG.md b/CHANGELOG.md index a86c3eb..56cb7a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,28 @@ Add an entry here when the project meaningfully changes for users, for example: ## Unreleased +## v2.3.0 + +### Added + +- Added end-to-end English output through User Configuration or `--language en`, including localized note schemas, paper-type planning contracts, figure callouts, grounding, final-note linting, and Formal Save validation. +- Added one device-local `config.json` for language and save preferences, with first-use/repair inspection, confirmed migration, atomic persistence, and run-scoped overrides. +- Added Vault-wide Save Target Admission backed by the original PDF SHA-256 and a hidden paper-directory sidecar, so source-identical language variants share one directory and same-language overwrites require explicit hash-bound confirmation. + +### Changed + +- Complete valid Run Overrides now satisfy Configuration Readiness without reading `~/.deeppapernote/config.json`; incomplete overrides continue through the existing User Configuration fallback and validation flow. +- Obsidian notes now use language-specific filenames, and Windows sidecars receive the native Hidden file attribute in addition to their dot-prefixed name. +- Successful runs now report the saved note path and domain, validation results, figure and table counts, warning count, and final note integrity check. + +### Fixed + +- Formal Save now preserves LF line endings on Windows instead of applying platform-specific newline conversion. + +### Contributors + +- Added configurable English note output from [PR #26](https://github.com/917Dhj/DeepPaperNote/pull/26) by [@basbun](https://github.com/basbun). + ## v2.2.0 ### Improved diff --git a/README.md b/README.md index 7e80726..3ceea15 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Read one paper deeply. Add one durable page to your academic wiki.

-**Do you know this situation? You sit down to study an important paper, but the exhausting part is not simply reading it. It is turning what you understood into a note you can still use later.** The time usually disappears into work like this: +**You sit down to study an important paper. The hard part is not reading it—it is turning what you understood into a note you can still use later.** The time usually disappears into work like this: - switching between the PDF, Zotero, web pages, and your note app - manually organizing metadata, the abstract, figures, and the method backbone @@ -45,9 +45,9 @@ DeepPaperNote is an agent skill for reading **one paper at a time**. The same co ## 📰 News +- **[v2.3.0]** 🌐 Added complete English note support across note structure, figure callouts, validation, and Formal Save. [Release notes](https://github.com/917Dhj/DeepPaperNote/releases/tag/v2.3.0) - **[2026-07-16]** 🧩 Added [`paper-glossary`](./skills/paper-glossary/README.md), an optional companion skill for building reusable Obsidian terminology notes. - **[2026-07-16]** 🔌 DeepPaperNote is now distributed as a plugin for multiple agents, with support for selecting multiple skills from the repository. [PR #12](https://github.com/917Dhj/DeepPaperNote/pull/12) -- **[v2.0.0]** 🚀 Released a deeper evidence-first paper-reading workflow with stronger note planning and figure handling. [Release notes](https://github.com/917Dhj/DeepPaperNote/releases/tag/v2.0.0) News lists only the three most recent user-facing milestones. See the [changelog](./CHANGELOG.md) and [GitHub Releases](https://github.com/917Dhj/DeepPaperNote/releases) for the full history. @@ -78,8 +78,6 @@ Generate a deep-reading note for this paper: ``` -DeepPaperNote currently generates Chinese notes by default, and its writing and validation rules are optimized for Chinese output. - ## 🎯 Why DeepPaperNote? ![DeepPaperNote usage example](./assets/usage-example.png) @@ -110,17 +108,18 @@ You do not need to install every skill. Choose the ones that match your workflow The canonical execution contract lives in [`skills/deeppapernote/SKILL.md`](./skills/deeppapernote/SKILL.md). -## 🗂️ Obsidian Setup +## 🗂️ First-Run Preferences -To make an Obsidian vault the default save target, set: +The first time you hand a paper to your Agent, DeepPaperNote helps you choose: -```bash -export DEEPPAPERNOTE_OBSIDIAN_VAULT="/absolute/path/to/your/vault" -``` +- the default note language: English or Simplified Chinese +- the default save target: the current workspace or your Obsidian Vault + +Once confirmed, these preferences are saved on your device and reused for future papers. You can still request a different language or save target for any individual run without changing your defaults. + +DeepPaperNote never silently overwrites an existing note or switches save destinations when a save is blocked. -- When a usable vault is configured or provided, DeepPaperNote saves the validated note and its paper-local `images/` directory there. -- When no vault is configured, DeepPaperNote asks first. It writes to the current workspace only after you explicitly choose not to use a vault. -- If a configured vault save fails, DeepPaperNote reports the blocked save instead of silently switching to another destination. +For advanced environment-variable and CLI configuration, see [User Configuration](./skills/deeppapernote/references/user-configuration.md). ## 🔧 Optional Enhancements diff --git a/README.zh-CN.md b/README.zh-CN.md index 2ff50ce..264e720 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -45,9 +45,9 @@ DeepPaperNote 是一个专注于**一次精读一篇论文**的 Agent Skill。 ## 📰 最新动态 +- **[v2.3.0]** 🌐 新增完整英文笔记支持，从章节结构、图表标注到校验与 Formal Save，全流程保持英文一致。[版本说明](https://github.com/917Dhj/DeepPaperNote/releases/tag/v2.3.0) - **[2026-07-16]** 🧩 新增可选 companion skill [`paper-glossary`](./skills/paper-glossary/README.md)，用于构建可复用的 Obsidian 术语笔记。 - **[2026-07-16]** 🔌 DeepPaperNote 现在以支持多个 Agent 的插件形式分发，并支持从仓库中选择多个 skill。[PR #12](https://github.com/917Dhj/DeepPaperNote/pull/12) -- **[v2.0.0]** 🚀 发布更深入的证据优先论文精读流程，并加强笔记规划与图表处理。[版本说明](https://github.com/917Dhj/DeepPaperNote/releases/tag/v2.0.0) 这里只保留最近三条用户可感知的重要动态。完整历史请查看 [CHANGELOG](./CHANGELOG.md) 与 [GitHub Releases](https://github.com/917Dhj/DeepPaperNote/releases)。 @@ -78,8 +78,6 @@ DeepPaperNote 需要 Python 3.10 或更高版本。核心 PDF 抽取路径依赖 把这篇论文整理成 Obsidian 笔记：<论文> ``` -DeepPaperNote 默认生成中文笔记，当前写作与校验规则也主要针对中文输出优化。 - ## 🎯 为什么选择 DeepPaperNote？ ![DeepPaperNote 使用示例](./assets/usage-example.png) @@ -110,17 +108,18 @@ DeepPaperNote 仍然是唯一主产品。仓库同时提供一个可选 companio 规范执行契约以 [`skills/deeppapernote/SKILL.md`](./skills/deeppapernote/SKILL.md) 为准。 -## 🗂️ Obsidian 配置 +## 🗂️ 首次使用设置 -如果希望默认保存到 Obsidian 库，请设置： +第一次把论文交给 Agent 时，DeepPaperNote 会引导你选择： -```bash -export DEEPPAPERNOTE_OBSIDIAN_VAULT="/你的/Obsidian/库/绝对路径" -``` +- 默认使用英文还是简体中文生成笔记 +- 默认保存到当前 workspace 还是 Obsidian Vault + +确认后，这些偏好会保存在当前设备上，并自动用于后续论文。你仍然可以为某一次任务临时指定不同的语言或保存位置，而不改变默认偏好。 + +DeepPaperNote 不会静默覆盖已有笔记，也不会在保存受阻时擅自切换保存位置。 -- 配置或提供了可用的 Vault 时，DeepPaperNote 会把校验完成的笔记及其论文专属 `images/` 目录保存到该 Vault。 -- 没有配置 Vault 时，DeepPaperNote 会先询问；只有你明确选择不使用 Vault 后，才会写入当前 workspace。 -- 如果已配置的 Vault 保存失败，DeepPaperNote 会报告保存受阻，而不会静默切换到其他目标。 +环境变量与 CLI 等高级配置请查看[用户配置说明](./skills/deeppapernote/references/user-configuration.md)。 ## 🔧 可选增强 diff --git a/assets/hero-academic.svg b/assets/hero-academic.svg index f0a740a..69ea8b9 100644 --- a/assets/hero-academic.svg +++ b/assets/hero-academic.svg @@ -42,7 +42,7 @@ AGENT SKILL DeepPaperNote - Turn comples papers into notes worth keeping. + Turn complex papers into notes worth keeping. Understand faster. Write less. Keep more. Structured notes for deep reading and real research workflows. diff --git a/pyproject.toml b/pyproject.toml index 05d615d..59700c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deeppapernote" -version = "2.2.0" +version = "2.3.0" description = "An agent-neutral skill for turning one research paper into a high-quality Obsidian note." readme = "README.md" requires-python = ">=3.10" diff --git a/skills/deeppapernote/SKILL.md b/skills/deeppapernote/SKILL.md index ad0f18f..3e1bb2a 100644 --- a/skills/deeppapernote/SKILL.md +++ b/skills/deeppapernote/SKILL.md @@ -8,7 +8,7 @@ description: Generate a high-quality deep-reading note for a single paper and wr Use this skill when the user wants one outcome: - read one paper carefully - generate a high-quality Markdown note -- save the note into an Obsidian-style vault when configured, or into the current workspace when no vault is configured +- save the note to the workspace or Obsidian target selected by resolved configuration Chinese trigger examples: - `给这篇论文生成深度笔记` @@ -16,7 +16,31 @@ Chinese trigger examples: - `把这篇文章整理成 obsidian 笔记` - `读这篇论文并生成 md 笔记` +English trigger examples: +- `Generate a deep-reading note for this paper` +- `Turn this paper into an Obsidian research note` + +## User Configuration + +Before a normal paper run, read `references/user-configuration.md` for configuration admission, migration, repair, Run Overrides, and Preference Changes. + +Resolve Run Overrides from the explicit request, CLI, and current process environment first. When they form a complete valid configuration for the selected Save Mode, Configuration Readiness is complete without reading User Configuration. Only inspect User Configuration when those Run Overrides need fallback values. + +## Language Integrity Contract + +After Configuration Readiness, resolve one `output_language` (`zh-CN` or `en`) for the run. `source_manifest.language_hint` describes source text only and never selects the note profile. + +Bind that exact value through Save Target Admission → Figure Plan → Figure/Table Decisions → Synthesis Bundle → Note Plan → Grounding Lint → Final Note Lint → Final Quality Review → Final Readability Review → Formal Save: + +- Every JSON artifact in the chain carries a top-level `output_language`; the Synthesis Bundle also carries the same value at `writing_contract.language`. +- Before producing its output, every adjacent consumer requires each input language and compares it with the resolved value. Missing, unsupported, or mismatched values stop the run; no stage infers or defaults an artifact language. +- Final Quality Review and Final Readability Review each receive the resolved value and check the note against only that profile. +- Final Note Lint records `note_sha256`. Any review edit invalidates the prior lint, so rerun Final Note Lint under the same language. Formal Save requires the lint language and `note_sha256` to match the final note, and validates Figure/Table Decisions language before any save side effect. + +This contract is complete only when every named stage is bound to the resolved value and Formal Save validates the final bytes. Read `references/output-language.md` for profile content while drafting or debugging either language. + This skill is intentionally narrow: +- it is one canonical Skill and one pipeline with two output profiles - it handles one paper at a time - it does not update daily reading lists - it does not treat a shallow abstract rewrite as a successful output @@ -44,21 +68,28 @@ The note must adapt to the paper type. Use the same base structure, but shift em ## Workflow Follow this order: -1. resolve the paper identity -2. collect metadata -3. acquire the best available PDF -4. extract canonical raw source text: `*_raw_sections.jsonl`, `*_source_manifest.json`, and optional derived `*_full_text.md` -5. extract structural indexes and PDF assets -6. plan figure placement -7. build the full figure/table decision table -8. build the manifest synthesis bundle -9. have the model read the bundle plus raw sections and create a short JSON `note_plan` that satisfies the generated bundle contract -10. draft from the plan only after the grounding gate passes -11. have the model write the note -12. lint the final note against the same `note_plan` — this stage completes only when the lint artifact exists and every reported `passes_*` gate is `true`; otherwise revise and rerun lint. If the lint output contains `passes_style_gate: false`, apply the Style Gate Enforcement rule before advancing to step 13, 14, or 15 -13. perform `final_quality_review` after lint passes -14. perform `final_readability_review` after the quality review passes -15. write into Obsidian +1. complete Configuration Readiness: resolve Run Overrides first, and inspect User Configuration only when they are incomplete; advance only after the resolved run configuration is complete and valid +2. resolve the paper identity +3. collect metadata +4. acquire the best available PDF +5. extract canonical raw source text: `*_raw_sections.jsonl`, `*_source_manifest.json`, and optional derived `*_full_text.md` +6. perform Save Target Admission before drafting or domain routing: + - for Obsidian mode, run `scripts/write_obsidian_note.py --preflight` with the resolved title, exact `output_language`, Vault, and `*_source_manifest.json`; this program result is authoritative, so do not replace it with prompt-only duplicate checking + - when admission returns `reuse_source_directory` or `reuse_empty_same_name_directory`, use that directory and skip domain selection + - when it returns `same_language_note_exists`, stop before drafting and ask whether to overwrite the reported note. If the user approves, rerun preflight with `--overwrite-existing-note --expected-existing-note-sha256 ` and carry that exact confirmation into Formal Save; if the user declines, stop without writing + - for any other blocked conflict, report the returned ambiguity and stop without creating a second directory + - workspace mode does not scan an Obsidian Vault and continues through its normal domain routing +7. extract structural indexes and PDF assets +8. plan figure placement +9. build the full figure/table decision table +10. build the manifest synthesis bundle +11. have the model read the bundle plus raw sections and create a short JSON `note_plan` that satisfies the generated bundle contract, including its exact `output_language` +12. draft from the plan only after the grounding gate passes +13. have the model write the note +14. lint the final note against the same `note_plan` — this stage completes only when the lint artifact exists and every reported `passes_*` gate is `true`; otherwise revise and rerun lint. If the lint output contains `passes_style_gate: false`, apply the Style Gate Enforcement rule before advancing to step 15, 16, or 17 +15. perform `final_quality_review` after lint passes +16. perform `final_readability_review` after the quality review passes +17. perform Formal Save to the admitted target with `scripts/write_obsidian_note.py`, the same Source Manifest, and any user-approved overwrite hash; the script repeats admission before the first save side effect This is the required workflow for a normal single-paper note request, not a loose suggestion. Unless this skill explicitly marks a stage as optional, required stages must not be silently skipped, reordered into a shortcut, or treated as complete just because a partial artifact already exists. @@ -83,6 +114,17 @@ Completion-language rule: - if the workflow stopped early, name the current stage and the still-missing required stages instead of using completion language - lint is a floor, not the writing objective +Final user report: +- Keep the completion wording defined above. After a successful Formal Save, report in the user's conversation language. +- Lead with the final note link or path, save mode, and actual saved domain. Read the domain from the final note path under the configured papers root in Obsidian mode or output root in workspace mode; when Save Target Admission reused an existing directory, report that directory's existing domain. +- Then report, in order: + 1. paper title and strongest verified identifier + 2. Grounding Lint, Final Note Lint, Final Quality Review, and Final Readability Review results, plus the warning count + 3. materialized and retained-placeholder figure/table counts + 4. whether the saved note SHA-256 matches the Final Note Lint `note_sha256` +- Add overwrite actions, preference changes, or user-relevant warnings only when they occurred. +- Keep the report to these fields and derive every claim from current-run artifacts. + ## Core Execution Contract `SKILL.md` plus the generated `synthesis_bundle.json` must be enough to complete a normal note-generation run. @@ -92,12 +134,11 @@ Non-negotiable rules: - evidence-first: draft from the synthesis bundle, `source_manifest`, raw sections, coverage metadata, explicit `note_plan`, and inspected paper evidence; never finish from title/abstract/headings alone - raw-source authority: for ordinary PDFs, `*_raw_sections.jsonl` and `*_source_manifest.json` are the canonical reading material; old top-N evidence buckets, truncated `section_texts`, and `candidate_chunks` are not model-facing writing inputs - fail-closed: if a usable PDF or sufficient evidence cannot be obtained after supported acquisition paths, stop and ask for better source material rather than producing a finished degraded note -- model-first: scripts structure evidence, but the model must decide emphasis, contribution, mechanism, limitations, and final Chinese prose -- required structure: include the canonical required sections, with `原文摘要翻译` before `一句话总结` and a dedicated `创新点` section immediately after `原文摘要翻译` -- abstract translation: when abstract metadata exists, `原文摘要翻译` is a faithful Chinese translation of the original abstract, not a bilingual block and not the model's own summary -- mechanism depth: method, framework, and system papers should include `### 机制流程` under `方法主线`, normally as a 3 to 4 step numbered flow with input, operation, and output destination +- model-first: scripts structure evidence, but the model must decide emphasis, contribution, mechanism, limitations, and final prose in the configured language +- required structure: include the localized canonical sections in the order declared by `writing_contract.must_include_sections` +- abstract fidelity: preserve the original abstract's meaning without adding later evidence or model judgments; translate it in `zh-CN` mode and render it faithfully in English in `en` mode +- mechanism depth: method, framework, and system papers should include the localized mechanism-flow subsection under the localized method section, normally as a 3 to 4 step numbered flow with input, operation, and output destination - placeholder-first figures: plan major figure/table placeholders first; replace one only when identity match and visual usability are both strong; otherwise keep the placeholder -- final quality gates: lint is a floor; after lint passes, first run `final_quality_review` for analytical depth, then run `final_readability_review` for language polish, and rerun lint if either review edits the note Reference usage policy: - do not load every reference file by default @@ -133,27 +174,26 @@ Formal Save states: | Save Target state | Required action | |---|---| -| Vault configured or provided and usable | Perform the Formal Save to that vault. | -| Vault configured or provided, but the Formal Save fails | Keep the current Save Target and attempt an in-scope recovery. If it still cannot complete, report `blocked`; do not switch to workspace. | -| No vault configured or provided | Ask whether the user wants to provide one. Use workspace only after the user explicitly chooses not to use a vault. | +| `save_mode=obsidian` and the configured Vault is usable | Perform the Formal Save to that Vault. | +| `save_mode=obsidian` and Formal Save fails | Keep the current Save Target and attempt an in-scope recovery. If it still cannot complete, report `blocked`; do not switch to workspace. | +| `save_mode=workspace` | Perform the Formal Save inside the current workspace output root. | - A normal note-generation request should complete in one pass: note text, figure placeholder decisions, image materialization when confident, and final save. - Do not stop after a text-only draft just to ask whether the user wants figures inserted. Finish the figure replacement decision inside the same task unless the user explicitly asked for text only. - The note must use real heading levels: `#`, `##`, and `###`. - Every final note must start with an Obsidian YAML properties block above the `#` title heading. Include at least a `tags` field with a `papers/` value and useful `aliases`; include `date`, `doi`, or `arxiv_id` when known, and omit unavailable fields rather than inventing placeholders. -- `## 核心信息` must be a fixed metadata block only. Use only these fields, in this order, as `- 字段名: 值` bullets: `标题`, `标题翻译`, `作者`, `机构`, `发表时间`, `发表渠道`, `DOI`, `arXiv`, `论文链接`, `代码 / 项目`, `数据 / 资源`, `论文类型`. Omit unavailable fields; put any guide sentence, takeaway, or analysis in `一句话总结` or a later section instead. -- The note should include `原文摘要翻译` near the beginning when abstract metadata is available, before `一句话总结`. -- When abstract metadata is available, `原文摘要翻译` should directly translate the original paper abstract into Chinese rather than restating it as your own summary. -- The `原文摘要翻译` section itself should be Chinese-only; do not place English abstract sentences or English paragraph excerpts in that section. -- Do not mix later judgments, innovation summaries, or hindsight explanations into `原文摘要翻译`; keep it as the original abstract translated into Chinese. -- The note should include a dedicated `创新点` section immediately after `原文摘要翻译` and before `一句话总结`. -- The `创新点` section should not be empty praise. It should enumerate the paper's actual innovations and briefly explain why each one matters. +- The localized Core Information section must be a fixed metadata block only. Use only the fields and order declared by `writing_contract.core_info_fields`; omit unavailable fields and move commentary to a later analysis section. +- Include the localized Abstract section near the beginning when abstract metadata is available, before the one-sentence summary. +- The Abstract section should faithfully render the paper's original abstract in the configured language rather than replacing it with a model-written summary. +- Do not mix later judgments, contribution summaries, or hindsight explanations into the Abstract section. +- Include a dedicated localized Contributions section immediately after Abstract and before the one-sentence summary. +- Contributions should enumerate the paper's actual innovations and explain why each matters rather than offering empty praise. - High-quality notes should usually contain multiple meaningful `###` subheadings in the technical sections when the paper is non-trivial. - Generate the complete figure/table decision table and satisfy the generated `writing_contract.figure_table_contract` before drafting or saving. - After the synthesis bundle is built, complete the model-led Visual Review Gate and Figure/Table Decision Freeze before creating `note_plan`; no `review_pending` item may cross that boundary. - Pass the grounding and final-note figure gates before advancing; revise any failed decision coverage, insertion, structure, or status check. - An `insert` decision is complete only after Formal Save materializes the selected image into the paper-local `images/` directory and the write succeeds. -- The note must pass a style gate: no mixed Chinese-English prose lines except stable proper nouns or citation metadata. +- The note must pass the style gate for its configured language: `zh-CN` rejects mixed Chinese-English prose artifacts, while `en` rejects Chinese prose outside citation metadata. - The style gate also rejects mechanical term-replacement artifacts such as `KV缓存 of`, `批量ing`, `In相关 Researcher`, or `Single 序列 generation`; rewrite the sentence naturally instead of preserving a partially translated phrase. - Style gate enforcement: when `lint_note.py` output contains `passes_style_gate: false`, fix the reported issues and re-run lint. Keep fixing and re-running until lint passes — multiple rounds are normal and expected. Do not decide that any failure is an acceptable exception — proper nouns, math formulas, and citation metadata are not automatic exemptions. Only escalate to the user if the same failures appear unchanged across multiple rounds with no reduction, indicating the model is unable to make further progress independently. - If PDF or evidence quality is insufficient for a real deep note, fail closed: stop, report the blocked stage, and ask for the better PDF, OCR/source material, or other input needed to continue. @@ -172,7 +212,7 @@ Model-first rule: - central quantitative comparisons with three or more systems, settings, tasks, datasets, metrics, or ablation rows should normally be written as compact Markdown tables, followed by interpretation; do not leave the main result table as a loose bullet list when a table would be clearer - short papers still need a complete deep note: use the saved space to explain protocol details, ablations, limitations, and deployment or replication implications rather than compressing the note into a terse summary - after `final_quality_review` passes, reread the full note once more for readability; do not stop at formal compliance only -- in `final_readability_review`, ordinary English phrase leftovers should usually be rewritten into natural Chinese, while stable proper nouns may remain in English +- in `final_readability_review`, rewrite language leftovers into natural prose in the configured language while preserving stable proper nouns - do not use `final_readability_review` to invent new facts, empty filler text, or shallower but safer wording just to satisfy lint The topic references above can improve difficult runs, but the normal execution path should not depend on reading all of them. @@ -181,6 +221,7 @@ The topic references above can improve difficult runs, but the normal execution Use these bundled scripts rather than rebuilding the workflow from scratch: - `scripts/check_environment.py` +- `scripts/user_configuration.py` - `scripts/create_input_record.py` - `scripts/locate_zotero_attachment.py` - `scripts/resolve_paper.py` diff --git a/skills/deeppapernote/references/deep-analysis.md b/skills/deeppapernote/references/deep-analysis.md index 0909834..fc32aca 100644 --- a/skills/deeppapernote/references/deep-analysis.md +++ b/skills/deeppapernote/references/deep-analysis.md @@ -1,10 +1,12 @@ # Deep Analysis +Language note: the analytical standard is language-independent. Draft in the configured output language; for `en`, use the exact headings and labels in `output-language.md`. Chinese-specific examples below apply only to `zh-CN`. + Use this guide when the user wants a note that feels like a real research note rather than a cleaned-up summary. ## Goal -Produce a Chinese paper note that helps future rereading answer: +Produce a paper note in the configured language that helps future rereading answer: - this paper is really solving what problem - the core route or method chain is what - which evidence actually supports the claim diff --git a/skills/deeppapernote/references/evidence-first.md b/skills/deeppapernote/references/evidence-first.md index c805874..5b4d728 100644 --- a/skills/deeppapernote/references/evidence-first.md +++ b/skills/deeppapernote/references/evidence-first.md @@ -58,6 +58,7 @@ Recommended shape: ```json { + "output_language": "zh-CN", "paper_type": "AI_method", "paper_type_rationale": "The paper proposes a model mechanism and evaluates it against baselines; the script suggestion was treated only as a hint.", "dominant_domain": "machine learning", diff --git a/skills/deeppapernote/references/figure-placement.md b/skills/deeppapernote/references/figure-placement.md index ce3cf01..8205b39 100644 --- a/skills/deeppapernote/references/figure-placement.md +++ b/skills/deeppapernote/references/figure-placement.md @@ -1,5 +1,7 @@ # Figure Placement +Language note: target sections and figure-callout labels must follow the generated writing contract. For English output, use the labels in `output-language.md`. + In MVP, the skill must plan figure placement even when it cannot extract image files. ## Goal diff --git a/skills/deeppapernote/references/final-writing.md b/skills/deeppapernote/references/final-writing.md index 8e0f2e4..d3aead2 100644 --- a/skills/deeppapernote/references/final-writing.md +++ b/skills/deeppapernote/references/final-writing.md @@ -1,5 +1,7 @@ # Final Writing +Language note: draft in `synthesis_bundle.writing_contract.language`. For `en`, use the exact schema in `output-language.md`; Chinese-only wording and typography rules below apply only to `zh-CN`. + The final note should not read like raw extracted evidence. Use the structured artifacts as inputs: diff --git a/skills/deeppapernote/references/note-quality.md b/skills/deeppapernote/references/note-quality.md index d420712..31bc596 100644 --- a/skills/deeppapernote/references/note-quality.md +++ b/skills/deeppapernote/references/note-quality.md @@ -1,5 +1,7 @@ # Note Quality +Language note: evaluate headings and language cleanliness against `synthesis_bundle.writing_contract`. For `en`, use `output-language.md`; Chinese-specific examples below apply only to `zh-CN`. + The note is high quality only if it satisfies most of the checks below. ## Minimum Bar diff --git a/skills/deeppapernote/references/obsidian-format.md b/skills/deeppapernote/references/obsidian-format.md index a5fbc03..7d0063a 100644 --- a/skills/deeppapernote/references/obsidian-format.md +++ b/skills/deeppapernote/references/obsidian-format.md @@ -1,5 +1,7 @@ # Obsidian Format +Language note: section names, metadata labels, figure callouts, and the mechanism-flow heading must follow `synthesis_bundle.writing_contract`. For `en`, use the exact schema in `output-language.md`; Chinese-specific examples below apply only to `zh-CN`. + ## Heading Rules - Use `#` for the note title only. @@ -14,8 +16,10 @@ Default file name: - sanitized English title with underscores - default note layout is folder-per-paper: - - `<领域>//.md` + - `<领域>//.zh-CN.md` + - `<领域>//.en.md` - `<领域>//images/...` + - `<领域>//.deeppapernote.json` - when deciding `<领域>`, prefer matching an existing first-level domain folder under the user's papers directory - domain routing uses the editable taxonomy in `references/domain_rules.yaml`: application domains are checked before fallback method domains - reuse existing first-level folders conservatively; method-only evidence should not force reuse of an unrelated application folder @@ -24,6 +28,10 @@ Default file name: - always create the paper-local `images/` directory during final save, even if no real image is inserted - the paper-local `images/` directory is part of the required note layout, not an optional optimization +The hidden `.deeppapernote.json` sidecar is the program-owned directory identity record. It stores the exact original PDF SHA-256, the frozen note stem, and the language variants already saved in that directory. Keep the dot-prefixed name on every platform; on Windows the save script also applies the native Hidden file attribute and verifies it after each sidecar replacement. + +Before an Obsidian draft begins, use the save script's preflight result rather than inspecting names manually. It searches the entire Vault for the source SHA-256 and same-name directories. An exact source match reuses its frozen directory without domain routing; another language is added beside the existing note, while an existing note in the requested language requires explicit hash-bound overwrite confirmation. A nonempty same-name directory without a valid sidecar, a same-name directory for different source bytes, or multiple identity matches fails closed. + If the user already has a vault convention, preserve it. ## Markdown Style diff --git a/skills/deeppapernote/references/output-language.md b/skills/deeppapernote/references/output-language.md new file mode 100644 index 0000000..597b025 --- /dev/null +++ b/skills/deeppapernote/references/output-language.md @@ -0,0 +1,71 @@ +# Output Language + +DeepPaperNote supports two output schemas: + +| Setting | Language | +|---|---| +| `zh-CN` | Simplified Chinese | +| `en` | English | + +Resolve the profile through `user-configuration.md`. For a single command, use `--language en` with `run_pipeline.py`, `build_synthesis_bundle.py`, `lint_note.py`, or `write_obsidian_note.py` where applicable. A Run Override never changes User Configuration. + +## Simplified Chinese note schema + +Use these top-level sections in this order: + +1. `核心信息` +2. `原文摘要翻译` +3. `创新点` +4. `一句话总结` +5. `研究问题` +6. `数据与任务定义` +7. `方法主线` +8. `关键结果` +9. `深度分析` +10. `局限` +11. `我的笔记` +12. `引用` + +Use `### 机制流程` for the mechanism-flow subsection. Chinese figure placeholders use the labels `建议位置：`, `放置原因：`, and `当前状态：`; a materialized image caption begins with `论文原图编号：`. + +`原文摘要翻译` is a faithful Chinese translation of the source abstract. Preserve its meaning and scope; contribution claims, result interpretation, and hindsight judgment belong later unless the source abstract itself contains them. + +## English note schema + +Use these top-level sections in this order: + +1. `Core Information` +2. `Abstract` +3. `Contributions` +4. `One-Sentence Summary` +5. `Research Question` +6. `Data and Task Definition` +7. `Method` +8. `Key Results` +9. `Deep Analysis` +10. `Limitations` +11. `Research Notes` +12. `References` + +The allowed Core Information fields, in order, are: + +`Title`, `Translated title`, `Authors`, `Institutions`, `Publication date`, `Venue`, `DOI`, `arXiv`, `Paper link`, `Code / Project`, `Data / Resources`, `Paper type`. + +Use `### Mechanism Flow` for the mechanism-flow subsection. Each figure placeholder uses: + +```md +> [!figure] Figure 2 Human-readable label +> Suggested location: Method +> Why it matters: This figure clarifies the execution path. +> Current status: Placeholder retained; the recovered crop is incomplete. +``` + +For a materialized image, use the normal image embed followed immediately by one italic caption beginning with `Original paper item:`. + +`Abstract` is a faithful rendering of the source abstract in English. Preserve its meaning and scope; contribution claims, result interpretation, and hindsight judgment belong later unless the source abstract itself contains them. + +The English style gate checks headings, Core Information labels, figure callouts, inserted-image captions, and prose. Mark original non-English metadata with inline code inside `Core Information`, or with inline code or a Markdown link inside `References`; closed fenced code blocks and URLs keep their normal source text. In prose, a CJK identifier or stable proper noun must be an HTTP Markdown link or Obsidian wikilink. A source formula may retain only these CJK `\operatorname{...}` labels: `输入`, `输出`, `损失`, `状态`, `动作`, `奖励`, `标签`, `样本`, `预测`, and `目标`. The surrounding text remains subject to the English gate; inline code and free-form math text do not exempt Chinese prose. + +## Contract ownership + +`SKILL.md` owns the cross-stage Language Integrity Contract. This reference owns only the profile-specific schema and labels above; apply them under the resolved language carried by that contract. diff --git a/skills/deeppapernote/references/paper-types.md b/skills/deeppapernote/references/paper-types.md index 758cff0..664719a 100644 --- a/skills/deeppapernote/references/paper-types.md +++ b/skills/deeppapernote/references/paper-types.md @@ -1,5 +1,7 @@ # Paper Types +Language note: use the localized `contracts_by_paper_type` and section names from the generated synthesis bundle. The examples below use the backward-compatible Chinese schema; `output-language.md` defines the English schema. + Every note keeps the same 12 top-level sections from `NOTE_REQUIRED_SECTIONS`. Paper type only changes the typed semantics of shared sections and the recommended `###` subsections used in `note_plan.section_plan`. diff --git a/skills/deeppapernote/references/user-configuration.md b/skills/deeppapernote/references/user-configuration.md new file mode 100644 index 0000000..77541c8 --- /dev/null +++ b/skills/deeppapernote/references/user-configuration.md @@ -0,0 +1,55 @@ +# User Configuration + +DeepPaperNote can resolve a run entirely from Run Overrides. It also supports one optional device-local User Configuration at `~/.deeppapernote/config.json` for fallback values and explicit future preferences: + +For isolated validation only, `DEEPPAPERNOTE_CONFIG_PATH` may relocate this one file for the current process. It is not a preference, does not create a second configuration layer, and must not be persisted into the configuration itself. + +- `output_language`: `zh-CN` or `en` +- `save_mode`: `workspace` or `obsidian` +- `obsidian_vault`: existing absolute directory, required only in Obsidian mode +- `papers_dir`: safe relative path inside the Vault, required only in Obsidian mode + +There is no implicit language or save-mode default. Workspace mode ignores stored Obsidian fields for the current run but preserves them for a later Obsidian run. Destination writability is a Formal Save concern; configuration inspection never creates a probe file in the workspace or Vault. + +## Configuration admission + +Complete Configuration Readiness before paper identity resolution: + +1. Resolve the explicit request, CLI arguments, and current process environment in precedence order. +2. When those Run Overrides contain every active field and pass validation, complete Configuration Readiness without reading User Configuration. +3. Otherwise run `scripts/user_configuration.py` without setters. The inspector returns exactly one structured User Configuration state: `ready`, `needs_input`, `invalid`, or `blocked`. +4. For `needs_input` on first use, ask one Configuration Prompt Batch for the unresolved active fields. Require `obsidian_vault` and `papers_dir` when the resolved Save Mode is Obsidian. For later repair, ask only for `prompt_fields`. +5. For migration candidates, show the candidates and obtain confirmation before persisting future preferences. Candidates may still act as current-process Run Overrides when they are actually present in the process environment. +6. Persist confirmed preferences with the relevant `--set-output-language`, `--set-save-mode`, `--set-vault`, and `--set-papers-dir` options. Use `--replace-invalid` only after the user explicitly confirms replacement of malformed or non-object JSON. +7. Run the inspector again after a Preference Change. Persistence completes only when it returns `ready` after atomic write and readback validation; then resolve the run again. + +Treat `invalid` as repairable input. Treat `blocked` as an I/O boundary: report its issue, preserve the current file, and stop before paper work. Never claim a preference was saved unless readback returned `ready`. + +## Resolution and persistence + +Resolve each preference using this exact precedence; an explicit request is an explicit current-run parameter, including a natural-language request: + +`explicit request > CLI > current process environment > User Configuration` + +An explicit request about the current paper is a Run Override. Translate it to the matching runtime override and leave `config.json` byte-for-byte unchanged. Persist only explicit future-default wording as a Preference Change. + +Current process environment values are first-class Run Overrides and may satisfy the entire run without a configuration-file read. Shell startup files are not read when the inherited process environment is complete. If fallback is required while `config.json` is absent, supported shell values may be shown as migration candidates; they become persistent preferences only after confirmation. + +Preference Changes preserve unknown JSON fields and report a warning. Malformed or non-object JSON receives a unique invalid backup before a confirmed replacement. Writes use a same-directory temporary file, atomic replacement, and exact reread comparison. + +## Advanced Run Overrides + +Normal Agent use does not require these options. For direct CLI or environment-based runs, use the following mappings: + +| Preference | Current-process environment | CLI option | +|---|---|---| +| `output_language` | `DEEPPAPERNOTE_OUTPUT_LANGUAGE` | `--language` | +| `save_mode` | `DEEPPAPERNOTE_SAVE_MODE` | `--save-mode` | +| `obsidian_vault` | `DEEPPAPERNOTE_OBSIDIAN_VAULT` | `--vault` | +| `papers_dir` | `DEEPPAPERNOTE_PAPERS_DIR` | `--papers-dir` | + +These values are Run Overrides for the current process or command. They do not become saved preferences unless the user separately confirms a Preference Change. + +## Completion criteria + +Configuration is ready only when every active field is present and valid, the resolved values contain no missing or invalid field, and the workflow has not begun identity resolution. A User Configuration file is not required when Run Overrides already meet that condition. Obsidian mode requires an existing absolute Vault and a traversal-safe relative paper directory. Workspace mode requires neither Obsidian field and cannot be redirected by stale values. diff --git a/skills/deeppapernote/scripts/build_synthesis_bundle.py b/skills/deeppapernote/scripts/build_synthesis_bundle.py index 539171b..e1af552 100644 --- a/skills/deeppapernote/scripts/build_synthesis_bundle.py +++ b/skills/deeppapernote/scripts/build_synthesis_bundle.py @@ -16,11 +16,11 @@ ) from contracts import ( NOTE_PLAN_REQUIRED_FIELDS, - NOTE_REQUIRED_SECTIONS, - PAPER_TYPE_CONTRACTS, PAPER_TYPE_VALUES, - WRITING_CONTRACT_RULES, + paper_type_contracts, + writing_contract_rules, ) +from localization import normalize_output_language, require_artifact_output_language def parser() -> argparse.ArgumentParser: @@ -35,6 +35,7 @@ def parser() -> argparse.ArgumentParser: required=True, help="Figure/table decision JSON path or string.", ) + p.add_argument("--language", default="", help="Run Override for output language: en or zh-CN.") p.add_argument("--output", default="", help="Output JSON path.") return p @@ -43,12 +44,14 @@ def load_record(value: str) -> dict: return maybe_load_json_record(value) or {} -def sanitize_reference_candidates(evidence_pack: dict, *, limit: int = 20) -> list[dict]: +def sanitize_reference_candidates( + evidence_pack: dict, config: dict, *, limit: int = 20 +) -> list[dict]: candidates = evidence_pack.get("reference_candidates", []) or [] if not isinstance(candidates, list): return [] try: - matched_candidates = resolve_reference_links(candidates[:limit], runtime_config()) + matched_candidates = resolve_reference_links(candidates[:limit], config) except Exception: matched_candidates = [ { @@ -315,17 +318,18 @@ def figure_table_manifest( } -def compact_writing_contract() -> dict: - depth_requirements = dict(WRITING_CONTRACT_RULES["note_plan_depth_requirements"]) +def compact_writing_contract(language: str | None = None) -> dict: + rules = writing_contract_rules(language) + depth_requirements = dict(rules["note_plan_depth_requirements"]) depth_requirements["required_section_focus_fields"] = list( depth_requirements["required_section_focus_fields"] ) depth_requirements["generic_focus_phrases"] = list( depth_requirements["generic_focus_phrases"] ) - usable_insert_candidate = dict(WRITING_CONTRACT_RULES["usable_insert_candidate"]) + usable_insert_candidate = dict(rules["usable_insert_candidate"]) usable_insert_candidate["kinds"] = list(usable_insert_candidate["kinds"]) - visual_review_contract = deepcopy(WRITING_CONTRACT_RULES["visual_review_contract"]) + visual_review_contract = deepcopy(rules["visual_review_contract"]) for field in ( "review_fields", "review_status_values", @@ -334,7 +338,7 @@ def compact_writing_contract() -> dict: "terminal_failure_reasons", ): visual_review_contract[field] = list(visual_review_contract[field]) - analysis_coverage = deepcopy(WRITING_CONTRACT_RULES["analysis_coverage_contract"]) + analysis_coverage = deepcopy(rules["analysis_coverage_contract"]) analysis_coverage["central_claim_fields"] = list( analysis_coverage["central_claim_fields"] ) @@ -344,19 +348,25 @@ def compact_writing_contract() -> dict: analysis_coverage["final_quality_review_checks"] = list( analysis_coverage["final_quality_review_checks"] ) - return { - "language": "zh-CN", + contract = { + "language": rules["language"], "contract_role": "manifest_quality_contract", "canonical_source": ( "SKILL.md defines the workflow; scripts/contracts.py defines " "machine-checkable contract data." ), - "must_include_sections": list(NOTE_REQUIRED_SECTIONS), + "must_include_sections": list(rules["required_sections"]), + "core_info_fields": list(rules["core_info_fields"]), + "figure_labels": dict(rules["figure_labels"]), + "mechanism_flow_heading": rules["mechanism_flow_heading"], "note_plan_contract": { - "required_fields": list(NOTE_PLAN_REQUIRED_FIELDS), - "field_types": dict(WRITING_CONTRACT_RULES["note_plan_field_types"]), + "required_fields": ["output_language", *NOTE_PLAN_REQUIRED_FIELDS], + "field_types": { + "output_language": "string", + **dict(rules["note_plan_field_types"]), + }, "required_field_checks": deepcopy( - WRITING_CONTRACT_RULES["note_plan_required_field_checks"] + rules["note_plan_required_field_checks"] ), "artifact_preference": "short_json_planning_file", "grounding_field": "section_plan[*].evidence_sources", @@ -367,21 +377,21 @@ def compact_writing_contract() -> dict: "suggested_paper_type_role": "none", "allowed_paper_types": list(PAPER_TYPE_VALUES), }, - "contracts_by_paper_type": PAPER_TYPE_CONTRACTS, + "contracts_by_paper_type": paper_type_contracts(rules["language"]), "grounding_contract": { "source_of_truth": "source_manifest", "source_index_source_of_truth": "source_manifest", "truncation_source_of_truth": "source_manifest.coverage_or_pdf", "partial_reading_acceptance_owner": "note_plan_or_grounding", "accepted_reference_forms": list( - WRITING_CONTRACT_RULES["allowed_grounding_reference_forms"] + rules["allowed_grounding_reference_forms"] ), - "required_sections": list(WRITING_CONTRACT_RULES["grounding_required_sections"]), + "required_sections": list(rules["grounding_required_sections"]), "note_plan_depth_requirements": depth_requirements, "excluded_model_input_fields": list( - WRITING_CONTRACT_RULES["excluded_model_input_fields"] + rules["excluded_model_input_fields"] ), - "reject_old_references": list(WRITING_CONTRACT_RULES["old_bundle_reference_prefixes"]), + "reject_old_references": list(rules["old_bundle_reference_prefixes"]), "lint_command": ( "scripts/lint_grounding.py --note-plan ... " "--source-manifest ... --bundle-json ... --figure-decisions ..." @@ -391,22 +401,26 @@ def compact_writing_contract() -> dict: "placeholder_first": True, "visual_quality_gate": "fail_closed", "decision_table_required": True, - "decision_values": list(WRITING_CONTRACT_RULES["figure_decision_values"]), + "decision_values": list(rules["figure_decision_values"]), "usable_insert_candidate": usable_insert_candidate, "allowed_usable_placeholder_reasons": list( - WRITING_CONTRACT_RULES["allowed_usable_placeholder_reasons"] + rules["allowed_usable_placeholder_reasons"] ), "manual_visual_review_required_statuses": list( - WRITING_CONTRACT_RULES["manual_visual_review_required_statuses"] + rules["manual_visual_review_required_statuses"] ), "automatic_fail_closed_visual_statuses": list( - WRITING_CONTRACT_RULES["automatic_fail_closed_visual_statuses"] + rules["automatic_fail_closed_visual_statuses"] ), "manual_review_claim_requires_image_inspection": True, "visual_review": visual_review_contract, }, "analysis_coverage_contract": analysis_coverage, } + abstract_contract = rules.get("abstract_contract") + if abstract_contract: + contract["abstract_contract"] = deepcopy(abstract_contract) + return contract def bundle( @@ -416,6 +430,7 @@ def bundle( assets_wrapper: dict, source_manifest: dict | None = None, figure_decisions_wrapper: dict | None = None, + output_language: str | None = None, ) -> dict: evidence_pack = ( evidence_wrapper.get("evidence_pack", {}) @@ -429,12 +444,25 @@ def bundle( ) source_manifest = source_manifest or {} figure_decisions_wrapper = figure_decisions_wrapper or {} + config = runtime_config(cli_overrides={"output_language": output_language or ""}) + resolved_output_language = normalize_output_language(config["output_language"]) + require_artifact_output_language( + figures_wrapper, + "Figure Plan", + resolved_output_language, + ) + require_artifact_output_language( + figure_decisions_wrapper, + "Figure/Table Decisions", + resolved_output_language, + ) identity_contract = identity_contract_summary(metadata, source_manifest) canonical_metadata = accepted_bundle_metadata(metadata, identity_contract) return { "status": "ok", "script": "build_synthesis_bundle.py", + "output_language": resolved_output_language, "paper_id": identity_contract.get("paper_id") or canonical_metadata.get("paper_id") or evidence_wrapper.get("paper_id", ""), @@ -476,7 +504,7 @@ def bundle( "coverage": source_manifest.get("coverage", {}) or {}, }, "source_index": source_index(source_manifest), - "references": {"candidates": sanitize_reference_candidates(evidence_pack)}, + "references": {"candidates": sanitize_reference_candidates(evidence_pack, config)}, "figure_plan": figure_plan, "figure_table_manifest": figure_table_manifest( figure_decisions_wrapper, @@ -491,7 +519,7 @@ def bundle( "figure_assets": sanitize_figure_assets(assets_wrapper), "ocr_available": assets_wrapper.get("ocr_available", False), }, - "writing_contract": compact_writing_contract(), + "writing_contract": compact_writing_contract(resolved_output_language), } @@ -510,7 +538,15 @@ def main() -> None: if decision_path.exists(): figure_decisions.setdefault("decisions_path", str(decision_path.resolve())) emit( - bundle(metadata, evidence, figures, assets, source_manifest, figure_decisions), + bundle( + metadata, + evidence, + figures, + assets, + source_manifest, + figure_decisions, + args.language, + ), args.output, ) diff --git a/skills/deeppapernote/scripts/check_environment.py b/skills/deeppapernote/scripts/check_environment.py index ac8ccfd..68f980e 100644 --- a/skills/deeppapernote/scripts/check_environment.py +++ b/skills/deeppapernote/scripts/check_environment.py @@ -11,6 +11,7 @@ from _zotero_local import probe_zotero_local_api from common import emit, env_config_value, runtime_config +from user_configuration import inspect_configuration def parser() -> argparse.ArgumentParser: @@ -68,7 +69,8 @@ def find_local_zotero_hints() -> list[str]: def main() -> None: args = parser().parse_args() - config = runtime_config() + configuration = inspect_configuration() + config = runtime_config() if configuration["state"] == "ready" else {} zotero_local_api = probe_zotero_local_api() obsidian_vault = str(config.get("obsidian_vault", "")).strip() @@ -81,6 +83,7 @@ def main() -> None: "status": "ok", "script": "check_environment.py", "tool_role": "maintenance", + "user_configuration": configuration, "python": { "executable": sys.executable, "version": sys.version.split()[0], @@ -101,8 +104,7 @@ def main() -> None: "current_working_directory": str(Path.cwd().resolve()), "workspace_output_dir": str(config.get("workspace_output_dir", "DeepPaperNote_output")), "note": ( - "If no Obsidian vault is configured, DeepPaperNote can still save notes under " - "the current working directory." + "With save_mode=workspace, DeepPaperNote saves under the current working directory." ), }, "zotero": { diff --git a/skills/deeppapernote/scripts/common.py b/skills/deeppapernote/scripts/common.py index b71ccb1..1cb6865 100644 --- a/skills/deeppapernote/scripts/common.py +++ b/skills/deeppapernote/scripts/common.py @@ -17,6 +17,12 @@ from pathlib import Path, PureWindowsPath from typing import Any +from user_configuration import ( + inspect_configuration, + resolve_preferences, + resolve_run_overrides, +) + ARXIV_NS = { "atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom", @@ -1263,6 +1269,25 @@ def _is_unique_exact_zotero_title_observation( ) +def _is_unique_exact_arxiv_title_observation( + anchor: dict[str, Any], + item: dict[str, Any], + observation: dict[str, Any], +) -> bool: + relation = item.get("relation") + if not isinstance(relation, dict): + return False + return bool( + _string_field(item, "provider").lower() == "arxiv" + and _string_field(relation, "kind") == "arxiv_lookup" + and _string_field(relation, "match_kind") == "title" + and _string_field(relation, "match_resolution") == "unique_exact" + and _record_arxiv_id(observation) + and normalize_identity_title(_string_field(anchor, "title")) + == normalize_identity_title(_string_field(observation, "title")) + ) + + def adjudicate_identity_observations( anchor: dict[str, Any], observations: list[Any], @@ -1334,6 +1359,11 @@ def adjudicate_identity_observations( item, observation, ) + unique_exact_arxiv_title_match = _is_unique_exact_arxiv_title_observation( + anchor, + item, + observation, + ) if not title_author_year_match and not shared_identifiers: observation_provider = ( _string_field(item, "provider") @@ -1377,6 +1407,7 @@ def adjudicate_identity_observations( not shared_identifiers and not title_author_year_match and not unique_exact_zotero_title_match + and not unique_exact_arxiv_title_match ): rejected_observations.append( _identity_observation_summary( @@ -1391,6 +1422,8 @@ def adjudicate_identity_observations( acceptance_reason = "shared_identifier" elif unique_exact_zotero_title_match: acceptance_reason = "unique_exact_zotero_title" + elif unique_exact_arxiv_title_match: + acceptance_reason = "unique_exact_arxiv_title" else: acceptance_reason = "title_author_year" summary = _identity_observation_summary( @@ -2344,7 +2377,13 @@ def collect_metadata_observations(record: dict[str, Any]) -> list[dict[str, Any] title = normalize_whitespace(str(base.get("title", ""))) arxiv_id = normalize_whitespace(str(base.get("arxiv_id", ""))) - def append(provider: str, kind: str, value: str, candidate: dict[str, Any] | None) -> None: + def append( + provider: str, + kind: str, + value: str, + candidate: dict[str, Any] | None, + relation: dict[str, str] | None = None, + ) -> None: if not candidate: return observation = { @@ -2352,6 +2391,8 @@ def append(provider: str, kind: str, value: str, candidate: dict[str, Any] | Non "retrieved_by": {"kind": kind, "value": value}, "record": deepcopy(candidate), } + if relation: + observation["relation"] = relation if observation not in observations: observations.append(observation) @@ -2375,11 +2416,26 @@ def append(provider: str, kind: str, value: str, candidate: dict[str, Any] | Non append("openalex", "title", title, oa) cross = choose_best_title_match(title, search_crossref_by_title(title, limit=5)) append("crossref", "title", title, cross) - arxiv = choose_best_title_match( - title, - safe_fetch_arxiv_entries(search_query=f'ti:"{title}"', max_results=5), + arxiv_candidates = safe_fetch_arxiv_entries( + search_query=f'ti:"{title}"', max_results=5 + ) + arxiv = choose_best_title_match(title, arxiv_candidates) + exact_matches = [ + candidate + for candidate in arxiv_candidates + if normalize_identity_title(_string_field(candidate, "title")) + == normalize_identity_title(title) + ] + relation = ( + { + "kind": "arxiv_lookup", + "match_kind": "title", + "match_resolution": "unique_exact", + } + if arxiv is not None and len(exact_matches) == 1 and arxiv == exact_matches[0] + else None ) - append("arxiv", "title", title, arxiv) + append("arxiv", "title", title, arxiv, relation) return observations @@ -2400,10 +2456,29 @@ def enrich_metadata(record: dict[str, Any]) -> dict[str, Any]: return apply_identity_confidence(merged) -def runtime_config() -> dict[str, Any]: +def runtime_config( + *, + explicit_overrides: dict[str, Any] | None = None, + cli_overrides: dict[str, Any] | None = None, +) -> dict[str, Any]: + resolved = resolve_run_overrides( + explicit_overrides=explicit_overrides, + cli_overrides=cli_overrides, + ) + if resolved["issues"] or resolved["missing"]: + inspection = inspect_configuration() + if inspection["state"] != "ready": + raise RuntimeError(json.dumps(inspection, ensure_ascii=False, sort_keys=True)) + resolved = resolve_preferences( + explicit_overrides=explicit_overrides, + cli_overrides=cli_overrides, + ) + if resolved["issues"] or resolved["missing"]: + raise RuntimeError(json.dumps(resolved, ensure_ascii=False, sort_keys=True)) return { - "obsidian_vault": env_config_value("DEEPPAPERNOTE_OBSIDIAN_VAULT"), - "papers_dir": env_config_value("DEEPPAPERNOTE_PAPERS_DIR", default="Research/Papers"), + "obsidian_vault": "", + "papers_dir": "", + **resolved["values"], "output_dir": env_config_value("DEEPPAPERNOTE_OUTPUT_DIR", default="tmp/DeepPaperNote"), "workspace_output_dir": env_config_value( "DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR", @@ -2413,6 +2488,8 @@ def runtime_config() -> dict[str, Any]: def configured_obsidian_vault(config: dict[str, Any]) -> Path | None: + if str(config.get("save_mode", "")).strip() == "workspace": + return None vault = str(config.get("obsidian_vault", "")).strip() if not vault: return None @@ -2425,14 +2502,18 @@ def configured_obsidian_vault(config: dict[str, Any]) -> Path | None: def require_obsidian_vault(config: dict[str, Any]) -> Path: vault_path = configured_obsidian_vault(config) if vault_path is None: - raise RuntimeError("Missing Obsidian vault configuration. Set DEEPPAPERNOTE_OBSIDIAN_VAULT.") + raise RuntimeError("Missing Obsidian Vault in User Configuration or the current Run Override.") return vault_path def resolve_note_output_mode(config: dict[str, Any]) -> tuple[str, Path]: - vault_path = configured_obsidian_vault(config) - if vault_path is not None: - return ("obsidian", vault_path) + save_mode = str(config.get("save_mode", "")).strip() + if save_mode == "obsidian": + return ("obsidian", require_obsidian_vault(config)) + if not save_mode: + vault_path = configured_obsidian_vault(config) + if vault_path is not None: + return ("obsidian", vault_path) workspace_root = Path.cwd().resolve() output_dir = str(config.get("workspace_output_dir", "DeepPaperNote_output")).strip() or "DeepPaperNote_output" output_path = workspace_root / Path( @@ -2983,8 +3064,15 @@ def infer_domain_label(title: str, abstract: str = "") -> str: def is_probable_paper_folder(path: Path) -> bool: if not path.is_dir(): return False - marker = path / f"{path.name}.md" - return marker.exists() + return any( + (path / marker).exists() + for marker in ( + f"{path.name}.md", + f"{path.name}.zh-CN.md", + f"{path.name}.en.md", + ".deeppapernote.json", + ) + ) def existing_domain_dirs(config: dict[str, Any]) -> list[str]: diff --git a/skills/deeppapernote/scripts/contracts.py b/skills/deeppapernote/scripts/contracts.py index e4a3dbe..0d37641 100644 --- a/skills/deeppapernote/scripts/contracts.py +++ b/skills/deeppapernote/scripts/contracts.py @@ -3,8 +3,11 @@ from __future__ import annotations +from copy import deepcopy from typing import Any, TypedDict +from localization import note_schema, normalize_output_language, required_sections + NOTE_REQUIRED_SECTIONS: tuple[str, ...] = ( "核心信息", "原文摘要翻译", @@ -20,6 +23,9 @@ "引用", ) +def note_required_sections(language: str | None = None) -> tuple[str, ...]: + return required_sections(language) + PAPER_TYPE_VALUES: tuple[str, ...] = ( "AI_method", "benchmark_or_dataset", @@ -269,6 +275,145 @@ def required_field_value_error( }, } +PAPER_TYPE_CONTRACTS_EN: dict[str, dict[str, Any]] = { + "AI_method": { + "paper_type": "AI_method", + "reader_lens": "A technical reader who may need to reproduce the method and its mechanism.", + "section_focus": ["problem setting", "method mechanism", "training or inference flow", "key equations", "strong baselines", "ablations and failure boundaries"], + "required_checks": ["Explain the mechanism flow, essential equations, experimental design, what the ablations establish, and the failure boundary."], + "formula_rules": ["Keep only the one to three equations needed to understand the method and explain their engineering meaning."], + "avoid_rules": ["Do not force a non-method paper into a model-architecture narrative."], + "boundary_questions": [ + "Which experiment or ablation supports the claimed benefit of the core mechanism?", + "Which comparisons apply only under the reported data, baselines, compute, or protocol?", + "What evidence shows failure, degradation, instability, or rising cost; if none is reported, what remains unproven?", + ], + "section_semantics": { + "Research Question": "The specific technical problem and the shortcomings of existing methods.", + "Data and Task Definition": "Datasets, inputs and outputs, evaluation tasks, and experimental settings.", + "Method": "Model, algorithm, training, and inference mechanisms.", + "Key Results": "Main results, strong baselines, ablations, and decisive numbers.", + "Deep Analysis": "Why the method works, where it is fragile, and the cost of reproduction or extension.", + }, + "recommended_subsections": { + "Method": ["Mechanism Flow", "Model Architecture", "Training Objective", "Inference and Sampling", "Implementation Details"], + "Key Results": ["Main Results and Strong Baselines", "What the Ablations Establish", "Failure or Unstable Settings"], + "Deep Analysis": ["Why It Works", "Complexity and Scalability", "Reproduction Notes"], + }, + "mechanism_flow_contract": {"apply_when_paper_type_in": ["AI_method"], "required_step_count": "3_to_4", "required_step_fields": ["input", "operation", "output_destination"]}, + }, + "benchmark_or_dataset": { + "paper_type": "benchmark_or_dataset", + "reader_lens": "A researcher assessing whether a benchmark or dataset is useful and where it is biased.", + "section_focus": ["task decomposition", "data sources and construction", "annotation protocol", "evaluation metrics", "coverage and bias", "sample statistics and access limits"], + "required_checks": ["Explain sources, construction or annotation, metrics, baselines, sample statistics, access or privacy constraints, and applicability."], + "formula_rules": ["Keep only essential metrics, sampling rules, or split definitions."], + "avoid_rules": ["Do not describe data construction as a model pipeline."], + "boundary_questions": [ + "What construct does the resource actually measure, and which capabilities are only proxies?", + "Which coverage gaps or biases follow from its tasks, labels, sampling, filtering, or evaluation protocol?", + "Do baseline results demonstrate discrimination, or only adaptation to this protocol?", + "How do sample composition, access, and privacy limits affect reproduction and generalization?", + ], + "section_semantics": { + "Research Question": "The evaluation or data gap the resource is designed to address.", + "Data and Task Definition": "Sources, task splits, labels, and sample scope.", + "Method": "Construction, filtering, annotation, and evaluation protocol—not a model pipeline.", + "Key Results": "Baseline performance, difficulty, coverage, and bias.", + "Deep Analysis": "What the resource measures and what it cannot represent.", + }, + "recommended_subsections": { + "Data and Task Definition": ["Data Sources", "Task Splits", "Annotation and Filtering"], + "Method": ["Construction Process", "Evaluation Protocol", "Baseline Setup"], + "Key Results": ["Baseline Performance", "Difficulty Distribution", "Coverage and Bias"], + "Deep Analysis": ["What It Actually Measures", "Applicability Boundary"], + }, + }, + "clinical_or_psychology_empirical": { + "paper_type": "clinical_or_psychology_empirical", + "reader_lens": "A research reader focused on samples, variable relationships, uncertainty, and generalization.", + "section_focus": ["sample source", "inclusion and exclusion", "variables and instruments", "analysis pipeline", "effect sizes and uncertainty", "ethics, access, and generalization"], + "required_checks": ["Distinguish association, prediction, group difference, and causal interpretation; report sample, ethics, privacy, and generalization limits."], + "formula_rules": ["Keep only essential statistical models, effect sizes, intervals, or instrument definitions."], + "avoid_rules": ["Do not turn association, prediction, or group differences into unsupported causal claims."], + "boundary_questions": [ + "How do recruitment, eligibility, measurement, and annotation constrain generalization?", + "Does the design support association, prediction, group difference, or causality?", + "Does the interpretation depend on unobserved confounding, thresholds, missingness, or setting?", + "How do sample composition, missing data, privacy, and unavailable materials constrain reproduction?", + ], + "section_semantics": { + "Research Question": "The clinical, psychological, or behavioral question, hypothesis, or variable relationship.", + "Data and Task Definition": "Recruitment, eligibility, variables, instruments, and measurement.", + "Method": "Study design, grouping, measurement flow, and statistical analysis.", + "Key Results": "Effects, associations, group differences, uncertainty, and significance.", + "Deep Analysis": "Interpretation, causal boundary, substantive meaning, and generalization limits.", + }, + "recommended_subsections": { + "Data and Task Definition": ["Sample and Eligibility", "Variables and Instruments", "Measurement Process"], + "Method": ["Study Design", "Analysis Model", "Primary Comparisons"], + "Key Results": ["Primary Effects", "Uncertainty and Significance", "Clinical or Psychological Interpretation"], + "Deep Analysis": ["Causal Interpretation Boundary", "Generalization Limits"], + }, + }, + "humanities_or_social_science": { + "paper_type": "humanities_or_social_science", + "reader_lens": "A reader evaluating theoretical framing, material interpretation, and argument structure.", + "section_focus": ["object of study", "materials", "theoretical framework", "argument path", "conceptual contribution", "interpretive boundary"], + "required_checks": ["Distinguish the author's argument, material evidence, normative judgment, and empirical fact."], + "formula_rules": ["Do not force equations; retain only essential formal definitions or coding rules."], + "avoid_rules": ["Do not present normative judgment, textual interpretation, or case analysis as experimental fact."], + "boundary_questions": [ + "Which materials, cases, or theoretical premises support the interpretation?", + "What alternative explanations fit the same material, and how are they addressed?", + "Which conclusions are conceptual or normative rather than directly empirical?", + ], + "section_semantics": { + "Research Question": "The social, cultural, historical, institutional, or theoretical problem.", + "Data and Task Definition": "Materials, cases, texts, interviews, archives, or corpus scope—not an ML task.", + "Method": "Theoretical framework, conceptual distinctions, and argument path.", + "Key Results": "Interpretive findings, conceptual contribution, or revision of prior views.", + "Deep Analysis": "Argument strength, material limits, alternative explanations, and transferability.", + }, + "recommended_subsections": { + "Data and Task Definition": ["Material Scope", "Selection Criteria", "Case or Corpus Boundary"], + "Method": ["Theoretical Framework", "Conceptual Distinctions", "Argument Path"], + "Key Results": ["Core Interpretive Findings", "Conceptual Contribution"], + "Deep Analysis": ["Argument Strength", "Alternative Explanations", "Material Boundary"], + }, + }, + "survey_or_review": { + "paper_type": "survey_or_review", + "reader_lens": "A reader mapping a literature, taxonomy, evidence boundary, and open questions.", + "section_focus": ["review scope", "inclusion and exclusion", "taxonomy", "method families", "consensus and disagreement", "open questions"], + "required_checks": ["Explain scope, study selection, taxonomy, consensus, disagreement, and open questions."], + "formula_rules": ["Keep only classification axes, eligibility rules, evidence-synthesis rules, or meta-analytic statistics."], + "avoid_rules": ["Do not present findings summarized from the literature as a new experiment by the review authors."], + "boundary_questions": [ + "Which research routes may be missed by the search scope, eligibility criteria, or taxonomy?", + "Which statements reflect consensus, author-defined categories, or unresolved disagreement?", + "Which trends are artifacts of the covered literature and cannot establish technical maturity?", + ], + "section_semantics": { + "Research Question": "The field problem, controversy, or knowledge gap organized by the review.", + "Data and Task Definition": "Literature scope, search and screening criteria, and review objects.", + "Method": "Taxonomy, review organization, and evidence-synthesis logic—not a single method architecture.", + "Key Results": "Consensus, disagreement, trends, representative directions, and open questions.", + "Deep Analysis": "Coverage blind spots, explanatory power of the taxonomy, and future opportunities.", + }, + "recommended_subsections": { + "Data and Task Definition": ["Review Scope", "Inclusion and Exclusion", "Literature Coverage"], + "Method": ["Taxonomy", "Method Families", "Evidence Organization"], + "Key Results": ["Representative Directions", "Consensus and Disagreement", "Open Questions"], + "Deep Analysis": ["Taxonomy Limits", "Uncovered Areas", "Future Research Opportunities"], + }, + }, +} + + +def paper_type_contracts(language: str | None = None) -> dict[str, dict[str, Any]]: + return deepcopy(PAPER_TYPE_CONTRACTS_EN if normalize_output_language(language) == "en" else PAPER_TYPE_CONTRACTS) + WRITING_CONTRACT_RULES: dict[str, Any] = { "required_sections": NOTE_REQUIRED_SECTIONS, "paper_type_values": PAPER_TYPE_VALUES, @@ -422,6 +567,20 @@ def required_field_value_error( }, } +def writing_contract_rules(language: str | None = None) -> dict[str, Any]: + resolved = normalize_output_language(language) + schema = note_schema(resolved) + rules = deepcopy(WRITING_CONTRACT_RULES) + rules["language"] = resolved + rules["required_sections"] = tuple(schema["sections"].values()) + rules["grounding_required_sections"] = tuple(schema["sections"][key] for key in ("research_questions", "data_and_task", "method", "key_results", "deep_analysis", "limitations")) + rules["core_info_fields"] = tuple(schema["core_info_fields"]) + rules["figure_labels"] = dict(schema["figure_labels"]) + rules["mechanism_flow_heading"] = schema["mechanism_flow"] + if schema.get("abstract_contract"): + rules["abstract_contract"] = deepcopy(schema["abstract_contract"]) + return rules + class MetadataRecord(TypedDict, total=False): title: str diff --git a/skills/deeppapernote/scripts/extract_source_text.py b/skills/deeppapernote/scripts/extract_source_text.py index 3d5f447..8746318 100644 --- a/skills/deeppapernote/scripts/extract_source_text.py +++ b/skills/deeppapernote/scripts/extract_source_text.py @@ -16,6 +16,7 @@ ensure_parent, extract_appendix_index, extract_caption_lines, + file_sha256, fitz, match_section_heading, maybe_load_json_record, @@ -302,6 +303,7 @@ def build_manifest( "schema_version": 1, "paper_id": record.get("paper_id") or paper_id_for_record(record), "title": record.get("title", ""), + "source_sha256": record.get("source_sha256", ""), "source_kind": "pdf_text", "raw_sections_path": ( str(Path(raw_sections_output).expanduser().resolve()) if raw_sections_output else "" @@ -353,6 +355,11 @@ def main() -> None: pdf_path = resolve_pdf_path(record) if pdf_path is None: raise SystemExit("extract_source_text.py requires a resolvable local PDF path.") + source_sha256 = file_sha256(pdf_path) + reported_source_sha256 = normalize_whitespace(str(record.get("source_sha256", ""))) + if reported_source_sha256 and reported_source_sha256 != source_sha256: + raise SystemExit("extract_source_text.py source_sha256 does not match acquired PDF.") + record["source_sha256"] = source_sha256 raw_sections_output = args.raw_sections_output if not raw_sections_output and args.output: diff --git a/skills/deeppapernote/scripts/fetch_pdf.py b/skills/deeppapernote/scripts/fetch_pdf.py index 4329a89..ee68af4 100644 --- a/skills/deeppapernote/scripts/fetch_pdf.py +++ b/skills/deeppapernote/scripts/fetch_pdf.py @@ -11,6 +11,7 @@ default_pdf_path, emit, fetch_record_from_canonical_identity, + file_sha256, http_get_bytes, maybe_load_json_record, paper_id_for_record, @@ -103,6 +104,7 @@ def main(argv: list[str] | None = None) -> None: "pdf_source": "local_pdf", "source_url": record.get("source_url", "") or str(pdf_path), "pdf_url": "", + "source_sha256": file_sha256(pdf_path), } if identity_summary: payload["identity_contract"] = identity_summary @@ -157,6 +159,7 @@ def main(argv: list[str] | None = None) -> None: "source_url": record.get("source_url", ""), "pdf_url": source_value, "file_size": target_path.stat().st_size, + "source_sha256": file_sha256(target_path), } if identity_summary: payload["identity_contract"] = identity_summary diff --git a/skills/deeppapernote/scripts/lint_grounding.py b/skills/deeppapernote/scripts/lint_grounding.py index 3a48333..9c6b257 100644 --- a/skills/deeppapernote/scripts/lint_grounding.py +++ b/skills/deeppapernote/scripts/lint_grounding.py @@ -21,7 +21,9 @@ PAPER_TYPE_VALUES, WRITING_CONTRACT_RULES, required_field_value_error, + writing_contract_rules, ) +from localization import normalize_output_language, require_artifact_output_language from source_corpus import SourceCorpusLoadError, load_source_corpus @@ -186,6 +188,11 @@ def parser() -> argparse.ArgumentParser: help="Figure/table decisions JSON path or JSON string.", ) p.add_argument("--output", default="", help="Output JSON path.") + p.add_argument( + "--language", + default="", + help="Run Override for output language: en or zh-CN.", + ) return p @@ -379,6 +386,7 @@ def source_grounding_errors(source: Any, valid_ids: set[str], max_page: int) -> def validate_note_plan( note_plan: dict[str, Any], source_manifest: dict[str, Any], + language: str | None = None, ) -> list[dict[str, Any]]: issues: list[dict[str, Any]] = [] required_checks = WRITING_CONTRACT_RULES["note_plan_required_field_checks"] @@ -400,7 +408,7 @@ def validate_note_plan( valid_ids = source_section_ids(source_manifest) max_page = total_pages(source_manifest) - required_sections = set(WRITING_CONTRACT_RULES["grounding_required_sections"]) + required_sections = set(writing_contract_rules(language)["grounding_required_sections"]) grounded_sections: set[str] = set() section_plan = note_plan.get("section_plan", []) if not isinstance(section_plan, list): @@ -638,14 +646,35 @@ def validate_figure_decisions( def main() -> None: + from common import runtime_config + args = parser().parse_args() note_plan = load_record(args.note_plan) source_manifest = load_record(args.source_manifest) bundle = load_record(args.bundle_json) if args.bundle_json else {} decisions = load_record(args.figure_decisions) + language = normalize_output_language( + runtime_config(cli_overrides={"output_language": args.language})["output_language"] + ) + writing_contract = bundle.get("writing_contract", {}) if isinstance(bundle, dict) else {} issues = [] - issues.extend(validate_note_plan(note_plan, source_manifest)) + for artifact, name in ( + (bundle, "Synthesis Bundle"), + (note_plan, "Note Plan"), + (decisions, "Figure/Table Decisions"), + ( + {"output_language": writing_contract.get("language")} + if isinstance(writing_contract, dict) + else {}, + "Synthesis Bundle writing_contract", + ), + ): + try: + require_artifact_output_language(artifact, name, language) + except ValueError as exc: + issues.append(issue("output_language_contract_failed", artifact=name, reason=str(exc))) + issues.extend(validate_note_plan(note_plan, source_manifest, language)) issues.extend(validate_bundle_contract(note_plan, bundle)) issues.extend(validate_figure_decisions(source_manifest, decisions, args.source_manifest)) error_issues = [item for item in issues if item.get("severity", "error") == "error"] @@ -653,6 +682,7 @@ def main() -> None: "status": "ok", "script": "lint_grounding.py", "paper_id": source_manifest.get("paper_id", note_plan.get("paper_id", "")), + "output_language": language, "issues": issues, "warnings": [item for item in issues if item.get("severity") == "warning"], "passes_grounding": not error_issues, diff --git a/skills/deeppapernote/scripts/lint_note.py b/skills/deeppapernote/scripts/lint_note.py index fcc71e3..619babd 100644 --- a/skills/deeppapernote/scripts/lint_note.py +++ b/skills/deeppapernote/scripts/lint_note.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import re from pathlib import Path @@ -16,35 +17,45 @@ WRITING_CONTRACT_RULES, required_field_value_error, ) +from localization import ( + normalize_output_language, + note_schema, + require_artifact_output_language, +) -REQUIRED_SECTIONS = NOTE_REQUIRED_SECTIONS - -CORE_INFO_FIELDS = [ - "标题", - "标题翻译", - "作者", - "机构", - "发表时间", - "发表渠道", - "DOI", - "arXiv", - "论文链接", - "代码 / 项目", - "数据 / 资源", - "论文类型", -] - -CORE_INFO_FIELD_INDEX = {field: idx for idx, field in enumerate(CORE_INFO_FIELDS)} - -FIGURE_TARGET_SECTIONS = { - "研究问题", - "数据与任务定义", - "方法主线", - "关键结果", - "深度分析", - "局限", - "我的笔记", -} +ACTIVE_LANGUAGE = "zh-CN" +SCHEMA: dict = {} +SECTIONS: dict[str, str] = {} +REQUIRED_SECTIONS: tuple[str, ...] = NOTE_REQUIRED_SECTIONS +CORE_INFO_FIELDS: list[str] = [] +CORE_INFO_FIELD_INDEX: dict[str, int] = {} +CORE_INFO_FIELD_ALIASES: dict[str, str] = {} +FIGURE_TARGET_SECTIONS: set[str] = set() +FIGURE_LABELS: dict[str, str] = {} +MECHANISM_FLOW_HEADING = "机制流程" + +def configure_output_language(language: str | None = None) -> str: + global ACTIVE_LANGUAGE, SCHEMA, SECTIONS, REQUIRED_SECTIONS, CORE_INFO_FIELDS + global CORE_INFO_FIELD_INDEX, CORE_INFO_FIELD_ALIASES, FIGURE_TARGET_SECTIONS, FIGURE_LABELS, MECHANISM_FLOW_HEADING + ACTIVE_LANGUAGE = normalize_output_language(language) + SCHEMA = note_schema(ACTIVE_LANGUAGE) + SECTIONS = dict(SCHEMA["sections"]) + REQUIRED_SECTIONS = tuple(SECTIONS.values()) + CORE_INFO_FIELDS = list(SCHEMA["core_info_fields"]) + CORE_INFO_FIELD_INDEX = {field: idx for idx, field in enumerate(CORE_INFO_FIELDS)} + CORE_INFO_FIELD_ALIASES = dict(SCHEMA.get("core_info_aliases", {})) + FIGURE_TARGET_SECTIONS = {SECTIONS[key] for key in ("research_questions", "data_and_task", "method", "key_results", "deep_analysis", "limitations", "my_notes")} + FIGURE_LABELS = dict(SCHEMA["figure_labels"]) + MECHANISM_FLOW_HEADING = str(SCHEMA["mechanism_flow"]) + return ACTIVE_LANGUAGE + +def section(key: str) -> str: + return SECTIONS[key] + +def figure_prefix(key: str) -> str: + return f"> {FIGURE_LABELS[key]}" + +configure_output_language() FIGURE_BUCKET_RESIDUE_TOKENS = { "剩余", @@ -132,7 +143,7 @@ | clear\s+crop | - high[-\s]*(?:confidence|match) + (?\s*\[!figure\][+-]?\s*(.*)$") HTTP_URL_RE = re.compile(r"https?://\S+") +ENGLISH_METADATA_SOURCE_SPAN_PATTERNS = ( + re.compile(r"`[^`\n]+`"), + re.compile(r"\[[^\]\n]+\]\([^)\n]+\)"), +) +ENGLISH_CJK_ENTITY_LINK_RE = re.compile( + r"\[[^\]\n]*[\u4e00-\u9fff][^\]\n]*\]\(https?://[^)\n]+\)" + r"|\[\[[^\]\n]*[\u4e00-\u9fff][^\]\n]*\]\]" +) +ENGLISH_CJK_MATH_LABEL_RE = re.compile( + r"\\operatorname\{(?:输入|输出|损失|状态|动作|奖励|标签|样本|预测|目标)\}" +) +ENGLISH_MATH_SPAN_RE = re.compile(r"\$\$[^$\n]+\$\$|\$[^$\n]+\$") RUNTIME_ARTIFACT_REFERENCE_PATTERNS = [ re.compile( @@ -230,6 +253,7 @@ def parser() -> argparse.ArgumentParser: p.add_argument("--plan-file", default="", help="Optional note_plan JSON path. Defaults to sibling .plan.json.") p.add_argument("--output", default="", help="Output JSON path.") p.add_argument("--paper-id", default="", help="Canonical paper id.") + p.add_argument("--language", default="", help="Run Override for output language: en or zh-CN.") return p @@ -239,7 +263,10 @@ def resolve_note_plan_path(note_path: Path, plan_file: str) -> Path: return note_path.with_suffix(".plan.json") -def inspect_note_plan(plan_path: Path) -> tuple[bool, list[str]]: +def inspect_note_plan( + plan_path: Path, + output_language: str | None = None, +) -> tuple[bool, list[str]]: if not plan_path.exists(): return False, ["planning_artifact_missing"] @@ -252,6 +279,14 @@ def inspect_note_plan(plan_path: Path) -> tuple[bool, list[str]]: return True, ["planning_required_fields_invalid"] issues: list[str] = [] + try: + require_artifact_output_language( + plan, + "Note Plan", + output_language or ACTIVE_LANGUAGE, + ) + except ValueError as exc: + issues.append(f"planning_output_language_contract_failed: {exc}") missing_fields = [field for field in NOTE_PLAN_REQUIRED_FIELDS if field not in plan] if missing_fields: issues.append("planning_required_fields_missing") @@ -322,10 +357,10 @@ def find_missing_sections(text: str) -> list[str]: def front_matter_order_warnings(text: str) -> list[str]: warnings: list[str] = [] - required_order = ["## 原文摘要翻译", "## 创新点", "## 一句话总结"] + required_order = [f"## {section('abstract')}", f"## {section('contributions')}", f"## {section('one_sentence_summary')}"] positions = [] - for section in required_order: - idx = text.find(section) + for required_heading in required_order: + idx = text.find(required_heading) if idx < 0: return warnings positions.append(idx) @@ -334,6 +369,13 @@ def front_matter_order_warnings(text: str) -> list[str]: return warnings +def english_top_level_section_warnings(text: str) -> list[str]: + if ACTIVE_LANGUAGE != "en": + return [] + actual = re.findall(r"^##\s+(.+?)\s*$", text, flags=re.MULTILINE) + return [] if actual == list(REQUIRED_SECTIONS) else ["top_level_section_profile_invalid"] + + def _inside_any_span(start: int, end: int, spans: list[tuple[int, int]]) -> bool: return any(span_start <= start and end <= span_end for span_start, span_end in spans) @@ -376,6 +418,9 @@ def inspect_reference_hygiene(text: str) -> list[dict[str, object]]: "decoder", "pipeline", "framework", + "model", + "system", + "module", ] MECHANISM_IO_TOKENS = [ @@ -385,6 +430,10 @@ def inspect_reference_hygiene(text: str) -> list[dict[str, object]]: "送到", "生成", "得到", + "input", + "output", + "produces", + "returns", ] MECHANISM_ACTION_TOKENS = [ @@ -399,6 +448,14 @@ def inspect_reference_hygiene(text: str) -> list[dict[str, object]]: "拼接", "查询", "更新", + "align", + "compute", + "estimate", + "extract", + "encode", + "decode", + "update", + "aggregate", ] @@ -498,7 +555,7 @@ def inspect_reference_hygiene(text: str) -> list[dict[str, object]]: def is_metadata_line(line: str) -> bool: stripped = line.strip() - prefixes = [f"- {field}:" for field in CORE_INFO_FIELDS] + prefixes = [f"- {field}:" for field in (*CORE_INFO_FIELDS, *CORE_INFO_FIELD_ALIASES)] return any(stripped.startswith(prefix) for prefix in prefixes) @@ -512,9 +569,9 @@ def is_exempt_line(line: str) -> bool: return True if ( stripped.startswith("> [!figure]") - or stripped.startswith("> 建议位置：") - or stripped.startswith("> 放置原因：") - or stripped.startswith("> 当前状态：") + or stripped.startswith(figure_prefix("location")) + or stripped.startswith(figure_prefix("reason")) + or stripped.startswith(figure_prefix("status")) ): return True if re.search(r"https?://", stripped): @@ -550,13 +607,42 @@ def subsection_name_for_line(lines: list[str], line_index: int) -> str: def mixed_language_issues(text: str) -> list[dict[str, object]]: issues: list[dict[str, object]] = [] lines = text.splitlines() + fenced_code_lines: set[int] = set() + fence_start: int | None = None + for line_index, line in enumerate(lines, start=1): + if not line.strip().startswith("```"): + continue + if fence_start is None: + fence_start = line_index + else: + fenced_code_lines.update(range(fence_start, line_index + 1)) + fence_start = None for idx, line in enumerate(lines, start=1): + stripped = line.strip() + if ACTIVE_LANGUAGE == "en": + if idx in fenced_code_lines: + continue + checked = stripped + section_name = section_name_for_line(lines, idx - 1) + if section_name in {section("core_information"), section("references")}: + for pattern in ENGLISH_METADATA_SOURCE_SPAN_PATTERNS: + checked = pattern.sub("", checked) + if stripped.startswith(f"*{FIGURE_LABELS['original_caption']}"): + checked = ENGLISH_METADATA_SOURCE_SPAN_PATTERNS[0].sub("", checked) + checked = ENGLISH_CJK_ENTITY_LINK_RE.sub("", checked) + checked = ENGLISH_MATH_SPAN_RE.sub( + lambda match: ENGLISH_CJK_MATH_LABEL_RE.sub("", match.group(0)), + checked, + ) + checked = HTTP_URL_RE.sub("", checked) + if re.search(r"[\u4e00-\u9fff]", checked): + issues.append({"line_number": idx, "line": stripped, "reason": "non_english_text_present"}) + continue if is_exempt_line(line): continue - stripped = line.strip() section_name = section_name_for_line(lines, idx - 1) subsection_name = subsection_name_for_line(lines, idx - 1) - if section_name in {"核心信息", "引用"}: + if section_name in {section("core_information"), section("references")}: continue if not re.search(r"[\u4e00-\u9fff]", stripped): continue @@ -578,6 +664,8 @@ def mixed_language_issues(text: str) -> list[dict[str, object]]: def mechanical_translation_artifact_issues(text: str) -> list[dict[str, object]]: + if ACTIVE_LANGUAGE == "en": + return [] issues: list[dict[str, object]] = [] for idx, line in enumerate(text.splitlines(), start=1): stripped = line.strip() @@ -620,11 +708,11 @@ def inspect_figure_callouts(text: str) -> list[str]: nxt = lines[j].strip() if not nxt.startswith(">"): break - if nxt.startswith("> 建议位置："): + if nxt.startswith(figure_prefix("location")): has_location = True - if nxt.startswith("> 放置原因："): + if nxt.startswith(figure_prefix("reason")): has_reason = True - if nxt.startswith("> 当前状态："): + if nxt.startswith(figure_prefix("status")): has_status = True j += 1 if not has_location: @@ -648,9 +736,10 @@ def figure_callout_title(line: str) -> str: def figure_status_text(line: str) -> str: stripped = line.strip() - if not stripped.startswith("> 当前状态："): + prefix = figure_prefix("status") + if not stripped.startswith(prefix): return "" - return stripped.removeprefix("> 当前状态：").strip() + return stripped.removeprefix(prefix).strip() def has_accepted_usable_placeholder_reason(status_text: str) -> bool: @@ -748,8 +837,9 @@ def figure_callout_placement_issues(text: str) -> list[dict[str, object]]: nxt = lines[j].strip() if not nxt.startswith(">"): break - if nxt.startswith("> 建议位置："): - location = nxt.removeprefix("> 建议位置：").strip() + prefix = figure_prefix("location") + if nxt.startswith(prefix): + location = nxt.removeprefix(prefix).strip() break j += 1 @@ -919,14 +1009,15 @@ def figure_structure_passes(text: str) -> bool: def core_info_structure_issues(text: str) -> list[dict[str, object]]: - body = section_body(text, "核心信息") + core_heading = section("core_information") + body = section_body(text, core_heading) if not body: return [] issues: list[dict[str, object]] = [] seen_fields: set[str] = set() last_known_index = -1 - base_line = _line_number_from_offset(text, text.find("## 核心信息")) + base_line = _line_number_from_offset(text, text.find(f"## {core_heading}")) for offset, raw_line in enumerate(body.splitlines(), start=1): stripped = raw_line.strip() @@ -945,6 +1036,7 @@ def core_info_structure_issues(text: str) -> list[dict[str, object]]: continue field = match.group(1).strip() + field = CORE_INFO_FIELD_ALIASES.get(field, field) if field not in CORE_INFO_FIELD_INDEX: issues.append( { @@ -987,7 +1079,7 @@ def is_prose_line(line: str) -> bool: stripped = line.strip() if not stripped: return False - if stripped.startswith(("#", "-", "*", "> ", "```", "![[", "*论文原图编号")): + if stripped.startswith(("#", "-", "*", "> ", "```", "![[", f"*{FIGURE_LABELS['original_caption']}")): return False if stripped.startswith("`") and stripped.endswith("`"): return False @@ -1085,6 +1177,14 @@ def _strip_fenced_code_preserve_newlines(text: str) -> str: def _extract_math_blocks(text: str) -> tuple[list[dict[str, object]], list[dict[str, object]]]: sanitized = _strip_fenced_code_preserve_newlines(text) + if ACTIVE_LANGUAGE == "en": + # Currency amounts such as "$25 billion" are prose, not LaTeX delimiters. + sanitized = re.sub( + r"\$(?=\d+(?:\.\d+)?\s+(?:thousand|million|billion|trillion|dollars?|usd)\b)", + r"\$", + sanitized, + flags=re.IGNORECASE, + ) blocks: list[dict[str, object]] = [] issues: list[dict[str, object]] = [] consumed_lines: set[int] = set() @@ -1304,14 +1404,14 @@ def cleaned_section_lines(body: str) -> list[str]: continue if ( stripped.startswith("> [!figure]") - or stripped.startswith("> 建议位置：") - or stripped.startswith("> 放置原因：") - or stripped.startswith("> 当前状态：") + or stripped.startswith(figure_prefix("location")) + or stripped.startswith(figure_prefix("reason")) + or stripped.startswith(figure_prefix("status")) ): continue if stripped.startswith("!["): continue - if stripped.startswith("*论文原图编号：") and stripped.endswith("*"): + if stripped.startswith(f"*{FIGURE_LABELS['original_caption']}") and stripped.endswith("*"): continue if stripped.startswith("> "): stripped = stripped[2:].strip() @@ -1405,6 +1505,8 @@ def has_reference_entry(text: str) -> bool: return True if re.search(r"\b[A-Z][A-Za-z-]+ et al\.?\s*,?\s*(?:19|20)\d{2}\b", normalized): return True + if re.search(r"\b[A-Z][A-Za-z-]+(?:\s+(?:and|&|et al\.?|[A-Z][A-Za-z-]+))*\s*\((?:19|20)\d{2}\)", normalized): + return True if re.search(r"(?:19|20)\d{2}.*(?:DOI|doi|会议|期刊|arXiv)", normalized): return True return False @@ -1412,62 +1514,66 @@ def has_reference_entry(text: str) -> bool: def inspect_substantive_content(text: str) -> list[dict[str, object]]: issues: list[dict[str, object]] = [] - for section in REQUIRED_SECTIONS: - body = section_body(text, section) + for section_heading in REQUIRED_SECTIONS: + body = section_body(text, section_heading) content = normalized_section_content(body) if is_placeholder_like(content): - issues.append(issue(section, "section_empty_shell", "error", content or section)) - if section not in {"关键结果", "引用"} and is_honest_missing_declaration(content): - issues.append(issue(section, "section_honest_missing_not_allowed", "error", content)) + issues.append(issue(section_heading, "section_empty_shell", "error", content or section_heading)) + if section_heading not in {section("key_results"), section("references")} and is_honest_missing_declaration(content): + issues.append(issue(section_heading, "section_honest_missing_not_allowed", "error", content)) - innovation = section_body(text, "创新点") + contributions_heading = section("contributions") + innovation = section_body(text, contributions_heading) innovation_content = normalized_section_content(innovation) innovation_units = meaningful_units(innovation, GENERIC_INNOVATION_PATTERNS) if not innovation_units: - issues.append(issue("创新点", "innovation_empty_shell", "error", innovation_content)) + issues.append(issue(contributions_heading, "innovation_empty_shell", "error", innovation_content)) elif len(innovation_units) < 2: - issues.append(issue("创新点", "innovation_too_few_specific_points", "warning", innovation_content)) + issues.append(issue(contributions_heading, "innovation_too_few_specific_points", "warning", innovation_content)) - key_results = section_body(text, "关键结果") + key_results_heading = section("key_results") + key_results = section_body(text, key_results_heading) key_results_content = normalized_section_content(key_results) if is_honest_missing_declaration(key_results_content): issues.append( issue( - "关键结果", + key_results_heading, "key_results_honest_missing_not_allowed", "error", key_results_content, ) ) elif not meaningful_units(key_results, GENERIC_KEY_RESULT_PATTERNS): - issues.append(issue("关键结果", "key_results_empty_shell", "error", key_results_content)) + issues.append(issue(key_results_heading, "key_results_empty_shell", "error", key_results_content)) elif not has_number_token(key_results_content): issues.append( issue( - "关键结果", + key_results_heading, "key_results_quantitative_result_missing", "warning", key_results_content, ) ) - references = section_body(text, "引用") + references_heading = section("references") + references = section_body(text, references_heading) references_content = normalized_section_content(references) if is_honest_missing_declaration(references_content): - issues.append(issue("引用", "references_unavailable_declared", "warning", references_content)) + issues.append(issue(references_heading, "references_unavailable_declared", "warning", references_content)) elif is_placeholder_like(references_content) or not has_reference_entry(references_content): - issues.append(issue("引用", "references_placeholder", "error", references_content)) + issues.append(issue(references_heading, "references_placeholder", "error", references_content)) - limitations = section_body(text, "局限") + limitations_heading = section("limitations") + limitations = section_body(text, limitations_heading) limitations_content = normalized_section_content(limitations) if not meaningful_units(limitations, GENERIC_LIMITATION_PATTERNS): - issues.append(issue("局限", "limitations_empty_shell", "error", limitations_content)) + issues.append(issue(limitations_heading, "limitations_empty_shell", "error", limitations_content)) - for section in ("方法主线", "深度分析"): - body = section_body(text, section) + for section_heading in (section("method"), section("deep_analysis")): + body = section_body(text, section_heading) content = normalized_section_content(body) if not meaningful_units(body): - issues.append(issue(section, "section_empty_shell", "error", content or section)) + issues.append(issue(section_heading, "section_empty_shell", "error", content or section_heading)) deduped: list[dict[str, object]] = [] seen: set[tuple[str, str, str]] = set() @@ -1481,7 +1587,7 @@ def inspect_substantive_content(text: str) -> list[dict[str, object]]: def method_section_requires_mechanism_flow(text: str) -> bool: - body = section_body(text, "方法主线") + body = section_body(text, section("method")) if not body: return False lower = body.lower() @@ -1492,13 +1598,19 @@ def method_section_requires_mechanism_flow(text: str) -> bool: def mechanism_flow_warnings(text: str) -> list[str]: warnings: list[str] = [] + if ACTIVE_LANGUAGE == "en": + for heading in re.findall(r"^###\s+(.+?)\s*$", text, flags=re.MULTILINE): + if heading.casefold() == MECHANISM_FLOW_HEADING.casefold() and heading != MECHANISM_FLOW_HEADING: + warnings.append("mechanism_flow_heading_invalid") + break if not method_section_requires_mechanism_flow(text): return warnings - if "### 机制流程" not in text: + heading_pattern = rf"^###\s+{re.escape(MECHANISM_FLOW_HEADING)}\s*$" + if not re.search(heading_pattern, text, flags=re.MULTILINE): warnings.append("mechanism_flow_subsection_missing") return warnings - body = subsection_body(text, "方法主线", "机制流程") + body = subsection_body(text, section("method"), MECHANISM_FLOW_HEADING) if not body: warnings.append("mechanism_flow_subsection_empty") return warnings @@ -1508,8 +1620,9 @@ def mechanism_flow_warnings(text: str) -> list[str]: warnings.append("mechanism_flow_step_count_unexpected") step_text = " ".join(step_lines) - has_io_signal = any(token in step_text for token in MECHANISM_IO_TOKENS) - has_action_signal = any(token in step_text for token in MECHANISM_ACTION_TOKENS) + step_text_lower = step_text.lower() + has_io_signal = any(token.lower() in step_text_lower for token in MECHANISM_IO_TOKENS) + has_action_signal = any(token.lower() in step_text_lower for token in MECHANISM_ACTION_TOKENS) if not (has_io_signal and has_action_signal): warnings.append("mechanism_flow_too_abstract") @@ -1526,9 +1639,11 @@ def strip_frontmatter(text: str) -> str: def main() -> None: - from common import emit + from common import emit, runtime_config args = parser().parse_args() + config = runtime_config(cli_overrides={"output_language": args.language}) + output_language = configure_output_language(str(config["output_language"])) path = Path(args.input).expanduser().resolve() # utf-8-sig strips a leading BOM and the replace() normalizes CRLF so # Windows-authored notes are linted identically to LF/BOM-less notes; @@ -1548,7 +1663,8 @@ def main() -> None: reference_hygiene_issues = inspect_reference_hygiene(text) substantive_issues = inspect_substantive_content(text) planning_artifact_found, planning_artifact_issues = inspect_note_plan( - resolve_note_plan_path(path, args.plan_file) + resolve_note_plan_path(path, args.plan_file), + output_language, ) warnings.extend(inspect_figure_callouts(text)) for issue in figure_issues: @@ -1567,6 +1683,7 @@ def main() -> None: if reason and reason not in warnings: warnings.append(reason) warnings.extend(front_matter_order_warnings(text)) + warnings.extend(english_top_level_section_warnings(text)) warnings.extend(mechanism_flow_warnings(text)) if not body_text.lstrip().startswith("# "): warnings.append("title_heading_missing") @@ -1596,8 +1713,10 @@ def main() -> None: payload = { "status": "ok", "script": "lint_note.py", + "output_language": output_language, "paper_id": args.paper_id, "input_path": str(path), + "note_sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), "headers": headers, "missing_sections": missing_sections, "warnings": warnings, @@ -1619,6 +1738,8 @@ def main() -> None: "title_heading_missing", "no_level2_sections", "front_matter_order_invalid", + "top_level_section_profile_invalid", + "mechanism_flow_heading_invalid", } & set(warnings) ), diff --git a/skills/deeppapernote/scripts/localization.py b/skills/deeppapernote/scripts/localization.py new file mode 100644 index 0000000..a73b028 --- /dev/null +++ b/skills/deeppapernote/scripts/localization.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Language schemas shared by DeepPaperNote contracts and validators.""" + +from __future__ import annotations + +import os +from copy import deepcopy +from typing import Any + +DEFAULT_OUTPUT_LANGUAGE = "zh-CN" +SUPPORTED_OUTPUT_LANGUAGES = ("zh-CN", "en") +_ALIASES = {"zh": "zh-CN", "zh-cn": "zh-CN", "zh_cn": "zh-CN", "chinese": "zh-CN", "en": "en", "en-us": "en", "en_us": "en", "english": "en"} +_SCHEMAS: dict[str, dict[str, Any]] = { + "zh-CN": { + "sections": {"core_information": "核心信息", "abstract": "原文摘要翻译", "contributions": "创新点", "one_sentence_summary": "一句话总结", "research_questions": "研究问题", "data_and_task": "数据与任务定义", "method": "方法主线", "key_results": "关键结果", "deep_analysis": "深度分析", "limitations": "局限", "my_notes": "我的笔记", "references": "引用"}, + "core_info_fields": ("标题", "标题翻译", "作者", "机构", "发表时间", "发表渠道", "DOI", "arXiv", "论文链接", "代码 / 项目", "数据 / 资源", "论文类型"), + "core_info_aliases": {}, + "figure_labels": {"location": "建议位置：", "reason": "放置原因：", "status": "当前状态：", "original_caption": "论文原图编号："}, + "mechanism_flow": "机制流程", + }, + "en": { + "sections": {"core_information": "Core Information", "abstract": "Abstract", "contributions": "Contributions", "one_sentence_summary": "One-Sentence Summary", "research_questions": "Research Question", "data_and_task": "Data and Task Definition", "method": "Method", "key_results": "Key Results", "deep_analysis": "Deep Analysis", "limitations": "Limitations", "my_notes": "Research Notes", "references": "References"}, + "core_info_fields": ("Title", "Translated title", "Authors", "Institutions", "Publication date", "Venue", "DOI", "arXiv", "Paper link", "Code / Project", "Data / Resources", "Paper type"), + "core_info_aliases": {}, + "figure_labels": {"location": "Suggested location:", "reason": "Why it matters:", "status": "Current status:", "original_caption": "Original paper item:"}, + "mechanism_flow": "Mechanism Flow", + "abstract_contract": { + "source": "source_abstract", + "requirement": "faithful_rendering_in_output_language", + "forbidden_additions": [ + "later_contribution_claims", + "later_result_interpretation", + "hindsight_judgment", + ], + }, + }, +} + +def normalize_output_language(value: str | None = None) -> str: + raw = (value if value is not None else os.environ.get("DEEPPAPERNOTE_OUTPUT_LANGUAGE", "")).strip() + if not raw: + return DEFAULT_OUTPUT_LANGUAGE + normalized = _ALIASES.get(raw.lower(), raw) + if normalized not in SUPPORTED_OUTPUT_LANGUAGES: + raise ValueError(f"Unsupported DeepPaperNote output language: {raw}. Choose one of: {', '.join(SUPPORTED_OUTPUT_LANGUAGES)}.") + return normalized + + +def require_artifact_output_language( + artifact: dict[str, Any], + artifact_name: str, + expected: str, +) -> str: + language = artifact.get("output_language") + if language not in SUPPORTED_OUTPUT_LANGUAGES: + raise ValueError( + f"{artifact_name} requires output_language with one of: " + f"{', '.join(SUPPORTED_OUTPUT_LANGUAGES)}." + ) + resolved = normalize_output_language(expected) + if language != resolved: + raise ValueError( + f"{artifact_name} output_language {language} does not match " + f"resolved output_language {resolved}." + ) + return str(language) + +def note_schema(language: str | None = None) -> dict[str, Any]: + return deepcopy(_SCHEMAS[normalize_output_language(language)]) + +def required_sections(language: str | None = None) -> tuple[str, ...]: + return tuple(note_schema(language)["sections"].values()) diff --git a/skills/deeppapernote/scripts/materialize_figure_asset.py b/skills/deeppapernote/scripts/materialize_figure_asset.py index 787a8c8..c4b087b 100644 --- a/skills/deeppapernote/scripts/materialize_figure_asset.py +++ b/skills/deeppapernote/scripts/materialize_figure_asset.py @@ -25,6 +25,8 @@ def parser() -> argparse.ArgumentParser: p.add_argument("--input", default="", help="Optional metadata JSON path or JSON string.") p.add_argument("--title", default="", help="Explicit paper title.") p.add_argument("--vault", default="", help="Target Obsidian vault path.") + p.add_argument("--save-mode", choices=("workspace", "obsidian"), default="") + p.add_argument("--papers-dir", default="", help="Vault-relative paper directory.") p.add_argument("--subdir", default="", help="Vault-relative note subdirectory.") p.add_argument("--filename", default="", help="Optional note filename override.") p.add_argument("--asset-subdir", default="images", help="Asset folder name relative to the note directory.") @@ -40,9 +42,13 @@ def main() -> None: if not title: raise SystemExit("materialize_figure_asset.py requires --title or metadata with a title.") - config = runtime_config() - if args.vault: - config["obsidian_vault"] = args.vault + config = runtime_config( + cli_overrides={ + "save_mode": args.save_mode or ("obsidian" if args.vault else ""), + "obsidian_vault": args.vault, + "papers_dir": args.papers_dir, + } + ) resolved_subdir = resolve_domain_subdir( config, title=title, diff --git a/skills/deeppapernote/scripts/plan_figure_table_decisions.py b/skills/deeppapernote/scripts/plan_figure_table_decisions.py index ab824b0..9c25338 100644 --- a/skills/deeppapernote/scripts/plan_figure_table_decisions.py +++ b/skills/deeppapernote/scripts/plan_figure_table_decisions.py @@ -15,9 +15,11 @@ file_sha256, maybe_load_json_record, normalize_whitespace, + runtime_config, ) from contracts import WRITING_CONTRACT_RULES from extract_pdf_assets import _render_crop, save_image_bytes +from localization import normalize_output_language, require_artifact_output_language from source_corpus import SourceCorpusLoadError, load_source_corpus DECISION_VALUES = set(WRITING_CONTRACT_RULES["figure_decision_values"]) @@ -39,6 +41,11 @@ def parser() -> argparse.ArgumentParser: ) p.add_argument("--output", default="", help="Output JSON path.") p.add_argument("--paper-id", default="", help="Canonical paper id.") + p.add_argument( + "--language", + default="", + help="Run Override for output language: en or zh-CN.", + ) return p @@ -475,8 +482,13 @@ def build_decisions( def main() -> None: args = parser().parse_args() + language = normalize_output_language( + runtime_config(cli_overrides={"output_language": args.language})["output_language"] + ) if args.review_decisions: - payload = apply_requested_repairs(load_record(args.review_decisions)) + payload = load_record(args.review_decisions) + require_artifact_output_language(payload, "Figure/Table Decisions", language) + payload = apply_requested_repairs(payload) payload["status"] = "ok" payload["script"] = "plan_figure_table_decisions.py" emit(payload, args.output) @@ -488,6 +500,7 @@ def main() -> None: ) source_manifest = load_record(args.source_manifest) figures = load_record(args.figures) if args.figures else {} + require_artifact_output_language(figures, "Figure Plan", language) assets = load_record(args.assets) if args.assets else {} decisions = build_decisions( source_manifest, @@ -498,6 +511,7 @@ def main() -> None: payload = { "status": "ok", "script": "plan_figure_table_decisions.py", + "output_language": language, "paper_id": args.paper_id or source_manifest.get("paper_id", ""), "decisions": decisions, "summary": { diff --git a/skills/deeppapernote/scripts/plan_figures.py b/skills/deeppapernote/scripts/plan_figures.py index c508550..039e3f9 100644 --- a/skills/deeppapernote/scripts/plan_figures.py +++ b/skills/deeppapernote/scripts/plan_figures.py @@ -6,7 +6,8 @@ import argparse import re -from common import caption_preference_score, maybe_load_json_record, normalize_whitespace +from common import caption_preference_score, maybe_load_json_record, normalize_whitespace, runtime_config +from localization import normalize_output_language def parser() -> argparse.ArgumentParser: @@ -16,6 +17,7 @@ def parser() -> argparse.ArgumentParser: p.add_argument("--assets", default="", help="PDF assets JSON path or string.") p.add_argument("--output", default="", help="Output JSON path.") p.add_argument("--paper-id", default="", help="Canonical paper id.") + p.add_argument("--language", default="", help="Output language: en or zh-CN.") p.add_argument("--max-items", type=int, default=12, help="Maximum number of figure/table items to keep. 0 means keep all.") return p @@ -34,7 +36,7 @@ def merge_inputs(primary: dict | None, evidence: dict | None, assets: dict | Non return merged -def classify_caption_kind(item_id: str, caption: str) -> tuple[str, str, str]: +def _classify_caption_kind_zh(item_id: str, caption: str) -> tuple[str, str, str]: text = f"{item_id} {caption}".lower() if any( token in text @@ -148,7 +150,27 @@ def classify_caption_kind(item_id: str, caption: str) -> tuple[str, str, str]: return "supporting_figure", "深度分析", "这张图更适合作为补充图，放在深度分析部分帮助解释作者论点。" -def build_figure_items(evidence_pack: dict, *, limit: int = 12) -> list[dict]: +ENGLISH_FIGURE_PLACEMENT: dict[str, tuple[str, str]] = { + "main_result": ("Key Results", "This figure or table carries a primary result and belongs in Key Results."), + "data_or_task_overview": ("Data and Task Definition", "This visual explains the source, construction, screening, or scope of the data and task."), + "method_overview": ("Mechanism Flow", "This visual summarizes the method or system flow and belongs in Mechanism Flow when the match is reliable."), + "data_or_task": ("Data and Task Definition", "This visual clarifies the task setting, sample, or dataset."), + "method_detail": ("Method", "This visual explains an internal mechanism or execution state and belongs in Method."), + "table_result": ("Key Results", "This result table helps readers locate the central quantitative evidence."), + "supporting_figure": ("Deep Analysis", "This supporting visual helps explain the authors' argument in Deep Analysis."), +} + + +def classify_caption_kind(item_id: str, caption: str, language: str | None = None) -> tuple[str, str, str]: + result = _classify_caption_kind_zh(item_id, caption) + if normalize_output_language(language) != "en": + return result + kind = result[0] + section, reason = ENGLISH_FIGURE_PLACEMENT[kind] + return kind, section, reason + + +def build_figure_items(evidence_pack: dict, *, limit: int = 12, language: str | None = None) -> list[dict]: raw_items = [] for item in evidence_pack.get("figure_captions", []) or []: if isinstance(item, dict): @@ -185,7 +207,7 @@ def build_figure_items(evidence_pack: dict, *, limit: int = 12) -> list[dict]: item = grouped[key] item_id = normalize_whitespace(str(item.get("id", ""))) caption = normalize_whitespace(str(item.get("caption", ""))) - kind, section, reason = classify_caption_kind(item_id, caption) + kind, section, reason = classify_caption_kind(item_id, caption, language) priority = 3 if kind == "method_overview": priority = 1 @@ -489,12 +511,16 @@ def main() -> None: page_assets = data.get("page_assets", []) if isinstance(data.get("page_assets"), list) else [] image_assets = data.get("image_assets", []) if isinstance(data.get("image_assets"), list) else [] figure_assets = data.get("figure_assets", []) if isinstance(data.get("figure_assets"), list) else [] - items = build_figure_items(evidence_pack, limit=args.max_items) + language = normalize_output_language( + runtime_config(cli_overrides={"output_language": args.language})["output_language"] + ) + items = build_figure_items(evidence_pack, limit=args.max_items, language=language) items = attach_candidate_images(items, page_assets, image_assets, figure_assets) payload = { "status": "ok", "script": "plan_figures.py", "paper_id": args.paper_id or data.get("paper_id", ""), + "output_language": language, "figure_plan": { "paper_id": args.paper_id or data.get("paper_id", ""), "figures": items, diff --git a/skills/deeppapernote/scripts/run_pipeline.py b/skills/deeppapernote/scripts/run_pipeline.py index 40a0780..cadc46b 100644 --- a/skills/deeppapernote/scripts/run_pipeline.py +++ b/skills/deeppapernote/scripts/run_pipeline.py @@ -4,10 +4,13 @@ from __future__ import annotations import argparse +import os import subprocess import sys from pathlib import Path +from common import runtime_config + def parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__ or "run pipeline") @@ -28,15 +31,48 @@ def parser() -> argparse.ArgumentParser: default="auto", help="Local Zotero lookup policy used by the resolve stage.", ) + p.add_argument( + "--language", + default="", + choices=("", "en", "zh-CN"), + help="Run Override for the output language contract.", + ) + p.add_argument("--save-mode", choices=("workspace", "obsidian"), default="") + p.add_argument("--vault", default="", help="Run Override for the Obsidian Vault.") + p.add_argument("--papers-dir", default="", help="Run Override for the Vault paper directory.") return p -def run_step(cmd: list[str]) -> None: - subprocess.run(cmd, check=True) +def run_step(cmd: list[str], *, env: dict[str, str]) -> None: + subprocess.run(cmd, check=True, env=env) def main() -> None: args = parser().parse_args() + try: + config = runtime_config( + cli_overrides={ + "output_language": args.language, + "save_mode": args.save_mode, + "obsidian_vault": args.vault, + "papers_dir": args.papers_dir, + } + ) + except RuntimeError as exc: + raise SystemExit(str(exc)) from exc + args.language = config["output_language"] + run_environment = os.environ.copy() + for field, name in { + "output_language": "DEEPPAPERNOTE_OUTPUT_LANGUAGE", + "save_mode": "DEEPPAPERNOTE_SAVE_MODE", + "obsidian_vault": "DEEPPAPERNOTE_OBSIDIAN_VAULT", + "papers_dir": "DEEPPAPERNOTE_PAPERS_DIR", + }.items(): + value = str(config.get(field, "")).strip() + if value: + run_environment[name] = value + else: + run_environment.pop(name, None) scripts_dir = Path(__file__).resolve().parent workdir = Path(args.workdir).expanduser().resolve() workdir.mkdir(parents=True, exist_ok=True) @@ -46,11 +82,13 @@ def main() -> None: identity_json = workdir / f"{args.prefix}_identity.json" identity_trace_json = workdir / f"{args.prefix}_identity_repair_trace.json" fetch_json = workdir / f"{args.prefix}_fetch.json" + pdf_dir = workdir / f"{args.prefix}_pdfs" source_manifest_json = workdir / f"{args.prefix}_source_manifest.json" raw_sections_jsonl = workdir / f"{args.prefix}_raw_sections.jsonl" full_text_md = workdir / f"{args.prefix}_full_text.md" evidence_json = workdir / f"{args.prefix}_evidence.json" assets_json = workdir / f"{args.prefix}_assets.json" + assets_dir = workdir / f"{args.prefix}_assets" figures_json = workdir / f"{args.prefix}_figures.json" figure_decisions_json = workdir / f"{args.prefix}_figure_table_decisions.json" bundle_json = workdir / f"{args.prefix}_bundle.json" @@ -65,7 +103,8 @@ def main() -> None: args.zotero_mode, "--output", str(resolve_json), - ] + ], + env=run_environment, ) run_step( [ @@ -75,7 +114,8 @@ def main() -> None: str(resolve_json), "--output", str(metadata_json), - ] + ], + env=run_environment, ) run_step( [ @@ -89,7 +129,8 @@ def main() -> None: str(identity_trace_json), "--output", str(identity_json), - ] + ], + env=run_environment, ) run_step( [ @@ -99,9 +140,12 @@ def main() -> None: str(metadata_json), "--identity", str(identity_json), + "--dest-dir", + str(pdf_dir), "--output", str(fetch_json), - ] + ], + env=run_environment, ) run_step( [ @@ -115,7 +159,8 @@ def main() -> None: str(raw_sections_jsonl), "--full-text-output", str(full_text_md), - ] + ], + env=run_environment, ) run_step( [ @@ -127,7 +172,8 @@ def main() -> None: str(source_manifest_json), "--output", str(evidence_json), - ] + ], + env=run_environment, ) run_step( [ @@ -135,9 +181,12 @@ def main() -> None: str(scripts_dir / "extract_pdf_assets.py"), "--input", str(fetch_json), + "--assets-dir", + str(assets_dir), "--output", str(assets_json), - ] + ], + env=run_environment, ) run_step( [ @@ -147,9 +196,12 @@ def main() -> None: str(evidence_json), "--assets", str(assets_json), + "--language", + args.language, "--output", str(figures_json), - ] + ], + env=run_environment, ) run_step( [ @@ -161,12 +213,14 @@ def main() -> None: str(figures_json), "--assets", str(assets_json), + "--language", + args.language, "--output", str(figure_decisions_json), - ] + ], + env=run_environment, ) - run_step( - [ + bundle_command = [ py, str(scripts_dir / "build_synthesis_bundle.py"), "--metadata", @@ -184,7 +238,9 @@ def main() -> None: "--output", str(bundle_json), ] - ) + if args.language: + bundle_command.extend(["--language", args.language]) + run_step(bundle_command, env=run_environment) print( "\n".join( diff --git a/skills/deeppapernote/scripts/user_configuration.py b/skills/deeppapernote/scripts/user_configuration.py new file mode 100644 index 0000000..8e09dbf --- /dev/null +++ b/skills/deeppapernote/scripts/user_configuration.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Inspect and persist DeepPaperNote's device-local User Configuration.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import tempfile +from datetime import datetime, timezone +from pathlib import Path, PureWindowsPath +from typing import Any, Mapping, Sequence + +OUTPUT_LANGUAGES = {"zh-CN", "en"} +SAVE_MODES = {"workspace", "obsidian"} +KNOWN_FIELDS = ("output_language", "save_mode", "obsidian_vault", "papers_dir") +ALWAYS_REQUIRED = ("output_language", "save_mode") +ENV_FIELDS = { + "output_language": "DEEPPAPERNOTE_OUTPUT_LANGUAGE", + "save_mode": "DEEPPAPERNOTE_SAVE_MODE", + "obsidian_vault": "DEEPPAPERNOTE_OBSIDIAN_VAULT", + "papers_dir": "DEEPPAPERNOTE_PAPERS_DIR", +} + + +class ConfigurationWriteError(RuntimeError): + """Raised when a Preference Change cannot be durably verified.""" + + +class ConfigurationValidationError(ConfigurationWriteError): + """Raised when a Preference Change still needs user input or repair.""" + + def __init__( + self, + result: dict[str, Any], + message: str = "Preference Change is incomplete or invalid.", + ) -> None: + super().__init__(message) + self.result = result + + +def user_config_path() -> Path: + override = os.environ.get("DEEPPAPERNOTE_CONFIG_PATH", "").strip() + if override: + return Path(override).expanduser() + return Path.home() / ".deeppapernote" / "config.json" + + +def default_shell_paths() -> tuple[Path, ...]: + home = Path.home() + return tuple( + home / name + for name in (".zshenv", ".zprofile", ".zshrc", ".bash_profile", ".bashrc") + ) + + +def _clean_values(values: Mapping[str, Any] | None) -> dict[str, str]: + if not values: + return {} + return { + field: str(values[field]).strip() + for field in KNOWN_FIELDS + if field in values and str(values[field]).strip() + } + + +def _migration_candidates( + environ: Mapping[str, str], shell_paths: Sequence[Path] +) -> dict[str, dict[str, str]]: + candidates: dict[str, dict[str, str]] = {} + for field, name in ENV_FIELDS.items(): + value = environ.get(name, "").strip() + if value and _migration_value_supported(field, value): + candidates[field] = {"value": value, "source": "process_environment"} + for path in shell_paths: + if not path.is_file(): + continue + try: + lines = path.read_text(encoding="utf-8-sig").splitlines() + except OSError: + continue + for field, name in ENV_FIELDS.items(): + if field in candidates: + continue + pattern = re.compile(rf"^\s*(?:export\s+)?{re.escape(name)}=(.*)$") + for line in reversed(lines): + match = pattern.match(line) + if not match: + continue + value = match.group(1).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: + value = value[1:-1] + if value and _migration_value_supported(field, value): + candidates[field] = {"value": value, "source": str(path)} + break + return candidates + + +def _issue(field: str, code: str, message: str) -> dict[str, str]: + return {"field": field, "code": code, "message": message} + + +def _safe_relative_path(value: str) -> bool: + native = Path(value) + windows = PureWindowsPath(value) + if native.is_absolute() or windows.is_absolute() or windows.drive or windows.root: + return False + parts = tuple(part for part in re.split(r"[\\/]+", value) if part and part != ".") + return bool(parts) and ".." not in parts + + +def _migration_value_supported(field: str, value: str) -> bool: + if field == "output_language": + return value in OUTPUT_LANGUAGES + if field == "save_mode": + return value in SAVE_MODES + if field == "obsidian_vault": + try: + vault = Path(value).expanduser() + except (OSError, RuntimeError): + return False + return vault.is_absolute() and vault.is_dir() + if field == "papers_dir": + return _safe_relative_path(value) + return False + + +def _validate(configuration: Mapping[str, Any]) -> tuple[list[dict[str, str]], list[str]]: + issues: list[dict[str, str]] = [] + missing: list[str] = [] + for field in ALWAYS_REQUIRED: + if not str(configuration.get(field, "")).strip(): + missing.append(field) + language = str(configuration.get("output_language", "")).strip() + save_mode = str(configuration.get("save_mode", "")).strip() + if language and language not in OUTPUT_LANGUAGES: + issues.append(_issue("output_language", "invalid_enum", "Expected zh-CN or en.")) + if save_mode and save_mode not in SAVE_MODES: + issues.append(_issue("save_mode", "invalid_enum", "Expected workspace or obsidian.")) + if save_mode == "obsidian": + vault = str(configuration.get("obsidian_vault", "")).strip() + papers_dir = str(configuration.get("papers_dir", "")).strip() + if not vault: + missing.append("obsidian_vault") + else: + vault_path = Path(vault).expanduser() + if not vault_path.is_absolute() or not vault_path.is_dir(): + issues.append( + _issue( + "obsidian_vault", + "missing_vault", + "Expected an existing absolute directory.", + ) + ) + if not papers_dir: + missing.append("papers_dir") + elif not _safe_relative_path(papers_dir): + issues.append( + _issue( + "papers_dir", + "unsafe_path", + "Expected a safe relative path inside the Vault.", + ) + ) + return issues, missing + + +def _blocked_result(path: Path, code: str, message: str) -> dict[str, Any]: + return { + "state": "blocked", + "config_path": str(path), + "affected_fields": ["configuration"], + "prompt_fields": [], + "migration_candidates": {}, + "warnings": [], + "issues": [_issue("configuration", code, message)], + } + + +def _repair_result( + path: Path, + configuration: Mapping[str, Any], + issues: list[dict[str, str]], + missing: list[str], +) -> dict[str, Any]: + affected = [issue["field"] for issue in issues] + for field in missing: + if field not in affected: + affected.append(field) + result: dict[str, Any] = { + "state": "invalid" if issues else "needs_input" if missing else "ready", + "config_path": str(path), + "affected_fields": affected, + "prompt_fields": [field for field in KNOWN_FIELDS if field in affected], + "migration_candidates": {}, + "warnings": [], + "configuration": dict(configuration), + } + if issues: + result["issues"] = issues + return result + + +def _read_configuration(path: Path) -> tuple[dict[str, Any] | None, str]: + try: + raw = path.read_text(encoding="utf-8-sig") + except OSError as exc: + return None, f"unreadable:{exc}" + try: + value = json.loads(raw) + except (json.JSONDecodeError, UnicodeError) as exc: + return None, f"malformed:{exc}" + if not isinstance(value, dict): + return None, "non_object:Expected a JSON object." + return value, "" + + +def _path_is_writable(path: Path) -> bool: + candidate = path.parent + while not candidate.exists() and candidate != candidate.parent: + candidate = candidate.parent + return os.access(candidate, os.W_OK) + + +def inspect_configuration( + *, + config_path: str | Path | None = None, + environ: Mapping[str, str] | None = None, + shell_paths: Sequence[Path] | None = None, +) -> dict[str, Any]: + path = Path(config_path or user_config_path()).expanduser() + environment = os.environ if environ is None else environ + if not _path_is_writable(path): + return _blocked_result(path, "unwritable", "User Configuration is not writable.") + if not path.exists(): + return { + "state": "needs_input", + "config_path": str(path), + "affected_fields": list(ALWAYS_REQUIRED), + "prompt_fields": list(ALWAYS_REQUIRED), + "migration_candidates": _migration_candidates( + environment, default_shell_paths() if shell_paths is None else shell_paths + ), + "warnings": [], + } + + configuration, read_error = _read_configuration(path) + if read_error.startswith("unreadable:"): + return _blocked_result(path, "unreadable", read_error.partition(":")[2]) + if read_error: + code, _, message = read_error.partition(":") + return { + "state": "invalid", + "config_path": str(path), + "affected_fields": ["configuration"], + "prompt_fields": list(ALWAYS_REQUIRED), + "migration_candidates": {}, + "warnings": [], + "issues": [_issue("configuration", code, message)], + } + + assert configuration is not None + issues, missing = _validate(configuration) + return _repair_result(path, configuration, issues, missing) + + +def resolve_preferences( + *, + config_path: str | Path | None = None, + explicit_overrides: Mapping[str, Any] | None = None, + cli_overrides: Mapping[str, Any] | None = None, + environ: Mapping[str, str] | None = None, +) -> dict[str, Any]: + path = Path(config_path or user_config_path()).expanduser() + configuration, read_error = _read_configuration(path) + if read_error or configuration is None: + raise RuntimeError(f"User Configuration is not readable and valid: {read_error}") + resolved = resolve_run_overrides( + explicit_overrides=explicit_overrides, + cli_overrides=cli_overrides, + environ=environ, + ) + values = _clean_values(configuration) + sources = {field: "user_configuration" for field in values} + values.update(resolved["values"]) + sources.update(resolved["sources"]) + issues, missing = _validate(values) + return {"values": values, "sources": sources, "issues": issues, "missing": missing} + + +def resolve_run_overrides( + *, + explicit_overrides: Mapping[str, Any] | None = None, + cli_overrides: Mapping[str, Any] | None = None, + environ: Mapping[str, str] | None = None, +) -> dict[str, Any]: + environment = os.environ if environ is None else environ + layers = ( + ( + "process_environment", + { + field: environment.get(name, "").strip() + for field, name in ENV_FIELDS.items() + if environment.get(name, "").strip() + }, + ), + ("cli", _clean_values(cli_overrides)), + ("explicit_request", _clean_values(explicit_overrides)), + ) + values: dict[str, str] = {} + sources: dict[str, str] = {} + for source, layer in layers: + for field, value in layer.items(): + values[field] = value + sources[field] = source + issues, missing = _validate(values) + return {"values": values, "sources": sources, "issues": issues, "missing": missing} + + +def _invalid_backup_path(path: Path) -> Path: + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + candidate = path.with_name(f"{path.stem}.invalid-{stamp}{path.suffix}") + counter = 1 + while candidate.exists(): + candidate = path.with_name(f"{path.stem}.invalid-{stamp}-{counter}{path.suffix}") + counter += 1 + return candidate + + +def _atomic_write(path: Path, configuration: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_name = "" + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) as handle: + temporary_name = handle.name + json.dump(configuration, handle, ensure_ascii=False, indent=2) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary_name, 0o600) + os.replace(temporary_name, path) + except OSError as exc: + if temporary_name: + try: + Path(temporary_name).unlink() + except OSError: + pass + raise ConfigurationWriteError(f"User Configuration write is blocked: {exc}") from exc + + +def persist_preferences( + preferences: Mapping[str, Any], + *, + config_path: str | Path | None = None, + environ: Mapping[str, str] | None = None, + replace_invalid: bool = False, +) -> dict[str, Any]: + path = Path(config_path or user_config_path()).expanduser() + if not _path_is_writable(path): + raise ConfigurationWriteError("User Configuration is not writable.") + existing: dict[str, Any] = {} + if path.exists(): + loaded, read_error = _read_configuration(path) + if read_error.startswith("unreadable:"): + raise ConfigurationWriteError(f"User Configuration is unreadable: {read_error}") + if read_error: + if not replace_invalid: + raise ConfigurationValidationError( + inspect_configuration(config_path=path, environ=environ), + "Invalid User Configuration requires an explicitly confirmed replacement.", + ) + backup = _invalid_backup_path(path) + try: + shutil.copy2(path, backup) + except OSError as exc: + raise ConfigurationWriteError( + f"Could not preserve invalid User Configuration: {exc}" + ) from exc + else: + assert loaded is not None + existing = loaded + updates = _clean_values(preferences) + candidate = {**existing, **updates} + issues, missing = _validate(candidate) + if issues or missing: + raise ConfigurationValidationError(_repair_result(path, candidate, issues, missing)) + unknown = sorted(set(existing) - set(KNOWN_FIELDS)) + _atomic_write(path, candidate) + reread, read_error = _read_configuration(path) + if read_error or reread != candidate: + raise ConfigurationWriteError( + "User Configuration readback did not match the persisted values." + ) + result = inspect_configuration(config_path=path, environ=environ) + if result["state"] != "ready": + raise ConfigurationWriteError("Persisted User Configuration did not pass validation.") + if unknown: + result["warnings"] = [ + f"Preserved unknown configuration fields: {', '.join(unknown)}" + ] + return result + + +def parser() -> argparse.ArgumentParser: + argument_parser = argparse.ArgumentParser(description=__doc__) + argument_parser.add_argument("--set-output-language", choices=sorted(OUTPUT_LANGUAGES)) + argument_parser.add_argument("--set-save-mode", choices=sorted(SAVE_MODES)) + argument_parser.add_argument("--set-vault") + argument_parser.add_argument("--set-papers-dir") + argument_parser.add_argument("--replace-invalid", action="store_true") + return argument_parser + + +def main() -> None: + args = parser().parse_args() + preferences = { + field: value + for field, value in { + "output_language": args.set_output_language, + "save_mode": args.set_save_mode, + "obsidian_vault": args.set_vault, + "papers_dir": args.set_papers_dir, + }.items() + if value is not None + } + try: + result = ( + persist_preferences(preferences, replace_invalid=args.replace_invalid) + if preferences + else inspect_configuration() + ) + except ConfigurationValidationError as exc: + result = exc.result + except ConfigurationWriteError as exc: + result = _blocked_result(user_config_path(), "write_failed", str(exc)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if result["state"] != "ready": + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/skills/deeppapernote/scripts/write_obsidian_note.py b/skills/deeppapernote/scripts/write_obsidian_note.py index e220f0b..0cb3790 100644 --- a/skills/deeppapernote/scripts/write_obsidian_note.py +++ b/skills/deeppapernote/scripts/write_obsidian_note.py @@ -4,10 +4,13 @@ from __future__ import annotations import argparse +import hashlib import json +import os import re import shutil import sys +import tempfile from pathlib import Path from common import ( @@ -22,6 +25,57 @@ runtime_config, ) from lint_note import inspect_reference_hygiene +from localization import normalize_output_language, require_artifact_output_language + +PAPER_DIRECTORY_SIDECAR = ".deeppapernote.json" +SOURCE_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +WINDOWS_FILE_ATTRIBUTE_HIDDEN = 0x2 +WINDOWS_INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF + + +def ensure_sidecar_hidden( + path: Path, + *, + platform: str | None = None, + get_attributes=None, + set_attributes=None, +) -> None: + if (platform or os.name) != "nt": + return + if (get_attributes is None) != (set_attributes is None): + raise ValueError("get_attributes and set_attributes must be provided together") + + if get_attributes is None: + import ctypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + get_attributes = kernel32.GetFileAttributesW + get_attributes.argtypes = [ctypes.c_wchar_p] + get_attributes.restype = ctypes.c_uint32 + set_attributes = kernel32.SetFileAttributesW + set_attributes.argtypes = [ctypes.c_wchar_p, ctypes.c_uint32] + set_attributes.restype = ctypes.c_int + + def fail() -> None: + raise ctypes.WinError(ctypes.get_last_error()) + + else: + + def fail() -> None: + raise OSError(f"Could not set Windows Hidden attribute: {path}") + + path_value = str(path) + current = int(get_attributes(path_value)) + if current == WINDOWS_INVALID_FILE_ATTRIBUTES: + fail() + if not set_attributes(path_value, current | WINDOWS_FILE_ATTRIBUTE_HIDDEN): + fail() + verified = int(get_attributes(path_value)) + if ( + verified == WINDOWS_INVALID_FILE_ATTRIBUTES + or not verified & WINDOWS_FILE_ATTRIBUTE_HIDDEN + ): + fail() def parser() -> argparse.ArgumentParser: @@ -30,19 +84,46 @@ def parser() -> argparse.ArgumentParser: p.add_argument("--content-file", default="", help="Path to the final Markdown content.") p.add_argument("--content", default="", help="Inline Markdown content.") p.add_argument("--stdin", action="store_true", help="Read Markdown content from stdin.") - p.add_argument("--lint-json", default="", help="Optional lint JSON path. Refuse write if structure, style, or math gate failed.") + p.add_argument( + "--lint-json", + default="", + help="Required Final Note Lint JSON for Formal Save.", + ) p.add_argument( "--figure-decisions", default="", - help="Optional figure/table decisions JSON. Insert decisions must have referenced materialized images.", + help="Required Figure/Table Decisions JSON for Formal Save.", + ) + p.add_argument( + "--source-manifest", + default="", + help="Required Source Manifest JSON for Obsidian Save Target Admission.", ) p.add_argument("--title", default="", help="Explicit title override.") p.add_argument("--output", default="", help="JSON status output path.") p.add_argument("--vault", default="", help="Target Obsidian vault path.") + p.add_argument("--save-mode", choices=("workspace", "obsidian"), default="") + p.add_argument("--papers-dir", default="", help="Vault-relative paper directory.") p.add_argument("--subdir", default="", help="Vault-relative subdirectory.") p.add_argument("--filename", default="", help="Explicit note filename.") p.add_argument("--asset-subdir", default="images", help="Asset folder name relative to the note directory.") p.add_argument("--paper-id", default="", help="Canonical paper id.") + p.add_argument("--language", default="", help="Run Override for output language: en or zh-CN.") + p.add_argument( + "--preflight", + action="store_true", + help="Resolve the Obsidian Save Target without writing files.", + ) + p.add_argument( + "--overwrite-existing-note", + action="store_true", + help="Overwrite a same-language note after explicit user confirmation.", + ) + p.add_argument( + "--expected-existing-note-sha256", + default="", + help="SHA-256 returned by the conflict that the user approved overwriting.", + ) return p @@ -106,9 +187,11 @@ def materialize_insert_decisions( target_path: Path, decisions: dict, asset_subdir: str, + created_paths: list[Path] | None = None, ) -> list[dict]: - materialized: list[dict] = [] asset_dir = target_path.parent / asset_subdir + pending: list[tuple[Path, Path, str, dict]] = [] + planned_hashes: dict[Path, str] = {} for item in insert_decisions(decisions): source_value = str(item.get("source_image_path", "")).strip() source_image = Path(source_value).expanduser() @@ -137,25 +220,58 @@ def materialize_insert_decisions( raise SystemExit( f"Insert decision for {label} is not referenced as an image embed: {expected_relative}." ) - asset_dir.mkdir(parents=True, exist_ok=True) dest_image = asset_dir / filename if dest_image.resolve().parent != asset_dir.resolve(): raise SystemExit(f"Unsafe figure image destination: {dest_image}") - if source_image.resolve() != dest_image.resolve(): + existing_planned_sha256 = planned_hashes.get(dest_image) + if existing_planned_sha256 and existing_planned_sha256 != current_sha256: + raise SystemExit( + f"Insert decisions assign different bytes to the same image: {filename}" + ) + planned_hashes[dest_image] = current_sha256 + if dest_image.is_file() and file_sha256(dest_image) != current_sha256: + raise SystemExit( + f"Refusing to save because an existing paper-local image has different bytes: " + f"{filename}" + ) + pending.append( + ( + source_image, + dest_image, + current_sha256, + { + "source_id": item.get("source_id") + or item.get("label") + or item.get("item_id") + or "", + "source_image": str(source_image.resolve()), + "dest_image_path": str(dest_image), + "relative_markdown_path": expected_relative, + "reviewed_asset_sha256": current_sha256, + }, + ) + ) + + if pending: + asset_dir.mkdir(parents=True, exist_ok=True) + materialized: list[dict] = [] + for source_image, dest_image, current_sha256, record in pending: + if dest_image.is_file(): + if file_sha256(dest_image) != current_sha256: + raise SystemExit( + f"Refusing to save because an existing paper-local image has different bytes: " + f"{dest_image.name}" + ) + elif source_image.resolve() != dest_image.resolve(): + if created_paths is not None: + created_paths.append(dest_image) shutil.copy2(source_image, dest_image) if file_sha256(dest_image) != current_sha256: raise SystemExit( - f"Materialized image bytes do not match the reviewed asset SHA-256: {filename}" + f"Materialized image bytes do not match the reviewed asset SHA-256: " + f"{dest_image.name}" ) - materialized.append( - { - "source_id": item.get("source_id") or item.get("label") or item.get("item_id") or "", - "source_image": str(source_image.resolve()), - "dest_image_path": str(dest_image), - "relative_markdown_path": expected_relative, - "reviewed_asset_sha256": current_sha256, - } - ) + materialized.append(record) return materialized @@ -195,30 +311,476 @@ def require_lint_gate(lint: dict, key: str, gate: str, lint_path: str) -> None: raise SystemExit(lint_failure_message(lint, gate, lint_path)) +def require_source_manifest(path_value: str) -> dict: + if not path_value: + raise SystemExit( + "Obsidian Save Target Admission requires a Source Manifest with source_sha256." + ) + manifest = maybe_load_json_record(path_value) + if manifest is None or str(manifest.get("status", "")).strip() != "ok": + raise SystemExit(f"Expected successful Source Manifest JSON: {path_value}") + source_sha256 = str(manifest.get("source_sha256", "")).strip().lower() + if not SOURCE_SHA256_RE.fullmatch(source_sha256): + raise SystemExit("Source Manifest requires a 64-character source_sha256.") + manifest["source_sha256"] = source_sha256 + return manifest + + +def language_note_path(target_path: Path, output_language: str) -> Path: + suffix = target_path.suffix or ".md" + stem = target_path.stem + if stem.endswith(f".{output_language}"): + return target_path + return target_path.with_name(f"{stem}.{output_language}{suffix}") + + +def same_name_directories(vault: Path, directory_name: str) -> list[Path]: + return sorted( + { + path.resolve() + for path in vault.rglob("*") + if ( + path.is_dir() + and path.name == directory_name + and path.resolve().is_relative_to(vault.resolve()) + ) + } + ) + + +def read_paper_directory_sidecar(path: Path) -> dict | None: + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError): + return None + if ( + not isinstance(payload, dict) + or payload.get("artifact_type") != "deeppapernote_paper_directory" + or payload.get("schema_version") != 1 + or not SOURCE_SHA256_RE.fullmatch(str(payload.get("source_sha256", ""))) + or not str(payload.get("note_stem", "")).strip() + or Path(str(payload.get("note_stem", ""))).name != payload.get("note_stem") + ): + return None + return payload + + +def source_directories(vault: Path, source_sha256: str) -> list[tuple[Path, dict]]: + matches: list[tuple[Path, dict]] = [] + for sidecar_path in vault.rglob(PAPER_DIRECTORY_SIDECAR): + if not sidecar_path.resolve().is_relative_to(vault.resolve()): + continue + payload = read_paper_directory_sidecar(sidecar_path) + if payload is not None and payload["source_sha256"] == source_sha256: + matches.append((sidecar_path.parent.resolve(), payload)) + return sorted(matches, key=lambda item: str(item[0])) + + +def block_directory_conflict( + args: argparse.Namespace, + *, + conflict_code: str, + target_directories: list[Path], + source_sha256: str, + output_language: str, +) -> None: + payload = { + "status": "blocked", + "script": "write_obsidian_note.py", + "conflict_code": conflict_code, + "requires_user_confirmation": False, + "target_directory": str(target_directories[0]), + "matching_directories": [str(path) for path in target_directories], + "source_sha256": source_sha256, + "output_language": output_language, + } + emit(payload, args.output) + raise SystemExit(2) + + +def block_note_conflict( + args: argparse.Namespace, + target_path: Path, + *, + conflict_code: str, + source_sha256: str, + output_language: str, +) -> None: + emit( + { + "status": "blocked", + "script": "write_obsidian_note.py", + "conflict_code": conflict_code, + "requires_user_confirmation": True, + "target_directory": str(target_path.parent), + "existing_note_path": str(target_path), + "existing_note_sha256": file_sha256(target_path), + "source_sha256": source_sha256, + "output_language": output_language, + }, + args.output, + ) + raise SystemExit(2) + + +def resolve_obsidian_save_target( + args: argparse.Namespace, + config: dict, + vault: Path, + *, + title: str, + abstract: str, + source_sha256: str, + output_language: str, +) -> tuple[Path, str, str, bool, str]: + source_matches = source_directories(vault, source_sha256) + if len(source_matches) > 1: + block_directory_conflict( + args, + conflict_code="multiple_source_directories", + target_directories=[path for path, _ in source_matches], + source_sha256=source_sha256, + output_language=output_language, + ) + + resolved_subdir = "" + recorded_language_target = False + if source_matches: + existing_dir, sidecar = source_matches[0] + notes = sidecar.get("notes", {}) + if not isinstance(notes, dict): + block_directory_conflict( + args, + conflict_code="invalid_language_note_record", + target_directories=[existing_dir], + source_sha256=source_sha256, + output_language=output_language, + ) + if output_language in notes: + note_record = notes[output_language] + filename = ( + str(note_record.get("filename", "")).strip() + if isinstance(note_record, dict) + else "" + ) + if ( + not filename + or filename in {".", ".."} + or "/" in filename + or "\\" in filename + or Path(filename).is_absolute() + or Path(filename).suffix.lower() != ".md" + ): + block_directory_conflict( + args, + conflict_code="invalid_language_note_record", + target_directories=[existing_dir], + source_sha256=source_sha256, + output_language=output_language, + ) + target_path = existing_dir / filename + recorded_language_target = True + if not target_path.is_file(): + block_directory_conflict( + args, + conflict_code="recorded_language_note_missing", + target_directories=[existing_dir], + source_sha256=source_sha256, + output_language=output_language, + ) + else: + target_path = existing_dir / f"{sidecar['note_stem']}.md" + admission = "reuse_source_directory" + domain_routing_skipped = True + else: + identity_path = resolve_obsidian_note_path( + config, + title=title, + filename=args.filename, + ) + name_matches = same_name_directories(vault, identity_path.parent.name) + if len(name_matches) > 1: + block_directory_conflict( + args, + conflict_code="multiple_same_name_directories", + target_directories=name_matches, + source_sha256=source_sha256, + output_language=output_language, + ) + if name_matches: + existing_dir = name_matches[0] + if any(existing_dir.iterdir()): + existing_sidecar = read_paper_directory_sidecar( + existing_dir / PAPER_DIRECTORY_SIDECAR + ) + block_directory_conflict( + args, + conflict_code=( + "same_name_different_source" + if existing_sidecar is not None + else "unidentified_same_name_directory" + ), + target_directories=name_matches, + source_sha256=source_sha256, + output_language=output_language, + ) + target_path = existing_dir / identity_path.name + admission = "reuse_empty_same_name_directory" + domain_routing_skipped = True + else: + resolved_subdir = resolve_domain_subdir( + config, + title=title, + abstract=abstract, + subdir=args.subdir, + ) + target_path = resolve_obsidian_note_path( + config, + title=title, + subdir=resolved_subdir, + filename=args.filename, + ) + admission = "new_directory" + domain_routing_skipped = False + + if not recorded_language_target: + target_path = language_note_path(target_path, output_language) + approved_existing_note_sha256 = "" + if target_path.is_file(): + existing_note_sha256 = file_sha256(target_path) + if not args.overwrite_existing_note: + block_note_conflict( + args, + target_path, + conflict_code="same_language_note_exists", + source_sha256=source_sha256, + output_language=output_language, + ) + expected_sha256 = args.expected_existing_note_sha256.strip().lower() + if not SOURCE_SHA256_RE.fullmatch(expected_sha256): + raise SystemExit( + "--overwrite-existing-note requires a 64-character " + "--expected-existing-note-sha256 from the conflict response." + ) + if existing_note_sha256 != expected_sha256: + block_note_conflict( + args, + target_path, + conflict_code="stale_overwrite_confirmation", + source_sha256=source_sha256, + output_language=output_language, + ) + approved_existing_note_sha256 = existing_note_sha256 + admission = "overwrite_same_language_note" + elif args.overwrite_existing_note: + raise SystemExit( + "--overwrite-existing-note is only valid when the target language note exists." + ) + + return ( + target_path, + resolved_subdir, + admission, + domain_routing_skipped, + approved_existing_note_sha256, + ) + + +def write_paper_directory_sidecar( + target_path: Path, + *, + paper_id: str, + title: str, + source_sha256: str, + output_language: str, + note_sha256: str, +) -> Path: + sidecar_path = target_path.parent / PAPER_DIRECTORY_SIDECAR + payload = read_paper_directory_sidecar(sidecar_path) or { + "artifact_type": "deeppapernote_paper_directory", + "schema_version": 1, + "paper_id": paper_id, + "title": title, + "source_sha256": source_sha256, + "note_stem": target_path.name.removesuffix(f".{output_language}{target_path.suffix}"), + "notes": {}, + } + if payload["source_sha256"] != source_sha256: + raise SystemExit(f"Paper directory sidecar source mismatch: {sidecar_path}") + notes = payload.get("notes") + if not isinstance(notes, dict): + notes = {} + payload["notes"] = notes + notes[output_language] = { + "filename": target_path.name, + "note_sha256": note_sha256, + } + fd, temp_name = tempfile.mkstemp( + dir=sidecar_path.parent, + prefix=f"{PAPER_DIRECTORY_SIDECAR}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + os.replace(temp_path, sidecar_path) + finally: + temp_path.unlink(missing_ok=True) + ensure_sidecar_hidden(sidecar_path) + return sidecar_path + + +def atomic_write_note(path: Path, note_text: str) -> None: + fd, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(note_text) + if path.is_file(): + shutil.copymode(path, temp_path) + os.replace(temp_path, path) + finally: + temp_path.unlink(missing_ok=True) + + +def backup_file(path: Path) -> Path | None: + if not path.is_file(): + return None + fd, backup_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".backup", + ) + os.close(fd) + backup_path = Path(backup_name) + try: + shutil.copy2(path, backup_path) + except BaseException: + backup_path.unlink(missing_ok=True) + raise + return backup_path + + +def restore_file(path: Path, backup_path: Path | None) -> None: + if backup_path is None: + path.unlink(missing_ok=True) + else: + os.replace(backup_path, path) + + def main() -> None: args = parser().parse_args() + config = runtime_config( + cli_overrides={ + "output_language": args.language, + "save_mode": args.save_mode or ("obsidian" if args.vault else ""), + "obsidian_vault": args.vault, + "papers_dir": args.papers_dir, + } + ) + output_language = normalize_output_language(args.language or str(config.get("output_language", "")) or None) + output_mode, root_path = resolve_note_output_mode(config) record = maybe_load_json_record(args.input) or {} title = args.title or str(record.get("title", "")).strip() if not title: raise SystemExit("write_obsidian_note.py requires --title or metadata with a title.") - if args.lint_json: - lint_path = str(Path(args.lint_json).expanduser().resolve()) - # utf-8-sig tolerates a BOM in the lint JSON (e.g. produced/edited on - # Windows) that would otherwise crash json.loads before any gate check. - lint = json.loads(Path(lint_path).read_text(encoding="utf-8-sig")) - require_lint_gate(lint, "passes_basic_structure", "basic structure", lint_path) - require_lint_gate(lint, "passes_style_gate", "style", lint_path) - require_lint_gate(lint, "passes_math_gate", "math", lint_path) - if "passes_figure_gate" in lint and not lint.get("passes_figure_gate", False): - raise SystemExit(lint_failure_message(lint, "figure", lint_path)) - if "passes_plan_gate" in lint and not lint.get("passes_plan_gate", False): - raise SystemExit(lint_failure_message(lint, "plan", lint_path)) - if "passes_substantive_content" in lint and not lint.get("passes_substantive_content", False): - raise SystemExit(lint_failure_message(lint, "substantive content", lint_path)) - if "passes_reference_hygiene_gate" in lint and not lint.get("passes_reference_hygiene_gate", False): - raise SystemExit(lint_failure_message(lint, "reference hygiene", lint_path)) + source_manifest = ( + require_source_manifest(args.source_manifest) if output_mode == "obsidian" else {} + ) + admission = "workspace" + domain_routing_skipped = False + approved_existing_note_sha256 = "" + if output_mode == "obsidian": + ( + target_path, + resolved_subdir, + admission, + domain_routing_skipped, + approved_existing_note_sha256, + ) = resolve_obsidian_save_target( + args, + config, + root_path, + title=title, + abstract=str(record.get("abstract", "")), + source_sha256=str(source_manifest["source_sha256"]), + output_language=output_language, + ) + else: + if args.preflight: + raise SystemExit("--preflight requires Obsidian save mode.") + resolved_subdir = resolve_domain_subdir( + config, + title=title, + abstract=str(record.get("abstract", "")), + subdir=args.subdir, + ) + target_path = resolve_obsidian_note_path( + config, + title=title, + subdir=resolved_subdir, + filename=args.filename, + ) + + if args.preflight: + emit( + { + "status": "ok", + "script": "write_obsidian_note.py", + "phase": "save_target_admission", + "admission": admission, + "domain_routing_skipped": domain_routing_skipped, + "target_directory": str(target_path.parent), + "note_path": str(target_path), + "source_sha256": str(source_manifest["source_sha256"]), + "output_language": output_language, + }, + args.output, + ) + return + + if not args.lint_json: + raise SystemExit("Formal Save requires Final Note Lint with output_language.") + lint_path = str(Path(args.lint_json).expanduser().resolve()) + # utf-8-sig tolerates a BOM in the lint JSON (e.g. produced/edited on + # Windows) that would otherwise crash json.loads before any gate check. + lint = json.loads(Path(lint_path).read_text(encoding="utf-8-sig")) + try: + require_artifact_output_language(lint, "lint artifact", output_language) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + require_lint_gate(lint, "passes_basic_structure", "basic structure", lint_path) + require_lint_gate(lint, "passes_style_gate", "style", lint_path) + require_lint_gate(lint, "passes_math_gate", "math", lint_path) + if "passes_figure_gate" in lint and not lint.get("passes_figure_gate", False): + raise SystemExit(lint_failure_message(lint, "figure", lint_path)) + if "passes_plan_gate" in lint and not lint.get("passes_plan_gate", False): + raise SystemExit(lint_failure_message(lint, "plan", lint_path)) + if "passes_substantive_content" in lint and not lint.get("passes_substantive_content", False): + raise SystemExit(lint_failure_message(lint, "substantive content", lint_path)) + if "passes_reference_hygiene_gate" in lint and not lint.get("passes_reference_hygiene_gate", False): + raise SystemExit(lint_failure_message(lint, "reference hygiene", lint_path)) + + if not args.figure_decisions: + raise SystemExit("Formal Save requires Figure/Table Decisions with output_language.") + figure_decisions = maybe_load_json_record(args.figure_decisions) + if figure_decisions is None: + raise SystemExit(f"Expected JSON object for --figure-decisions: {args.figure_decisions}") + try: + require_artifact_output_language( + figure_decisions, + "Figure/Table Decisions", + output_language, + ) + except ValueError as exc: + raise SystemExit(str(exc)) from exc if args.content_file: # utf-8-sig strips a leading BOM so it is never written into the saved @@ -230,55 +792,145 @@ def main() -> None: note_text = sys.stdin.read() else: raise SystemExit("write_obsidian_note.py requires --content-file, --content, or --stdin.") + note_text = note_text.replace("\r\n", "\n") + lint_note_sha256 = str(lint.get("note_sha256", "")).strip() + if not lint_note_sha256: + raise SystemExit("lint artifact requires note_sha256.") + note_sha256 = hashlib.sha256(note_text.encode("utf-8")).hexdigest() + if lint_note_sha256 != note_sha256: + raise SystemExit( + "write_obsidian_note.py refused to write note because the final note " + "changed after Final Note Lint; rerun lint under the same output_language." + ) require_reference_hygiene(note_text, "before save") - config = runtime_config() - if args.vault: - config["obsidian_vault"] = args.vault - resolved_subdir = resolve_domain_subdir( - config, - title=title, - abstract=str(record.get("abstract", "")), - subdir=args.subdir, - ) + overwrote_existing_note = bool(approved_existing_note_sha256) + if output_mode == "obsidian": + # ponytail: this closes sequential TOCTOU changes; add a Vault lock only + # if concurrent DeepPaperNote writers must be fully serialized. + rechecked_target, _, _, _, rechecked_existing_sha256 = resolve_obsidian_save_target( + args, + config, + root_path, + title=title, + abstract=str(record.get("abstract", "")), + source_sha256=str(source_manifest["source_sha256"]), + output_language=output_language, + ) + if ( + rechecked_target.resolve() != target_path.resolve() + or rechecked_existing_sha256 != approved_existing_note_sha256 + ): + block_directory_conflict( + args, + conflict_code="save_target_changed", + target_directories=[target_path.parent, rechecked_target.parent], + source_sha256=str(source_manifest["source_sha256"]), + output_language=output_language, + ) - target_path = resolve_obsidian_note_path( - config, - title=title, - subdir=resolved_subdir, - filename=args.filename, - ) asset_dir = resolve_note_asset_dir(target_path, args.asset_subdir) asset_subdir = asset_dir.relative_to(target_path.parent).as_posix() + asset_directory_existed = asset_dir.exists() ensure_parent(target_path) - figure_decisions = maybe_load_json_record(args.figure_decisions) if args.figure_decisions else {} - if args.figure_decisions and figure_decisions is None: - raise SystemExit(f"Expected JSON object for --figure-decisions: {args.figure_decisions}") - materialized_figures = ( - materialize_insert_decisions( - note_text, - target_path, - figure_decisions, - asset_subdir, - ) - if figure_decisions - else [] + paper_id = ( + args.paper_id + or str(source_manifest.get("paper_id", "")) + or str(record.get("paper_id", "")) ) - Path(target_path).write_text(note_text, encoding="utf-8") - require_reference_hygiene(Path(target_path).read_text(encoding="utf-8"), "after save") - asset_dir.mkdir(parents=True, exist_ok=True) + sidecar_path = ( + target_path.parent / PAPER_DIRECTORY_SIDECAR + if output_mode == "obsidian" + else None + ) + note_backup = None + sidecar_backup = None + created_assets: list[Path] = [] + note_attempted = False + sidecar_attempted = False + try: + note_backup = backup_file(target_path) + sidecar_backup = backup_file(sidecar_path) if sidecar_path is not None else None + materialized_figures = ( + materialize_insert_decisions( + note_text, + target_path, + figure_decisions, + asset_subdir, + created_assets, + ) + if figure_decisions + else [] + ) + if ( + approved_existing_note_sha256 + and file_sha256(target_path) != approved_existing_note_sha256 + ): + block_note_conflict( + args, + target_path, + conflict_code="stale_overwrite_confirmation", + source_sha256=str(source_manifest["source_sha256"]), + output_language=output_language, + ) + note_attempted = True + atomic_write_note(target_path, note_text) + require_reference_hygiene( + target_path.read_text(encoding="utf-8"), + "after save", + ) + asset_dir.mkdir(parents=True, exist_ok=True) + if sidecar_path is not None: + sidecar_attempted = True + write_paper_directory_sidecar( + target_path, + paper_id=paper_id, + title=title, + source_sha256=str(source_manifest["source_sha256"]), + output_language=output_language, + note_sha256=note_sha256, + ) + except BaseException: + if sidecar_attempted and sidecar_path is not None: + restore_file(sidecar_path, sidecar_backup) + if sidecar_backup is not None: + try: + ensure_sidecar_hidden(sidecar_path) + except OSError: + pass + if note_attempted: + restore_file(target_path, note_backup) + for created_asset in reversed(created_assets): + created_asset.unlink(missing_ok=True) + if not asset_directory_existed: + try: + asset_dir.rmdir() + except OSError: + pass + raise + finally: + if note_backup is not None: + note_backup.unlink(missing_ok=True) + if sidecar_backup is not None: + sidecar_backup.unlink(missing_ok=True) payload = { "status": "ok", "script": "write_obsidian_note.py", - "paper_id": args.paper_id or record.get("paper_id", ""), + "output_language": output_language, + "paper_id": paper_id, "title": title, "note_path": str(target_path), "subdir": resolved_subdir, "images_dir": str(asset_dir), "materialized_figures": materialized_figures, + "overwrote_existing_note": overwrote_existing_note, + "admission": admission, + "domain_routing_skipped": domain_routing_skipped, } - output_mode, root_path = resolve_note_output_mode(config) + if sidecar_path is not None: + payload["sidecar_path"] = str(sidecar_path) + payload["source_sha256"] = str(source_manifest["source_sha256"]) payload["output_mode"] = output_mode payload["base_output_root"] = str(root_path) if config.get("obsidian_vault"): diff --git a/tests/conftest.py b/tests/conftest.py index f8ace52..9430268 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,38 @@ from __future__ import annotations +import json import sys from pathlib import Path +import pytest PROJECT_ROOT = Path(__file__).resolve().parents[1] SCRIPTS_DIR = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" if str(SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(SCRIPTS_DIR)) + + +@pytest.fixture(autouse=True) +def configured_user_home(tmp_path: Path, monkeypatch) -> Path: + config_path = tmp_path / ".deeppapernote" / "config.json" + config_path.parent.mkdir(exist_ok=True) + for name in ( + "DEEPPAPERNOTE_OUTPUT_LANGUAGE", + "DEEPPAPERNOTE_SAVE_MODE", + "DEEPPAPERNOTE_OBSIDIAN_VAULT", + "DEEPPAPERNOTE_PAPERS_DIR", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("DEEPPAPERNOTE_CONFIG_PATH", str(config_path)) + config_path.write_text( + json.dumps( + { + "output_language": "zh-CN", + "save_mode": "workspace", + "papers_dir": "Research/Papers", + } + ), + encoding="utf-8", + ) + return config_path diff --git a/tests/test_acquisition_artifacts.py b/tests/test_acquisition_artifacts.py index bc9da64..aefebb8 100644 --- a/tests/test_acquisition_artifacts.py +++ b/tests/test_acquisition_artifacts.py @@ -1505,6 +1505,57 @@ def test_collect_metadata_preserves_provider_result_as_identity_observation( ] +def test_exact_swe_bench_title_admits_one_unique_arxiv_match( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + title = "SWE-bench: Can Language Models Resolve Real-world Github Issues?" + resolve_payload = { + "status": "ok", + "script": "resolve_paper.py", + "paper_id": "title:2156776fbf55", + "source_type": "title_query", + "title": title, + "metadata_sources": ["title_query"], + } + monkeypatch.setattr(common, "search_semantic_scholar", lambda *args, **kwargs: []) + monkeypatch.setattr(common, "search_openalex_by_title", lambda *args, **kwargs: []) + monkeypatch.setattr(common, "search_crossref_by_title", lambda *args, **kwargs: []) + monkeypatch.setattr( + common, + "safe_fetch_arxiv_entries", + lambda **kwargs: [ + { + "source": "arxiv", + "source_type": "arxiv", + "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", + "authors": ["Carlos E. Jimenez"], + "published": "2023-10-10T16:47:29Z", + "arxiv_id": "2310.06770", + "pdf_url": "https://arxiv.org/pdf/2310.06770v3", + } + ], + ) + + observations = common.collect_metadata_observations(resolve_payload) + identity, _ = build_identity_from_payloads( + tmp_path, + monkeypatch, + resolve_payload=resolve_payload, + metadata_payload={ + **resolve_payload, + "script": "collect_metadata.py", + "identity_observations": observations, + }, + ) + + assert identity["identity_verdict"] == "accepted" + assert identity["accepted_metadata"]["arxiv_id"] == "2310.06770" + assert identity["accepted_observations"][0]["reason"] == ( + "unique_exact_arxiv_title" + ) + + @pytest.mark.parametrize( "module", [extract_source_text, extract_evidence, extract_pdf_assets], diff --git a/tests/test_build_synthesis_bundle_contract.py b/tests/test_build_synthesis_bundle_contract.py index 5208637..bf030d4 100644 --- a/tests/test_build_synthesis_bundle_contract.py +++ b/tests/test_build_synthesis_bundle_contract.py @@ -4,10 +4,48 @@ import pytest -from build_synthesis_bundle import bundle +from build_synthesis_bundle import bundle as build_bundle from contracts import WRITING_CONTRACT_RULES +def accepted_source_manifest() -> dict: + return { + "paper_id": "paper:contract", + "identity_contract": { + "artifact_type": "canonical_identity", + "schema_version": 2, + "paper_id": "paper:contract", + "identity_verdict": "accepted", + "work_level_identity": {"title": "Contract Paper"}, + "accepted_metadata": {"title": "Contract Paper"}, + }, + } + + +def bundle(*args: object, **kwargs: object) -> dict: + if not kwargs.get("figures_wrapper"): + kwargs["figures_wrapper"] = {"output_language": "zh-CN", "figure_plan": {}} + if not kwargs.get("figure_decisions_wrapper"): + kwargs["figure_decisions_wrapper"] = {"output_language": "zh-CN", "decisions": []} + return build_bundle(*args, **kwargs) + + +def test_bundle_rejects_mismatched_adjacent_artifact_language() -> None: + with pytest.raises( + ValueError, + match="Figure Plan output_language en does not match resolved output_language zh-CN", + ): + bundle( + metadata={"title": "Contract Paper"}, + evidence_wrapper={"evidence_pack": {}}, + figures_wrapper={"output_language": "en", "figure_plan": {}}, + assets_wrapper={}, + source_manifest=accepted_source_manifest(), + figure_decisions_wrapper={"output_language": "zh-CN", "decisions": []}, + output_language="zh-CN", + ) + + def test_bundle_refuses_raw_metadata_without_an_identity_contract() -> None: with pytest.raises(ValueError, match="accepted canonical identity contract"): bundle( diff --git a/tests/test_citation_links.py b/tests/test_citation_links.py index cbbf158..fdd638c 100644 --- a/tests/test_citation_links.py +++ b/tests/test_citation_links.py @@ -280,7 +280,11 @@ def test_bundle_exposes_reference_candidates_under_references(monkeypatch) -> No monkeypatch.setattr( build_synthesis_bundle, "runtime_config", - lambda: {"obsidian_vault": "", "papers_dir": "Research/Papers"}, + lambda **_kwargs: { + "output_language": "zh-CN", + "obsidian_vault": "", + "papers_dir": "Research/Papers", + }, ) synthesis = build_synthesis_bundle.bundle( @@ -295,8 +299,9 @@ def test_bundle_exposes_reference_candidates_under_references(monkeypatch) -> No ] } }, - figures_wrapper={}, + figures_wrapper={"output_language": "zh-CN", "figure_plan": {}}, assets_wrapper={}, + figure_decisions_wrapper={"output_language": "zh-CN", "decisions": []}, source_manifest={ "identity_contract": { "artifact_type": "canonical_identity", @@ -314,3 +319,41 @@ def test_bundle_exposes_reference_candidates_under_references(monkeypatch) -> No assert candidates[0]["display_text"] == "Vaswani et al. (2017). Attention Is All You Need." assert candidates[0]["match_status"] == "vault_unavailable" assert candidates[0]["wikilink"] == "" + + +def test_bundle_reuses_cli_language_configuration_for_reference_resolution(monkeypatch) -> None: + def configured_runtime(*, cli_overrides): + assert cli_overrides == {"output_language": "en"} + return { + "output_language": "en", + "obsidian_vault": "", + "papers_dir": "Research/Papers", + } + + monkeypatch.setattr(build_synthesis_bundle, "runtime_config", configured_runtime) + + synthesis = build_synthesis_bundle.bundle( + metadata={"title": "Citation Paper"}, + evidence_wrapper={ + "evidence_pack": { + "reference_candidates": [{"display_text": "Unknown Paper."}] + } + }, + figures_wrapper={"output_language": "en", "figure_plan": {}}, + assets_wrapper={}, + figure_decisions_wrapper={"output_language": "en", "decisions": []}, + source_manifest={ + "identity_contract": { + "artifact_type": "canonical_identity", + "schema_version": 2, + "paper_id": "paper:citation-test", + "identity_verdict": "accepted", + "work_level_identity": {"title": "Citation Paper"}, + "accepted_metadata": {"title": "Citation Paper"}, + } + }, + output_language="en", + ) + + assert synthesis["writing_contract"]["language"] == "en" + assert synthesis["references"]["candidates"][0]["match_status"] == "vault_unavailable" diff --git a/tests/test_cli_fallback.py b/tests/test_cli_fallback.py index 3cef532..0720b72 100644 --- a/tests/test_cli_fallback.py +++ b/tests/test_cli_fallback.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os import subprocess @@ -7,14 +8,996 @@ from pathlib import Path import pytest +import write_obsidian_note +PROJECT_ROOT = Path(__file__).resolve().parents[1] +WRITE_SCRIPT = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "write_obsidian_note.py" +MATERIALIZE_SCRIPT = ( + PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "materialize_figure_asset.py" +) +ENV_SCRIPT = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "check_environment.py" + + +def formal_save_args( + tmp_path: Path, + note_text: str, + output_language: str = "zh-CN", +) -> list[str]: + lint_path = tmp_path / "passing-lint.json" + lint_path.write_text( + json.dumps( + { + "output_language": output_language, + "note_sha256": hashlib.sha256(note_text.encode("utf-8")).hexdigest(), + "passes_basic_structure": True, + "passes_style_gate": True, + "passes_math_gate": True, + } + ), + encoding="utf-8", + ) + decisions_path = tmp_path / "figure-decisions.json" + decisions_path.write_text( + json.dumps({"output_language": output_language, "decisions": []}), + encoding="utf-8", + ) + return ["--lint-json", str(lint_path), "--figure-decisions", str(decisions_path)] + + +def source_manifest_args(tmp_path: Path, source_sha256: str = "a" * 64) -> list[str]: + source_manifest_path = tmp_path / "source-manifest.json" + source_manifest_path.write_text( + json.dumps( + { + "status": "ok", + "paper_id": "paper:test", + "source_sha256": source_sha256, + } + ), + encoding="utf-8", + ) + return ["--source-manifest", str(source_manifest_path)] + + +def test_write_note_creates_language_variant_and_directory_sidecar(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# My Test Paper\n\nVault write admission test.\n" + + result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "Benchmark", + "--content", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + note_path = Path(payload["note_path"]) + sidecar_path = note_path.parent / ".deeppapernote.json" + assert note_path.name == "My_Test_Paper.zh-CN.md" + assert note_path.read_text(encoding="utf-8") == note_text + assert json.loads(sidecar_path.read_text(encoding="utf-8")) == { + "artifact_type": "deeppapernote_paper_directory", + "schema_version": 1, + "paper_id": "paper:test", + "title": "My Test Paper", + "source_sha256": "a" * 64, + "note_stem": "My_Test_Paper", + "notes": { + "zh-CN": { + "filename": "My_Test_Paper.zh-CN.md", + "note_sha256": hashlib.sha256(note_text.encode("utf-8")).hexdigest(), + } + }, + } + + +def test_write_note_reuses_empty_same_name_directory_before_domain_routing( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + existing_dir = vault / "LegacyArchive/My_Test_Paper" + existing_dir.mkdir(parents=True) + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# My Test Paper\n\nReuse the empty directory.\n" + + result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "NewDomain", + "--content", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + note_path = Path(json.loads(result.stdout)["note_path"]) + assert note_path.parent == existing_dir.resolve() + assert not (vault / "Research/Papers/NewDomain/My_Test_Paper").exists() + + +def test_write_note_reuses_source_directory_for_another_language(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + chinese_note = "# My Test Paper\n\n中文笔记。\n" + english_note = "# My Test Paper\n\nEnglish note.\n" + + chinese_result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "ChineseDomain", + "--content", + chinese_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, chinese_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert chinese_result.returncode == 0, chinese_result.stderr + chinese_path = Path(json.loads(chinese_result.stdout)["note_path"]) + + english_result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "EnglishDomain", + "--language", + "en", + "--content", + english_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, english_note, "en"), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert english_result.returncode == 0, english_result.stderr + english_path = Path(json.loads(english_result.stdout)["note_path"]) + assert english_path.parent == chinese_path.parent + assert english_path.name == "My_Test_Paper.en.md" + sidecar = json.loads((english_path.parent / ".deeppapernote.json").read_text(encoding="utf-8")) + assert set(sidecar["notes"]) == {"zh-CN", "en"} + + +def test_write_note_does_not_replace_another_language_note_image(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + + def save_args(note_text: str, language: str, source_image: Path) -> list[str]: + args = formal_save_args(tmp_path, note_text, language) + decisions_path = Path(args[-1]) + digest = hashlib.sha256(source_image.read_bytes()).hexdigest() + decisions_path.write_text( + json.dumps( + { + "output_language": language, + "decisions": [ + { + "source_id": "Figure 1", + "decision": "insert", + "source_image_path": str(source_image), + "source_image_filename": "shared.png", + "source_image_sha256": digest, + "visual_review": { + "status": "pass", + "reviewed_asset_sha256": digest, + }, + } + ], + } + ), + encoding="utf-8", + ) + return args + + chinese_image = tmp_path / "chinese.png" + english_image = tmp_path / "english.png" + chinese_image.write_bytes(b"chinese-image") + english_image.write_bytes(b"english-image") + chinese_note = "# My Test Paper\n\n![Figure 1](images/shared.png)\n" + english_note = "# My Test Paper\n\n![Figure 1](images/shared.png)\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + chinese_note, + *source_manifest_args(tmp_path), + *save_args(chinese_note, "zh-CN", chinese_image), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + directory = Path(json.loads(first.stdout)["note_path"]).parent + + second = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--language", + "en", + "--content", + english_note, + *source_manifest_args(tmp_path), + *save_args(english_note, "en", english_image), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert second.returncode != 0 + assert "existing paper-local image has different bytes" in second.stderr + assert (directory / "images/shared.png").read_bytes() == b"chinese-image" + assert not (directory / "My_Test_Paper.en.md").exists() + + +def test_formal_save_rolls_back_when_sidecar_finalization_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + original_note = "# Rollback Paper\n\nOriginal body.\n" + replacement_note = "# Rollback Paper\n\nReplacement body.\n" + monkeypatch.setenv("DEEPPAPERNOTE_DISABLE_SHELL_CONFIG", "1") + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "Rollback Paper", + "--content", + original_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, original_note), + ], + cwd=tmp_path, + env=os.environ.copy(), + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + note_path = Path(json.loads(first.stdout)["note_path"]) + sidecar_path = note_path.parent / ".deeppapernote.json" + original_sidecar = sidecar_path.read_bytes() + + monkeypatch.setattr( + sys, + "argv", + [ + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "Rollback Paper", + "--content", + replacement_note, + "--overwrite-existing-note", + "--expected-existing-note-sha256", + hashlib.sha256(original_note.encode("utf-8")).hexdigest(), + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, replacement_note), + ], + ) + + def fail_hidden(_path: Path) -> None: + raise OSError("simulated Hidden attribute failure") + + monkeypatch.setattr(write_obsidian_note, "ensure_sidecar_hidden", fail_hidden) + + with pytest.raises(OSError, match="simulated Hidden attribute failure"): + write_obsidian_note.main() + + assert note_path.read_text(encoding="utf-8") == original_note + assert sidecar_path.read_bytes() == original_sidecar + + +def test_write_note_blocks_same_source_and_language_without_mutating_vault( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + original_note = "# My Test Paper\n\nOriginal note.\n" + replacement_note = "# My Test Paper\n\nReplacement note.\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "OriginalDomain", + "--content", + original_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, original_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + note_path = Path(json.loads(first.stdout)["note_path"]) + sidecar_path = note_path.parent / ".deeppapernote.json" + original_sidecar = sidecar_path.read_bytes() + + second = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "DifferentDomain", + "--content", + replacement_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, replacement_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert second.returncode == 2, second.stderr + conflict = json.loads(second.stdout) + assert conflict["status"] == "blocked" + assert conflict["conflict_code"] == "same_language_note_exists" + assert conflict["requires_user_confirmation"] is True + assert conflict["existing_note_path"] == str(note_path) + assert conflict["existing_note_sha256"] == hashlib.sha256( + original_note.encode("utf-8") + ).hexdigest() + assert note_path.read_text(encoding="utf-8") == original_note + assert sidecar_path.read_bytes() == original_sidecar + assert not (vault / "Research/Papers/DifferentDomain").exists() + + +def test_write_note_overwrites_only_with_matching_existing_note_sha256( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + original_note = "# My Test Paper\n\nOriginal note.\n" + replacement_note = "# My Test Paper\n\nApproved replacement.\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + original_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, original_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + note_path = Path(json.loads(first.stdout)["note_path"]) + original_sha256 = hashlib.sha256(original_note.encode("utf-8")).hexdigest() + + overwrite = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + replacement_note, + "--overwrite-existing-note", + "--expected-existing-note-sha256", + original_sha256, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, replacement_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert overwrite.returncode == 0, overwrite.stderr + payload = json.loads(overwrite.stdout) + assert payload["overwrote_existing_note"] is True + assert Path(payload["note_path"]) == note_path + assert note_path.read_text(encoding="utf-8") == replacement_note + sidecar = json.loads((note_path.parent / ".deeppapernote.json").read_text(encoding="utf-8")) + assert sidecar["notes"]["zh-CN"]["note_sha256"] == hashlib.sha256( + replacement_note.encode("utf-8") + ).hexdigest() + + +def test_write_note_blocks_overwrite_when_confirmed_hash_is_stale(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + original_note = "# My Test Paper\n\nOriginal note.\n" + replacement_note = "# My Test Paper\n\nStale replacement.\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + original_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, original_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + note_path = Path(json.loads(first.stdout)["note_path"]) + + overwrite = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + replacement_note, + "--overwrite-existing-note", + "--expected-existing-note-sha256", + "b" * 64, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, replacement_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert overwrite.returncode == 2, overwrite.stderr + conflict = json.loads(overwrite.stdout) + assert conflict["conflict_code"] == "stale_overwrite_confirmation" + assert conflict["existing_note_sha256"] == hashlib.sha256( + original_note.encode("utf-8") + ).hexdigest() + assert note_path.read_text(encoding="utf-8") == original_note + + +def test_write_note_blocks_same_name_directory_for_a_different_source( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + first_note = "# Shared Title\n\nFirst source.\n" + second_note = "# Shared Title\n\nSecond source.\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "Shared Title", + "--subdir", + "FirstDomain", + "--content", + first_note, + *source_manifest_args(tmp_path, "a" * 64), + *formal_save_args(tmp_path, first_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + first_path = Path(json.loads(first.stdout)["note_path"]) + archived_dir = vault / "ArchivedPapers" / first_path.parent.name + archived_dir.parent.mkdir() + first_path.parent.rename(archived_dir) + first_path = archived_dir / first_path.name + + second = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "Shared Title", + "--subdir", + "SecondDomain", + "--language", + "en", + "--content", + second_note, + *source_manifest_args(tmp_path, "b" * 64), + *formal_save_args(tmp_path, second_note, "en"), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert second.returncode == 2, second.stderr + conflict = json.loads(second.stdout) + assert conflict["conflict_code"] == "same_name_different_source" + assert conflict["target_directory"] == str(first_path.parent) + assert first_path.read_text(encoding="utf-8") == first_note + assert not (vault / "Research/Papers/SecondDomain").exists() + + +def test_write_note_freezes_directory_name_for_the_same_source_bytes(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + chinese_note = "# First Metadata Title\n\n中文笔记。\n" + english_note = "# Revised Metadata Title\n\nEnglish note.\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "First Metadata Title", + "--subdir", + "FirstDomain", + "--content", + chinese_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, chinese_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + first_path = Path(json.loads(first.stdout)["note_path"]) + + archived_dir = vault / "ArchivedSources" / first_path.parent.name + archived_dir.parent.mkdir() + first_path.parent.rename(archived_dir) + first_path = archived_dir / first_path.name + + second = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "Revised Metadata Title", + "--subdir", + "SecondDomain", + "--language", + "en", + "--content", + english_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, english_note, "en"), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert second.returncode == 0, second.stderr + second_path = Path(json.loads(second.stdout)["note_path"]) + assert second_path.parent == first_path.parent + assert second_path.name == "First_Metadata_Title.en.md" + assert not (vault / "Research/Papers/SecondDomain/Revised_Metadata_Title").exists() + + +def test_save_target_preflight_reuses_source_before_note_generation(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + chinese_note = "# My Test Paper\n\n中文笔记。\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--subdir", + "OriginalDomain", + "--content", + chinese_note, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, chinese_note), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + directory = Path(json.loads(first.stdout)["note_path"]).parent + + preflight = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--preflight", + "--vault", + str(vault), + "--title", + "Changed Metadata Title", + "--subdir", + "MustNotBeUsed", + "--language", + "en", + *source_manifest_args(tmp_path), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert preflight.returncode == 0, preflight.stderr + payload = json.loads(preflight.stdout) + assert payload["status"] == "ok" + assert payload["phase"] == "save_target_admission" + assert payload["admission"] == "reuse_source_directory" + assert payload["domain_routing_skipped"] is True + assert payload["target_directory"] == str(directory) + assert Path(payload["note_path"]).name == "My_Test_Paper.en.md" + assert not Path(payload["note_path"]).exists() + + +def test_save_target_preflight_reports_same_language_conflict_without_lint( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# My Test Paper\n\nExisting note.\n" + + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + + preflight = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--preflight", + "--vault", + str(vault), + "--title", + "My Test Paper", + *source_manifest_args(tmp_path), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) -PROJECT_ROOT = Path(__file__).resolve().parents[1] -WRITE_SCRIPT = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "write_obsidian_note.py" -MATERIALIZE_SCRIPT = ( - PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "materialize_figure_asset.py" -) -ENV_SCRIPT = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "check_environment.py" + assert preflight.returncode == 2, preflight.stderr + conflict = json.loads(preflight.stdout) + assert conflict["conflict_code"] == "same_language_note_exists" + assert conflict["requires_user_confirmation"] is True + + +def test_save_target_preflight_honors_recorded_language_note_filename( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# My Test Paper\n\nExisting note.\n" + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + original_path = Path(json.loads(first.stdout)["note_path"]) + renamed_path = original_path.with_name("Custom Chinese Note.md") + original_path.rename(renamed_path) + sidecar_path = renamed_path.parent / ".deeppapernote.json" + sidecar = json.loads(sidecar_path.read_text(encoding="utf-8")) + sidecar["notes"]["zh-CN"]["filename"] = renamed_path.name + updated_sidecar_path = sidecar_path.with_name(f"{sidecar_path.name}.tmp") + updated_sidecar_path.write_text(json.dumps(sidecar), encoding="utf-8") + os.replace(updated_sidecar_path, sidecar_path) + + preflight = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--preflight", + "--vault", + str(vault), + "--title", + "My Test Paper", + *source_manifest_args(tmp_path), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert preflight.returncode == 2, preflight.stderr + conflict = json.loads(preflight.stdout) + assert conflict["conflict_code"] == "same_language_note_exists" + assert conflict["existing_note_path"] == str(renamed_path) + + +def test_save_target_preflight_blocks_missing_recorded_language_note(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# My Test Paper\n\nExisting note.\n" + first = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--vault", + str(vault), + "--title", + "My Test Paper", + "--content", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + assert first.returncode == 0, first.stderr + note_path = Path(json.loads(first.stdout)["note_path"]) + note_path.unlink() + + preflight = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--preflight", + "--vault", + str(vault), + "--title", + "My Test Paper", + *source_manifest_args(tmp_path), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert preflight.returncode == 2, preflight.stderr + conflict = json.loads(preflight.stdout) + assert conflict["conflict_code"] == "recorded_language_note_missing" + assert conflict["target_directory"] == str(note_path.parent) + + +def test_save_target_preflight_blocks_unidentified_nonempty_same_name_directory( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + legacy_dir = vault / "Research/Papers/Legacy/My_Test_Paper" + legacy_dir.mkdir(parents=True) + legacy_note = legacy_dir / "My_Test_Paper.zh-CN.md" + legacy_note.write_text("# 手工创建的中文笔记\n", encoding="utf-8") + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + + preflight = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--preflight", + "--vault", + str(vault), + "--title", + "My Test Paper", + "--language", + "en", + *source_manifest_args(tmp_path), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert preflight.returncode == 2, preflight.stderr + conflict = json.loads(preflight.stdout) + assert conflict["conflict_code"] == "unidentified_same_name_directory" + assert legacy_note.read_text(encoding="utf-8") == "# 手工创建的中文笔记\n" + + +def test_save_target_preflight_blocks_multiple_directories_for_the_same_source( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + source_sha256 = "a" * 64 + directories = [ + vault / "Research/Papers/One/First_Title", + vault / "Research/Papers/Two/Second_Title", + ] + for index, directory in enumerate(directories, start=1): + directory.mkdir(parents=True) + (directory / ".deeppapernote.json").write_text( + json.dumps( + { + "artifact_type": "deeppapernote_paper_directory", + "schema_version": 1, + "paper_id": "paper:test", + "title": f"Title {index}", + "source_sha256": source_sha256, + "note_stem": directory.name, + "notes": {}, + } + ), + encoding="utf-8", + ) + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + + preflight = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--preflight", + "--vault", + str(vault), + "--title", + "Any Title", + *source_manifest_args(tmp_path, source_sha256), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + ) + + assert preflight.returncode == 2, preflight.stderr + conflict = json.loads(preflight.stdout) + assert conflict["conflict_code"] == "multiple_source_directories" + assert set(conflict["matching_directories"]) == {str(path) for path in directories} def test_write_note_falls_back_to_workspace(tmp_path: Path) -> None: @@ -22,7 +1005,7 @@ def test_write_note_falls_back_to_workspace(tmp_path: Path) -> None: env.pop("DEEPPAPERNOTE_OBSIDIAN_VAULT", None) env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "DeepPaperNote_output" env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" - + note_text = "# Fallback Output Test\n\nThis is a workspace fallback write test.\n" result = subprocess.run( [ sys.executable, @@ -30,7 +1013,8 @@ def test_write_note_falls_back_to_workspace(tmp_path: Path) -> None: "--title", "Fallback Output Test", "--content", - "# Fallback Output Test\n\nThis is a workspace fallback write test.\n", + note_text, + *formal_save_args(tmp_path, note_text), ], cwd=tmp_path, env=env, @@ -56,6 +1040,7 @@ def test_write_note_rejects_asset_directory_outside_save_target(tmp_path: Path) env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" outside_asset_dir = tmp_path / "outside-assets" + note_text = "# Unsafe Asset Directory\n" result = subprocess.run( [ sys.executable, @@ -63,7 +1048,9 @@ def test_write_note_rejects_asset_directory_outside_save_target(tmp_path: Path) "--title", "Unsafe Asset Directory", "--content", - "# Unsafe Asset Directory\n", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), "--asset-subdir", str(outside_asset_dir), ], @@ -178,6 +1165,7 @@ def test_write_note_in_vault_mode_does_not_duplicate_paper_slug(tmp_path: Path) env.pop("DEEPPAPERNOTE_OBSIDIAN_VAULT", None) env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# My Test Paper\n\nVault write regression test.\n" result = subprocess.run( [ sys.executable, @@ -189,7 +1177,9 @@ def test_write_note_in_vault_mode_does_not_duplicate_paper_slug(tmp_path: Path) "--subdir", "心理健康/My_Test_Paper", "--content", - "# My Test Paper\n\nVault write regression test.\n", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), ], cwd=tmp_path, env=env, @@ -199,15 +1189,75 @@ def test_write_note_in_vault_mode_does_not_duplicate_paper_slug(tmp_path: Path) ) payload = json.loads(result.stdout) note_path = Path(payload["note_path"]) - assert note_path == vault / "Research/Papers" / "心理健康" / "My_Test_Paper" / "My_Test_Paper.md" + assert note_path == ( + vault + / "Research/Papers" + / "心理健康" + / "My_Test_Paper" + / "My_Test_Paper.zh-CN.md" + ) assert note_path.exists() +def test_write_note_uses_ready_chinese_obsidian_configuration( + tmp_path: Path, configured_user_home: Path +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + configured_user_home.write_text( + json.dumps( + { + "output_language": "zh-CN", + "save_mode": "obsidian", + "obsidian_vault": str(vault), + "papers_dir": "Research/Papers", + } + ), + encoding="utf-8", + ) + env = os.environ.copy() + for name in ( + "DEEPPAPERNOTE_OUTPUT_LANGUAGE", + "DEEPPAPERNOTE_SAVE_MODE", + "DEEPPAPERNOTE_OBSIDIAN_VAULT", + "DEEPPAPERNOTE_PAPERS_DIR", + ): + env.pop(name, None) + + note_text = "# 配置驱动保存\n\n中文 Obsidian 保存测试。\n" + result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--title", + "配置驱动保存", + "--subdir", + "机器学习", + "--content", + note_text, + *source_manifest_args(tmp_path), + *formal_save_args(tmp_path, note_text), + ], + cwd=tmp_path, + env=env, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + + assert payload["output_language"] == "zh-CN" + assert payload["output_mode"] == "obsidian" + assert Path(payload["note_path"]).is_file() + assert Path(payload["note_path"]).is_relative_to(vault) + + def test_write_note_refuses_when_math_gate_fails(tmp_path: Path) -> None: lint_path = tmp_path / "lint.json" lint_path.write_text( json.dumps( { + "output_language": "zh-CN", "passes_basic_structure": True, "passes_style_gate": True, "passes_math_gate": False, @@ -239,7 +1289,9 @@ def test_write_note_refuses_when_math_gate_fails(tmp_path: Path) -> None: assert "math gate failed" in result.stderr -def test_write_note_accepts_legacy_lint_json_without_figure_gate(tmp_path: Path) -> None: +def test_write_note_rejects_legacy_lint_json_without_output_language_before_save( + tmp_path: Path, +) -> None: lint_path = tmp_path / "lint.json" lint_path.write_text( json.dumps( @@ -255,7 +1307,6 @@ def test_write_note_accepts_legacy_lint_json_without_figure_gate(tmp_path: Path) env.pop("DEEPPAPERNOTE_OBSIDIAN_VAULT", None) env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "DeepPaperNote_output" env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" - result = subprocess.run( [ sys.executable, @@ -269,12 +1320,99 @@ def test_write_note_accepts_legacy_lint_json_without_figure_gate(tmp_path: Path) ], cwd=tmp_path, env=env, - check=True, + check=False, capture_output=True, text=True, ) - payload = json.loads(result.stdout) - assert Path(payload["note_path"]).exists() + assert result.returncode != 0 + assert "lint artifact requires output_language" in result.stderr + assert not (tmp_path / "DeepPaperNote_output").exists() + + +def test_write_note_rejects_note_edited_after_final_lint_before_save( + tmp_path: Path, +) -> None: + linted_note = "# Final Review Test\n\nLinted body.\n" + lint_path = tmp_path / "lint.json" + lint_path.write_text( + json.dumps( + { + "output_language": "zh-CN", + "note_sha256": hashlib.sha256(linted_note.encode("utf-8")).hexdigest(), + "passes_basic_structure": True, + "passes_style_gate": True, + "passes_math_gate": True, + } + ), + encoding="utf-8", + ) + env = os.environ.copy() + env.pop("DEEPPAPERNOTE_OBSIDIAN_VAULT", None) + env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "DeepPaperNote_output" + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + decisions_args = formal_save_args(tmp_path, linted_note)[2:] + + result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--title", + "Final Review Test", + "--content", + linted_note.replace("Linted", "Reviewed"), + "--lint-json", + str(lint_path), + *decisions_args, + ], + cwd=tmp_path, + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "changed after Final Note Lint" in result.stderr + assert not (tmp_path / "DeepPaperNote_output").exists() + + +def test_write_note_rejects_mismatched_figure_decisions_before_save_side_effects( + tmp_path: Path, +) -> None: + decisions_path = tmp_path / "decisions.json" + decisions_path.write_text( + json.dumps({"output_language": "en", "decisions": []}), + encoding="utf-8", + ) + env = os.environ.copy() + env.pop("DEEPPAPERNOTE_OBSIDIAN_VAULT", None) + env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "DeepPaperNote_output" + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# Language Mismatch\n" + lint_args = formal_save_args(tmp_path, note_text)[:2] + + result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--title", + "Language Mismatch", + "--content", + note_text, + *lint_args, + "--figure-decisions", + str(decisions_path), + ], + cwd=tmp_path, + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "does not match resolved output_language zh-CN" in result.stderr + assert not (tmp_path / "DeepPaperNote_output").exists() def test_write_note_refuses_when_reference_hygiene_gate_fails(tmp_path: Path) -> None: @@ -282,6 +1420,7 @@ def test_write_note_refuses_when_reference_hygiene_gate_fails(tmp_path: Path) -> lint_path.write_text( json.dumps( { + "output_language": "zh-CN", "passes_basic_structure": True, "passes_style_gate": True, "passes_math_gate": True, @@ -322,6 +1461,7 @@ def test_write_note_refuses_runtime_artifact_reference_without_lint_json(tmp_pat env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "DeepPaperNote_output" env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + note_text = "# Direct Reference Hygiene Test\n\n## 引用\n\n- /private/tmp/dpn-test-runs/candidate/artifacts/llama_source_manifest.json\n" result = subprocess.run( [ sys.executable, @@ -329,7 +1469,8 @@ def test_write_note_refuses_runtime_artifact_reference_without_lint_json(tmp_pat "--title", "Direct Reference Hygiene Test", "--content", - "# Direct Reference Hygiene Test\n\n## 引用\n\n- /private/tmp/dpn-test-runs/candidate/artifacts/llama_source_manifest.json\n", + note_text, + *formal_save_args(tmp_path, note_text), ], cwd=tmp_path, env=env, @@ -342,11 +1483,39 @@ def test_write_note_refuses_runtime_artifact_reference_without_lint_json(tmp_pat assert not (tmp_path / "DeepPaperNote_output").exists() +def test_write_note_requires_integrity_artifacts_before_save(tmp_path: Path) -> None: + env = os.environ.copy() + env.pop("DEEPPAPERNOTE_OBSIDIAN_VAULT", None) + env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "DeepPaperNote_output" + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + + result = subprocess.run( + [ + sys.executable, + str(WRITE_SCRIPT), + "--title", + "Missing Integrity Artifacts", + "--content", + "# Missing Integrity Artifacts\n", + ], + cwd=tmp_path, + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "Final Note Lint" in result.stderr + assert not (tmp_path / "DeepPaperNote_output").exists() + + def test_write_note_refuses_when_figure_gate_fails(tmp_path: Path) -> None: lint_path = tmp_path / "lint.json" lint_path.write_text( json.dumps( { + "output_language": "zh-CN", "passes_basic_structure": True, "passes_style_gate": True, "passes_math_gate": True, diff --git a/tests/test_common.py b/tests/test_common.py index 1b02b71..7880b5e 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -525,6 +525,52 @@ def test_runtime_config_ignores_read_arxiv_obsidian_vault(tmp_path: Path, monkey assert root == tmp_path / "DeepPaperNote_output" +def test_runtime_config_accepts_complete_environment_without_user_configuration( + tmp_path: Path, monkeypatch +) -> None: + config_path = tmp_path / "missing" / "config.json" + monkeypatch.setenv("DEEPPAPERNOTE_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("DEEPPAPERNOTE_OUTPUT_LANGUAGE", "en") + monkeypatch.setenv("DEEPPAPERNOTE_SAVE_MODE", "workspace") + + config = common.runtime_config() + + assert config["output_language"] == "en" + assert config["save_mode"] == "workspace" + assert not config_path.exists() + + +def test_runtime_config_fills_partial_environment_from_user_configuration( + monkeypatch, +) -> None: + monkeypatch.setenv("DEEPPAPERNOTE_OUTPUT_LANGUAGE", "en") + + config = common.runtime_config() + + assert config["output_language"] == "en" + assert config["save_mode"] == "workspace" + + +def test_runtime_config_uses_complete_obsidian_environment_without_reading_invalid_file( + tmp_path: Path, monkeypatch +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + config_path = tmp_path / "config.json" + config_path.write_text("{broken", encoding="utf-8") + monkeypatch.setenv("DEEPPAPERNOTE_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("DEEPPAPERNOTE_OUTPUT_LANGUAGE", "zh-CN") + monkeypatch.setenv("DEEPPAPERNOTE_SAVE_MODE", "obsidian") + monkeypatch.setenv("DEEPPAPERNOTE_OBSIDIAN_VAULT", str(vault)) + monkeypatch.setenv("DEEPPAPERNOTE_PAPERS_DIR", "Research/Papers") + + config = common.runtime_config() + + assert config["obsidian_vault"] == str(vault) + assert config["papers_dir"] == "Research/Papers" + assert config_path.read_text(encoding="utf-8") == "{broken" + + def test_resolve_obsidian_note_path_in_workspace_mode(tmp_path: Path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) config = { @@ -681,7 +727,10 @@ def test_existing_domain_dirs_excludes_root_level_paper_folder(tmp_path: Path) - (papers / "大模型").mkdir(parents=True) paper_dir = papers / "Attention_Is_All_You_Need" paper_dir.mkdir(parents=True) - (paper_dir / "Attention_Is_All_You_Need.md").write_text("# note\n", encoding="utf-8") + (paper_dir / "Attention_Is_All_You_Need.zh-CN.md").write_text( + "# note\n", + encoding="utf-8", + ) config = { "obsidian_vault": str(vault), diff --git a/tests/test_contracts_consistency.py b/tests/test_contracts_consistency.py index ac7518b..576a2bd 100644 --- a/tests/test_contracts_consistency.py +++ b/tests/test_contracts_consistency.py @@ -52,6 +52,15 @@ def bundle(**kwargs: object) -> dict: + if not kwargs.get("figures_wrapper"): + kwargs["figures_wrapper"] = {"output_language": "zh-CN", "figure_plan": {}} + if not kwargs.get("figure_decisions_wrapper"): + kwargs["figure_decisions_wrapper"] = {"output_language": "zh-CN", "decisions": []} + elif "output_language" not in kwargs["figure_decisions_wrapper"]: + kwargs["figure_decisions_wrapper"] = { + "output_language": "zh-CN", + **kwargs["figure_decisions_wrapper"], + } source_manifest = dict(kwargs.pop("source_manifest", {}) or {}) source_manifest["identity_contract"] = { "artifact_type": "canonical_identity", @@ -92,6 +101,53 @@ def test_topic_references_do_not_redefine_canonical_workflow() -> None: assert "three-stage model-first pipeline" not in evidence_first +def test_skill_owns_one_final_user_report_contract() -> None: + skill = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8") + + assert skill.count("Final user report:") == 1 + for required in ( + "user's conversation language", + "actual saved domain", + "report that directory's existing domain", + "materialized and retained-placeholder figure/table counts", + "Final Note Lint `note_sha256`", + "derive every claim from current-run artifacts", + ): + assert required in skill + + for reference in (SKILL_ROOT / "references").glob("*.md"): + assert "Final user report:" not in reference.read_text(encoding="utf-8") + + +def test_skill_owns_one_fail_closed_output_language_contract() -> None: + skill = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8") + output_reference = (SKILL_ROOT / "references" / "output-language.md").read_text( + encoding="utf-8" + ) + evidence_reference = (SKILL_ROOT / "references" / "evidence-first.md").read_text( + encoding="utf-8" + ) + + assert skill.count("## Language Integrity Contract") == 1 + for stage in ( + "Figure Plan", + "Figure/Table Decisions", + "Synthesis Bundle", + "Note Plan", + "Grounding Lint", + "Final Note Lint", + "Final Quality Review", + "Final Readability Review", + "Formal Save", + ): + assert stage in skill + assert "source_manifest.language_hint" in skill + assert "note_sha256" in skill + assert "before any save side effect" in skill + assert "`SKILL.md` owns the cross-stage Language Integrity Contract" in output_reference + assert '"output_language": "zh-CN"' in evidence_reference + + def test_codex_adapter_stays_thin() -> None: adapter = (SKILL_ROOT / "agents" / "openai.yaml").read_text(encoding="utf-8") @@ -268,6 +324,7 @@ def test_bundle_exposes_exact_note_plan_types_and_grounding_command() -> None: note_plan_contract = writing_contract["note_plan_contract"] assert note_plan_contract["field_types"] == { + "output_language": "string", **{field: "string" for field in NOTE_PLAN_STRING_FIELDS}, **{field: "array" for field in NOTE_PLAN_LIST_FIELDS}, } @@ -565,11 +622,23 @@ def test_skill_owns_formal_save_state_policy() -> None: assert marker not in obsidian_format_text assert "After such a refusal" not in skill_text assert "do not switch to workspace" in skill_text - assert "explicitly chooses not to use a vault" in skill_text + assert "`save_mode=workspace`" in skill_text + assert "references/user-configuration.md" in skill_text for policy_phrase in ("permission escalation", "workspace fallback", "explicit user consent"): assert policy_phrase not in obsidian_format_text +def test_skill_requires_programmatic_save_target_admission_before_drafting() -> None: + text = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8") + workflow = text.split("## Workflow", 1)[1].split("## Core Execution Contract", 1)[0] + + assert "Save Target Admission" in workflow + assert "write_obsidian_note.py --preflight" in workflow + assert workflow.index("Save Target Admission") < workflow.index("plan figure placement") + assert "same_language_note_exists" in workflow + assert "--expected-existing-note-sha256" in workflow + + def test_topic_references_keep_separate_note_plan_responsibilities() -> None: evidence_first = (SKILL_ROOT / "references" / "evidence-first.md").read_text( encoding="utf-8" @@ -671,7 +740,8 @@ def test_evidence_first_note_plan_example_matches_lint_contract() -> None: assert match is not None example = json.loads(match.group(1)) - assert tuple(example.keys()) == NOTE_PLAN_REQUIRED_FIELDS + assert tuple(example.keys()) == ("output_language", *NOTE_PLAN_REQUIRED_FIELDS) + assert example["output_language"] == "zh-CN" assert all(isinstance(example[field], str) for field in NOTE_PLAN_REQUIRED_FIELDS[:3]) assert all(isinstance(example[field], list) for field in NOTE_PLAN_REQUIRED_FIELDS[3:]) assert example["paper_type"] in PAPER_TYPE_VALUES diff --git a/tests/test_extract_evidence.py b/tests/test_extract_evidence.py index 0513d4f..298febf 100644 --- a/tests/test_extract_evidence.py +++ b/tests/test_extract_evidence.py @@ -24,6 +24,10 @@ def bundle(*, metadata: dict, source_manifest: dict | None = None, **kwargs) -> dict: + if not kwargs.get("figures_wrapper"): + kwargs["figures_wrapper"] = {"output_language": "zh-CN", "figure_plan": {}} + if not kwargs.get("figure_decisions_wrapper"): + kwargs["figure_decisions_wrapper"] = {"output_language": "zh-CN", "decisions": []} manifest = dict(source_manifest or {}) if "identity_contract" not in manifest: title = str(metadata.get("title", "")) diff --git a/tests/test_fetch_pdf.py b/tests/test_fetch_pdf.py index 50b2e24..2bd7b33 100644 --- a/tests/test_fetch_pdf.py +++ b/tests/test_fetch_pdf.py @@ -163,6 +163,54 @@ def fake_http_get_bytes(url: str) -> bytes: assert payload["paper_id"] == "doi:10.1234/trusted" assert payload["title"] == "Trusted Paper" assert payload["pdf_url"] == "https://example.test/trusted.pdf" + assert payload["source_sha256"] == ( + "0cdae4135f26b10e67e6c4972b573c29913948373e8dc6210624b283b99c830d" + ) + + +def test_fetch_pdf_hashes_the_original_bound_local_pdf(tmp_path: Path) -> None: + pdf_path = tmp_path / "original.pdf" + metadata_path = tmp_path / "metadata.json" + identity_path = tmp_path / "identity.json" + output_path = tmp_path / "fetch.json" + pdf_path.write_bytes(b"%PDF-1.7\nlocal-original") + metadata_path.write_text( + json.dumps({"status": "ok", "script": "collect_metadata.py"}), + encoding="utf-8", + ) + identity_path.write_text( + json.dumps( + { + "status": "ok", + "script": "build_identity_contract.py", + "artifact_type": "canonical_identity", + "schema_version": 2, + "paper_id": "paper:local-original", + "identity_verdict": "accepted", + "work_level_identity": {"title": "Local Original"}, + "source_manifestation": {"source_kind": "local_pdf"}, + "bound_sources": [{"kind": "local_pdf", "value": str(pdf_path)}], + } + ), + encoding="utf-8", + ) + + fetch_pdf.main( + [ + "--input", + str(metadata_path), + "--identity", + str(identity_path), + "--output", + str(output_path), + ] + ) + + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["pdf_path"] == str(pdf_path.resolve()) + assert payload["source_sha256"] == ( + "f131d003c562388dc22cf50d93c60ae52a598b4b2c72963adeb08d812aa92fca" + ) def test_fetch_pdf_refuses_legacy_identity_without_bound_sources( diff --git a/tests/test_figure_table_decisions.py b/tests/test_figure_table_decisions.py index af7cd41..4d76a55 100644 --- a/tests/test_figure_table_decisions.py +++ b/tests/test_figure_table_decisions.py @@ -40,6 +40,7 @@ def run_decisions( figures: dict, assets: dict | None = None, ) -> dict: + figures = {"output_language": "zh-CN", **figures} source_path = write_json(tmp_path / "source_manifest.json", source_manifest) figures_path = write_json(tmp_path / "figures.json", figures) output_path = tmp_path / "figure_table_decisions.json" @@ -66,6 +67,7 @@ def run_review_decisions( *, check: bool = True, ) -> tuple[subprocess.CompletedProcess[str], dict | None]: + decisions = {"output_language": "zh-CN", **decisions} decisions_path = write_json(tmp_path / "review_decisions.json", decisions) output_path = tmp_path / "reviewed_decisions.json" result = subprocess.run( @@ -113,6 +115,7 @@ def test_figure_table_decisions_cover_every_caption(tmp_path: Path) -> None: }, } figures = { + "output_language": "zh-CN", "figure_plan": { "figures": [ { @@ -142,6 +145,36 @@ def test_figure_table_decisions_cover_every_caption(tmp_path: Path) -> None: assert decisions["Table 1"]["decision"] == "placeholder" assert decisions["Figure 2"]["decision"] == "low_priority" assert payload["summary"]["total_items"] == 3 + assert payload["output_language"] == "zh-CN" + + +def test_figure_table_decisions_reject_mismatched_figure_plan_language( + tmp_path: Path, +) -> None: + source_path = write_json(tmp_path / "source_manifest.json", {"captions": {}}) + figures_path = write_json( + tmp_path / "figures.json", + {"output_language": "en", "figure_plan": {"figures": []}}, + ) + + result = subprocess.run( + [ + sys.executable, + str(DECISIONS_SCRIPT), + "--source-manifest", + str(source_path), + "--figures", + str(figures_path), + "--language", + "zh-CN", + ], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "Figure Plan output_language en does not match resolved output_language zh-CN" in result.stderr def test_figure_table_decisions_cover_source_corpus_captions(tmp_path: Path) -> None: @@ -392,6 +425,16 @@ def test_figure_table_decisions_apply_one_normalized_bbox_repair( ], } + result, payload = run_review_decisions( + tmp_path, + {"output_language": "en", **decisions}, + check=False, + ) + assert result.returncode != 0 + assert payload is None + assert "does not match resolved output_language zh-CN" in result.stderr + assert not (tmp_path / "candidate_repair1.png").exists() + _, payload = run_review_decisions(tmp_path, decisions) assert payload is not None decision = payload["decisions"][0] diff --git a/tests/test_lint_grounding.py b/tests/test_lint_grounding.py index a96b25b..090cca9 100644 --- a/tests/test_lint_grounding.py +++ b/tests/test_lint_grounding.py @@ -55,11 +55,15 @@ def run_lint_grounding( bundle: dict | None = None, figure_decisions: dict | None = None, ) -> dict: + if figure_decisions is not None: + figure_decisions = {"output_language": "zh-CN", **figure_decisions} note_plan_path = write_json(tmp_path / "note_plan.json", note_plan) source_manifest_path = write_json(tmp_path / "source_manifest.json", source_manifest) figure_decisions_path = write_json( tmp_path / "figure_decisions.json", - figure_decisions if figure_decisions is not None else {"decisions": []}, + figure_decisions + if figure_decisions is not None + else {"output_language": "zh-CN", "decisions": []}, ) output_path = tmp_path / "grounding.json" cmd = [ @@ -113,6 +117,7 @@ def note_plan_with_sources(sources_by_section: dict[str, list[dict] | list[str]] for section, sources in sources_by_section.items() ] return { + "output_language": "zh-CN", "paper_type": "AI_method", "paper_type_rationale": "method paper", "dominant_domain": "AI", @@ -152,13 +157,30 @@ def grounded_note_plan() -> dict: def slim_bundle() -> dict: return { + "output_language": "zh-CN", "writing_contract": { + "language": "zh-CN", "contracts_by_paper_type": PAPER_TYPE_CONTRACTS, "paper_type_selection": {"source_of_truth": "note_plan.paper_type"}, } } +def test_lint_grounding_rejects_mismatched_note_plan_language(tmp_path: Path) -> None: + plan = grounded_note_plan() + plan["output_language"] = "en" + + result = run_lint_grounding(tmp_path, plan, source_manifest(), slim_bundle()) + failures = [ + item for item in result["issues"] if item["code"] == "output_language_contract_failed" + ] + + assert result["output_language"] == "zh-CN" + assert result["passes_grounding"] is False + assert failures[0]["artifact"] == "Note Plan" + assert "does not match resolved output_language zh-CN" in failures[0]["reason"] + + def test_lint_grounding_accepts_section_id_or_page_range(tmp_path: Path) -> None: result = run_lint_grounding(tmp_path, grounded_note_plan(), source_manifest(), slim_bundle()) diff --git a/tests/test_lint_note.py b/tests/test_lint_note.py index 5ca4761..e42b55d 100644 --- a/tests/test_lint_note.py +++ b/tests/test_lint_note.py @@ -91,6 +91,7 @@ def _valid_note_text() -> str: def _valid_plan_payload() -> dict: return { + "output_language": "zh-CN", "paper_type": "AI_method", "paper_type_rationale": "The paper proposes a model mechanism and evaluates it experimentally.", "dominant_domain": "reasoning", @@ -115,6 +116,15 @@ def _valid_plan_payload() -> dict: } +def _source_manifest_path(tmp_path: Path) -> Path: + path = tmp_path / "source_manifest.json" + path.write_text( + json.dumps({"status": "ok", "source_sha256": "a" * 64}), + encoding="utf-8", + ) + return path + + def test_reference_hygiene_allows_images_doi_arxiv_and_urls() -> None: note = ( _valid_note_text() @@ -957,6 +967,32 @@ def test_note_plan_missing_fails_plan_gate(tmp_path) -> None: assert payload["passes_plan_gate"] is False +def test_note_plan_language_mismatch_fails_plan_gate(tmp_path) -> None: + note_path = tmp_path / "Paper.md" + plan_path = tmp_path / "Paper.plan.json" + note_path.write_text(_valid_note_text(), encoding="utf-8") + plan = _valid_plan_payload() + plan["output_language"] = "en" + plan_path.write_text(json.dumps(plan), encoding="utf-8") + + script_path = Path(__file__).resolve().parents[1] / "skills" / "deeppapernote" / "scripts" / "lint_note.py" + result = subprocess.run( + [sys.executable, str(script_path), "--input", str(note_path)], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + + assert payload["output_language"] == "zh-CN" + assert payload["passes_plan_gate"] is False + assert any( + issue.startswith("planning_output_language_contract_failed:") + for issue in payload["planning_artifact_issues"] + ) + assert len(payload["note_sha256"]) == 64 + + def test_mechanical_translation_artifacts_fail_style_gate(tmp_path) -> None: note_path = tmp_path / "Paper.md" plan_path = tmp_path / "Paper.plan.json" @@ -1023,8 +1059,9 @@ def test_note_plan_empty_required_values_fail_plan_gate(tmp_path) -> None: note_path.write_text(_valid_note_text(), encoding="utf-8") plan_path.write_text( json.dumps( - { - "paper_type": "", + { + "output_language": "zh-CN", + "paper_type": "", "paper_type_rationale": "", "dominant_domain": " ", "must_cover": [], @@ -1078,8 +1115,9 @@ def test_note_plan_explicit_not_reported_entries_pass_plan_gate(tmp_path) -> Non note_path.write_text(_valid_note_text(), encoding="utf-8") plan_path.write_text( json.dumps( - { - "paper_type": "AI_method", + { + "output_language": "zh-CN", + "paper_type": "AI_method", "paper_type_rationale": "The paper proposes a model mechanism and evaluates it experimentally.", "dominant_domain": "reasoning", "must_cover": ["方法主线"], @@ -1119,11 +1157,13 @@ def test_note_plan_explicit_not_reported_entries_pass_plan_gate(tmp_path) -> Non def test_write_obsidian_note_refuses_failed_plan_gate(tmp_path) -> None: + (tmp_path / "vault").mkdir() lint_path = tmp_path / "lint.json" lint_path.write_text( json.dumps( - { - "passes_basic_structure": True, + { + "output_language": "zh-CN", + "passes_basic_structure": True, "passes_style_gate": True, "passes_math_gate": True, "passes_figure_gate": True, @@ -1144,6 +1184,8 @@ def test_write_obsidian_note_refuses_failed_plan_gate(tmp_path) -> None: "# Plan Gate Paper", "--lint-json", str(lint_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(tmp_path / "vault"), ], @@ -1157,11 +1199,13 @@ def test_write_obsidian_note_refuses_failed_plan_gate(tmp_path) -> None: def test_write_obsidian_note_reports_lint_warning_details(tmp_path) -> None: + (tmp_path / "vault").mkdir() lint_path = tmp_path / "lint.json" lint_path.write_text( json.dumps( - { - "passes_basic_structure": True, + { + "output_language": "zh-CN", + "passes_basic_structure": True, "passes_style_gate": False, "passes_math_gate": True, "warnings": ["mixed_language_lines_present"], @@ -1182,6 +1226,8 @@ def test_write_obsidian_note_reports_lint_warning_details(tmp_path) -> None: "# Style Gate Paper", "--lint-json", str(lint_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(tmp_path / "vault"), ], @@ -1275,11 +1321,13 @@ def test_real_image_embed_counts_as_figure_marker_in_full_lint(tmp_path) -> None def test_write_obsidian_note_refuses_failed_substantive_gate(tmp_path) -> None: + (tmp_path / "vault").mkdir() lint_path = tmp_path / "lint.json" lint_path.write_text( json.dumps( - { - "passes_basic_structure": True, + { + "output_language": "zh-CN", + "passes_basic_structure": True, "passes_style_gate": True, "passes_math_gate": True, "passes_figure_gate": True, @@ -1301,6 +1349,8 @@ def test_write_obsidian_note_refuses_failed_substantive_gate(tmp_path) -> None: "# Substantive Gate Paper", "--lint-json", str(lint_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(tmp_path / "vault"), ], @@ -1312,8 +1362,10 @@ def test_write_obsidian_note_refuses_failed_substantive_gate(tmp_path) -> None: assert "substantive content gate failed" in result.stderr -def passing_lint_payload() -> dict: +def passing_lint_payload(note_text: str) -> dict: return { + "output_language": "zh-CN", + "note_sha256": hashlib.sha256(note_text.encode("utf-8")).hexdigest(), "passes_basic_structure": True, "passes_style_gate": True, "passes_math_gate": True, @@ -1346,12 +1398,14 @@ def test_write_obsidian_note_materializes_insert_decision(tmp_path) -> None: source_image = tmp_path / "page_001_fig_figure_1.png" source_image.write_bytes(b"fake-png") digest = hashlib.sha256(b"fake-png").hexdigest() + note_text = "# Figure Insert Paper\n\n![Figure 1](images/page_001_fig_figure_1.png)\n*Fig. 1 caption.*\n" lint_path = tmp_path / "lint.json" - lint_path.write_text(json.dumps(passing_lint_payload()), encoding="utf-8") + lint_path.write_text(json.dumps(passing_lint_payload(note_text)), encoding="utf-8") decisions_path = tmp_path / "figure_decisions.json" decisions_path.write_text( json.dumps( { + "output_language": "zh-CN", "decisions": [ { "source_id": "Figure 1", @@ -1389,11 +1443,13 @@ def test_write_obsidian_note_materializes_insert_decision(tmp_path) -> None: "--subdir", "Research/Papers/Figure Insert Paper", "--content", - "# Figure Insert Paper\n\n![Figure 1](images/page_001_fig_figure_1.png)\n*Fig. 1 caption.*\n", + note_text, "--lint-json", str(lint_path), "--figure-decisions", str(decisions_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(vault), "--output", @@ -1417,12 +1473,14 @@ def test_write_obsidian_note_rejects_stale_reviewed_insert_bytes(tmp_path) -> No source_image = tmp_path / "page_001_fig_figure_1.png" reviewed_digest = hashlib.sha256(b"reviewed").hexdigest() source_image.write_bytes(b"changed-after-review") + note_text = "# Stale Figure Review\n\n![Figure 1](images/page_001_fig_figure_1.png)\n*Fig. 1 caption.*\n" lint_path = tmp_path / "lint.json" - lint_path.write_text(json.dumps(passing_lint_payload()), encoding="utf-8") + lint_path.write_text(json.dumps(passing_lint_payload(note_text)), encoding="utf-8") decisions_path = tmp_path / "figure_decisions.json" decisions_path.write_text( json.dumps( { + "output_language": "zh-CN", "decisions": [ { "source_id": "Figure 1", @@ -1461,11 +1519,13 @@ def test_write_obsidian_note_rejects_stale_reviewed_insert_bytes(tmp_path) -> No "--title", "Stale Figure Review", "--content", - "# Stale Figure Review\n\n![Figure 1](images/page_001_fig_figure_1.png)\n*Fig. 1 caption.*\n", + note_text, "--lint-json", str(lint_path), "--figure-decisions", str(decisions_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(vault), ], @@ -1482,12 +1542,14 @@ def test_write_obsidian_note_rejects_unreferenced_insert_decision(tmp_path) -> N vault.mkdir() source_image = tmp_path / "page_001_fig_figure_1.png" source_image.write_bytes(b"fake-png") + note_text = "# Figure Insert Paper\n\n正文没有引用图片。\n" lint_path = tmp_path / "lint.json" - lint_path.write_text(json.dumps(passing_lint_payload()), encoding="utf-8") + lint_path.write_text(json.dumps(passing_lint_payload(note_text)), encoding="utf-8") decisions_path = tmp_path / "figure_decisions.json" decisions_path.write_text( json.dumps( { + "output_language": "zh-CN", "decisions": [ { "source_id": "Figure 1", @@ -1510,11 +1572,13 @@ def test_write_obsidian_note_rejects_unreferenced_insert_decision(tmp_path) -> N "--title", "Figure Insert Paper", "--content", - "# Figure Insert Paper\n\n正文没有引用图片。\n", + note_text, "--lint-json", str(lint_path), "--figure-decisions", str(decisions_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(vault), ], @@ -1531,12 +1595,14 @@ def test_write_obsidian_note_rejects_plain_path_for_insert_decision(tmp_path) -> vault.mkdir() source_image = tmp_path / "page_001_fig_figure_1.png" source_image.write_bytes(b"fake-png") + note_text = "# Figure Insert Paper\n\n正文只提到 images/page_001_fig_figure_1.png 这个路径。\n" lint_path = tmp_path / "lint.json" - lint_path.write_text(json.dumps(passing_lint_payload()), encoding="utf-8") + lint_path.write_text(json.dumps(passing_lint_payload(note_text)), encoding="utf-8") decisions_path = tmp_path / "figure_decisions.json" decisions_path.write_text( json.dumps( { + "output_language": "zh-CN", "decisions": [ { "source_id": "Figure 1", @@ -1559,11 +1625,13 @@ def test_write_obsidian_note_rejects_plain_path_for_insert_decision(tmp_path) -> "--title", "Figure Insert Paper", "--content", - "# Figure Insert Paper\n\n正文只提到 images/page_001_fig_figure_1.png 这个路径。\n", + note_text, "--lint-json", str(lint_path), "--figure-decisions", str(decisions_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(vault), ], @@ -1580,12 +1648,14 @@ def test_write_obsidian_note_rejects_unsafe_insert_filename(tmp_path) -> None: vault.mkdir() source_image = tmp_path / "page_001_fig_figure_1.png" source_image.write_bytes(b"fake-png") + note_text = "# Figure Insert Paper\n\n![Figure 1](images/../escaped.png)\n*Fig. 1 caption.*\n" lint_path = tmp_path / "lint.json" - lint_path.write_text(json.dumps(passing_lint_payload()), encoding="utf-8") + lint_path.write_text(json.dumps(passing_lint_payload(note_text)), encoding="utf-8") decisions_path = tmp_path / "figure_decisions.json" decisions_path.write_text( json.dumps( { + "output_language": "zh-CN", "decisions": [ { "source_id": "Figure 1", @@ -1608,11 +1678,13 @@ def test_write_obsidian_note_rejects_unsafe_insert_filename(tmp_path) -> None: "--title", "Figure Insert Paper", "--content", - "# Figure Insert Paper\n\n![Figure 1](images/../escaped.png)\n*Fig. 1 caption.*\n", + note_text, "--lint-json", str(lint_path), "--figure-decisions", str(decisions_path), + "--source-manifest", + str(_source_manifest_path(tmp_path)), "--vault", str(vault), ], @@ -1688,8 +1760,9 @@ def test_inspect_note_plan_reports_invalid_field_types(tmp_path) -> None: plan_path = tmp_path / "note.plan.json" plan_path.write_text( json.dumps( - { - "paper_type": "AI_method", + { + "output_language": "zh-CN", + "paper_type": "AI_method", "paper_type_rationale": "The paper proposes a model mechanism.", "dominant_domain": "reasoning", "must_cover": "method", @@ -1717,8 +1790,9 @@ def test_inspect_note_plan_reports_empty_section_plan(tmp_path) -> None: plan_path = tmp_path / "note.plan.json" plan_path.write_text( json.dumps( - { - "paper_type": "AI_method", + { + "output_language": "zh-CN", + "paper_type": "AI_method", "paper_type_rationale": "The paper proposes a model mechanism.", "dominant_domain": "reasoning", "must_cover": [], @@ -1758,8 +1832,9 @@ def test_inspect_note_plan_accepts_valid_plan(tmp_path) -> None: plan_path = tmp_path / "note.plan.json" plan_path.write_text( json.dumps( - { - "paper_type": "AI_method", + { + "output_language": "zh-CN", + "paper_type": "AI_method", "paper_type_rationale": "The paper proposes a model mechanism.", "dominant_domain": "reasoning", "must_cover": ["方法主线"], diff --git a/tests/test_output_language.py b/tests/test_output_language.py new file mode 100644 index 0000000..7ce679a --- /dev/null +++ b/tests/test_output_language.py @@ -0,0 +1,510 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from build_synthesis_bundle import compact_writing_contract +from lint_grounding import validate_note_plan +from localization import ( + normalize_output_language, + require_artifact_output_language, + required_sections, +) +from plan_figures import build_figure_items + +ENGLISH_SECTIONS = ( + "Core Information", + "Abstract", + "Contributions", + "One-Sentence Summary", + "Research Question", + "Data and Task Definition", + "Method", + "Key Results", + "Deep Analysis", + "Limitations", + "Research Notes", + "References", +) + + +def english_note() -> str: + return """--- +tags: + - papers/methods +aliases: + - "Auditable Tool Use" +date: 2024 +doi: 10.1234/example +--- + +# Auditable Tool Use + +## Core Information + +- Title: Auditable Tool Use +- Authors: Smith et al. +- Publication date: 2024 +- Venue: Example Journal +- DOI: 10.1234/example +- Paper type: AI method + +## Abstract + +The paper develops an auditable state machine for multi-step question answering and evaluates whether explicit failure records improve answer reliability. + +## Contributions + +- It joins evidence selection and tool-state tracking in one execution record, preventing failed evidence from silently becoming trusted input. +- It adds explicit rollback states that distinguish missing evidence from reasoning errors and make the final answer traceable. + +## One-Sentence Summary + +An explicit tool-state record reduces error propagation in multi-step question answering. + +## Research Question + +How can a multi-step question-answering system remain traceable when retrieval is incomplete, external tools fail, or intermediate results are misused? + +## Data and Task Definition + +The input contains a question, candidate evidence, and available tools; the output contains an answer, a state trace, and a failure label when completion is unsupported. + +## Method + +### Mechanism Flow + +1. **Input:** A question and candidate evidence. **Operation:** Extract relevant evidence. **Output:** A grounded initial state. +2. **Input:** The current state and tool registry. **Operation:** Align the request with an available tool. **Output:** A planned call. +3. **Input:** Tool output and confidence. **Operation:** Update or roll back the state. **Output:** An auditable execution record. +4. **Input:** The final state. **Operation:** Decode an answer or refusal. **Output:** A response with provenance. + +> [!figure] Figure 1 System overview +> Suggested location: Method +> Why it matters: The figure shows how evidence and tool states move through the execution chain. +> Current status: Placeholder retained; the extracted crop is incomplete and cannot be interpreted independently. + +## Key Results + +Across three datasets, answer accuracy increased from 71.2% to 78.5%, while untraceable errors fell from 18% to 9%. + +## Deep Analysis + +The important contribution is not only the score increase. Failed calls become inspectable evidence rather than hidden intermediate state, which supports auditing and targeted recovery. + +## Limitations + +The evaluation covers English question-answering data and a narrow tool set, so it does not establish robustness for multimodal tools or high-latency services. + +## Research Notes + +The state-record design is reusable in evidence-first paper workflows because it separates missing source material from model interpretation failure. + +## References + +- Smith et al. (2024). Auditable Tool Use for Multi-hop Question Answering. DOI: 10.1234/example +""" + + +def plan_payload() -> dict: + return { + "output_language": "en", + "paper_type": "AI_method", + "paper_type_rationale": "The paper proposes and evaluates a model mechanism.", + "dominant_domain": "reasoning", + "must_cover": ["Method"], + "key_numbers": ["78.5%"], + "real_comparisons": ["71.2% versus 78.5%"], + "central_claims": [{ + "claim": "The method improves traceability.", + "supporting_evidence": [{"section_id": "sec:results"}], + "what_it_actually_proves": "The reported protocol records tool states.", + "what_it_does_not_prove": "It does not prove production robustness.", + }], + "claim_boundaries": ["Evidence is limited to the reported workflow."], + "negative_or_limiting_results": ["Multimodal tools were not tested."], + "mechanism_result_map": ["Rollback states explain fewer untraceable errors."], + "comparative_positioning": ["Compared with answer-only baselines."], + "reuse_takeaways": ["Track failure state explicitly."], + "followup_questions": ["Test missing and delayed tool outputs."], + "section_plan": [{"section": "Method", "evidence_sources": [{"section_id": "sec:method"}]}], + } + + +def test_language_aliases_and_invalid_value() -> None: + assert normalize_output_language("English") == "en" + assert normalize_output_language("zh") == "zh-CN" + with pytest.raises(ValueError): + normalize_output_language("fr") + + +def test_artifact_language_requires_exact_supported_value() -> None: + with pytest.raises(ValueError, match="requires output_language"): + require_artifact_output_language({}, "Note Plan", "zh-CN") + with pytest.raises(ValueError, match="requires output_language"): + require_artifact_output_language( + {"output_language": "English"}, + "Note Plan", + "en", + ) + + +def test_english_contract_exposes_localized_schema() -> None: + contract = compact_writing_contract("en") + assert contract["language"] == "en" + assert tuple(contract["must_include_sections"]) == ENGLISH_SECTIONS + assert tuple(required_sections("en")) == ENGLISH_SECTIONS + assert contract["mechanism_flow_heading"] == "Mechanism Flow" + assert contract["core_info_fields"] == [ + "Title", + "Translated title", + "Authors", + "Institutions", + "Publication date", + "Venue", + "DOI", + "arXiv", + "Paper link", + "Code / Project", + "Data / Resources", + "Paper type", + ] + assert contract["figure_labels"] == { + "location": "Suggested location:", + "reason": "Why it matters:", + "status": "Current status:", + "original_caption": "Original paper item:", + } + assert contract["abstract_contract"] == { + "source": "source_abstract", + "requirement": "faithful_rendering_in_output_language", + "forbidden_additions": [ + "later_contribution_claims", + "later_result_interpretation", + "hindsight_judgment", + ], + } + assert not any("\u4e00" <= character <= "\u9fff" for character in json.dumps(contract, ensure_ascii=False)) + + +def test_english_grounding_accepts_english_section_plan() -> None: + plan = plan_payload() + plan["central_claims"][0]["supporting_evidence"] = [{"section_id": "sec:method"}] + plan["section_plan"] = [ + { + "section": section, + "focus": f"Explain the paper-specific evidence and analytical role of {section}.", + "evidence_sources": [{"section_id": "sec:method"}], + } + for section in ( + "Research Question", + "Data and Task Definition", + "Method", + "Key Results", + "Deep Analysis", + "Limitations", + ) + ] + manifest = { + "coverage": {"total_pages": 10, "text_truncated": False}, + "sections": [{"section_id": "sec:method", "title": "Method", "page_start": 1, "page_end": 10}], + "pages": [], + } + assert validate_note_plan(plan, manifest, "en") == [] + + +def test_english_figure_plan_uses_english_targets_and_reasons() -> None: + items = build_figure_items( + {"figure_captions": [{"id": "Figure 1", "caption": "Overview of the system architecture."}]}, + language="en", + ) + assert items[0]["section"] == "Mechanism Flow" + assert "visual" in items[0]["reason"].lower() + assert not any("\u4e00" <= character <= "\u9fff" for character in json.dumps(items, ensure_ascii=False)) + + +def test_english_note_passes_every_lint_gate_from_user_configuration( + tmp_path: Path, configured_user_home: Path +) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note_path.write_text(english_note(), encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + configured_user_home.write_text( + json.dumps({"output_language": "en", "save_mode": "workspace"}), + encoding="utf-8", + ) + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [ + sys.executable, + str(script), + "--input", + str(note_path), + "--plan-file", + str(plan_path), + "--output", + str(output_path), + ], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["output_language"] == "en" + assert payload["warnings"] == [] + assert all(value is True for key, value in payload.items() if key.startswith("passes_")) + + +def test_english_lint_rejects_chinese_prose(tmp_path: Path) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note_path.write_text(english_note().replace("The important contribution", "这项工作的 contribution"), encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + subprocess.run([sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], check=True) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_style_gate"] is False + assert "mixed_language_lines_present" in payload["warnings"] + + +@pytest.mark.parametrize( + ("original", "mixed"), + [ + ("## Method", "## 方法 Method"), + ("- Title: Auditable Tool Use", "- 标题 Title: Auditable Tool Use"), + ("> Why it matters: The figure shows how evidence and tool states move through the execution chain.", "> Why it matters: 这张图 shows how evidence moves."), + ("*Original paper item: Figure 1. System overview.*", "*Original paper item: Figure 1. 这是系统概览.*"), + ], +) +def test_english_lint_rejects_chinese_structure_labels_and_captions( + tmp_path: Path, + original: str, + mixed: str, +) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note = english_note() + if original.startswith("*Original paper item:"): + note = note.replace( + "> [!figure] Figure 1 System overview", + "![Figure 1](images/figure_1.png)\n" + original, + ) + note_path.write_text(note.replace(original, mixed), encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_style_gate"] is False + assert "mixed_language_lines_present" in payload["warnings"] + + +def test_english_lint_limits_chinese_exemptions_to_marked_source_spans(tmp_path: Path) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + allowed = english_note().replace( + "- Title: Auditable Tool Use", + "- Title: `可审计工具使用`", + ).replace( + "The important contribution is not only the score increase.", + "The benchmark also evaluates [通义千问](https://example.org/qwen), and the source defines $\\operatorname{输入}=x$.", + ).replace( + "- Smith et al. (2024). Auditable Tool Use for Multi-hop Question Answering. DOI: 10.1234/example", + "- Smith et al. (2024). [可审计工具使用](https://example.org/paper). DOI: 10.1234/example", + ) + "\n```json\n{\"label\": \"原始代码输出\"}\n```\n" + note_path.write_text(allowed, encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_style_gate"] is True + + note_path.write_text( + allowed.replace("- Title: `可审计工具使用`", "- Title: `可审计工具使用` 这是额外说明"), + encoding="utf-8", + ) + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_style_gate"] is False + assert payload["mixed_language_issues"][0]["reason"] == "non_english_text_present" + + +@pytest.mark.parametrize( + "disguised", + [ + "`这是一整句中文分析不是稳定专有名词`", + "`这个方法显著提高了性能`", + "$\\text{这是一整句中文分析不是公式标签}=x$", + "$\\text{方法提高了性能}=x$", + "Plain prose cannot use \\operatorname{输入} outside math delimiters.", + ], +) +def test_english_lint_rejects_chinese_prose_disguised_as_bounded_span( + tmp_path: Path, + disguised: str, +) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note_path.write_text( + english_note().replace( + "The important contribution is not only the score increase.", + disguised, + ), + encoding="utf-8", + ) + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_style_gate"] is False + + +@pytest.mark.parametrize( + ("original", "invalid"), + [ + ("- Authors: Smith et al.", "- Author: Smith et al."), + ("### Mechanism Flow", "### mechanism flow"), + ], +) +def test_english_lint_requires_exact_labels_and_mechanism_heading( + tmp_path: Path, + original: str, + invalid: str, +) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note_path.write_text(english_note().replace(original, invalid), encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert not all(value is True for key, value in payload.items() if key.startswith("passes_")) + + +def test_english_lint_requires_complete_top_level_heading_order(tmp_path: Path) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note = english_note().replace("## Method", "## TEMP").replace("## Key Results", "## Method").replace("## TEMP", "## Key Results") + note_path.write_text(note, encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_basic_structure"] is False + assert "top_level_section_profile_invalid" in payload["warnings"] + + +def test_english_lint_does_not_extend_unclosed_code_exemption_to_rest_of_note( + tmp_path: Path, +) -> None: + note_path = tmp_path / "paper.md" + plan_path = tmp_path / "paper.plan.json" + output_path = tmp_path / "lint.json" + note_path.write_text(english_note() + "\n```text\n这是未闭合代码块后的中文\n", encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + script = Path(__file__).resolve().parents[1] / "skills/deeppapernote/scripts/lint_note.py" + + subprocess.run( + [sys.executable, str(script), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(output_path)], + check=True, + ) + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["passes_style_gate"] is False + assert "mixed_language_lines_present" in payload["warnings"] + + +def test_english_local_pdf_pipeline_lints_and_formally_saves_note_with_images_dir( + tmp_path: Path, +) -> None: + fitz = pytest.importorskip("fitz") + project_root = Path(__file__).resolve().parents[1] + scripts = project_root / "skills/deeppapernote/scripts" + pdf_path = tmp_path / "paper.pdf" + document = fitz.open() + page = document.new_page() + source_abstract = "The paper develops an auditable state machine for multi-step question answering and evaluates whether explicit failure records improve answer reliability." + page.insert_textbox( + fitz.Rect(72, 72, 540, 500), + f"Abstract\n{source_abstract}\nMethod\nThe system records tool states.\nResults\nAccuracy is 78.5%.", + ) + document.save(pdf_path) + document.close() + + workdir = tmp_path / "run" + subprocess.run( + [sys.executable, str(scripts / "run_pipeline.py"), "--input", str(pdf_path), "--workdir", str(workdir), "--prefix", "paper", "--language", "en"], + check=True, + ) + assert json.loads((workdir / "paper_bundle.json").read_text(encoding="utf-8"))["output_language"] == "en" + raw_text = " ".join( + json.loads(line)["text"] + for line in (workdir / "paper_raw_sections.jsonl").read_text(encoding="utf-8").splitlines() + ) + assert source_abstract in " ".join(raw_text.split()) + + note_path = workdir / "paper.md" + plan_path = workdir / "paper.plan.json" + lint_path = workdir / "paper_lint.json" + note_path.write_text(english_note(), encoding="utf-8") + plan_path.write_text(json.dumps(plan_payload()), encoding="utf-8") + subprocess.run( + [sys.executable, str(scripts / "lint_note.py"), "--language", "en", "--input", str(note_path), "--plan-file", str(plan_path), "--output", str(lint_path)], + check=True, + ) + lint = json.loads(lint_path.read_text(encoding="utf-8")) + assert all(value is True for key, value in lint.items() if key.startswith("passes_")) + + save_root = tmp_path / "save" + save_root.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "saved" + saved = subprocess.run( + [sys.executable, str(scripts / "write_obsidian_note.py"), "--language", "en", "--title", "English Local PDF", "--content-file", str(note_path), "--lint-json", str(lint_path), "--figure-decisions", str(workdir / "paper_figure_table_decisions.json")], + cwd=save_root, + env=env, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(saved.stdout) + saved_note = Path(payload["note_path"]) + images_dir = Path(payload["images_dir"]) + assert payload["output_language"] == "en" + assert saved_note.read_text(encoding="utf-8") == english_note() + assert source_abstract in saved_note.read_text(encoding="utf-8") + assert images_dir == saved_note.parent / "images" + assert images_dir.is_dir() diff --git a/tests/test_run_pipeline_manifest.py b/tests/test_run_pipeline_manifest.py index 29c4c9d..671b5c8 100644 --- a/tests/test_run_pipeline_manifest.py +++ b/tests/test_run_pipeline_manifest.py @@ -1,6 +1,8 @@ from __future__ import annotations +import hashlib import json +import os import subprocess import sys from pathlib import Path @@ -16,6 +18,10 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] RUN_PIPELINE_SCRIPT = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "run_pipeline.py" +EXTRACT_SOURCE_SCRIPT = ( + PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "extract_source_text.py" +) +WRITE_NOTE_SCRIPT = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" / "write_obsidian_note.py" def write_test_pdf(path: Path) -> None: @@ -62,6 +68,7 @@ def test_run_pipeline_emits_manifest_raw_decisions_and_lightweight_bundle(tmp_pa identity_trace_path = workdir / "paper_identity_repair_trace.json" raw_sections_path = workdir / "paper_raw_sections.jsonl" evidence_path = workdir / "paper_evidence.json" + figures_path = workdir / "paper_figures.json" decisions_path = workdir / "paper_figure_table_decisions.json" bundle_path = workdir / "paper_bundle.json" assert identity_path.exists() @@ -76,6 +83,7 @@ def test_run_pipeline_emits_manifest_raw_decisions_and_lightweight_bundle(tmp_pa identity_trace = json.loads(identity_trace_path.read_text(encoding="utf-8")) source_manifest = json.loads(source_manifest_path.read_text(encoding="utf-8")) evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + figures = json.loads(figures_path.read_text(encoding="utf-8")) decisions = json.loads(decisions_path.read_text(encoding="utf-8")) bundle = json.loads(bundle_path.read_text(encoding="utf-8")) @@ -87,9 +95,14 @@ def test_run_pipeline_emits_manifest_raw_decisions_and_lightweight_bundle(tmp_pa assert identity_trace["repair_attempts"] == [] assert source_manifest["coverage"]["text_pages_extracted"] == 4 assert source_manifest["coverage"]["text_truncated"] is False + assert source_manifest["source_sha256"] == hashlib.sha256(pdf_path.read_bytes()).hexdigest() assert source_manifest["identity_contract"]["identity_verdict"] == "accepted" assert any(section["section_id"] == "sec:method" for section in source_manifest["sections"]) assert evidence["summary"]["source_corpus_used"] is True + assert figures["output_language"] == "zh-CN" + assert decisions["output_language"] == "zh-CN" + assert bundle["output_language"] == "zh-CN" + assert bundle["writing_contract"]["language"] == "zh-CN" assert {item["source_id"] for item in decisions["decisions"]} == {"Figure 1", "Table 1"} assert bundle["source_manifest"]["raw_sections_path"] == str(raw_sections_path.resolve()) assert bundle["identity_contract"]["identity_verdict"] == "accepted" @@ -98,6 +111,129 @@ def test_run_pipeline_emits_manifest_raw_decisions_and_lightweight_bundle(tmp_pa removed_bundle_keys = ("evidence", "candidate_chunks", "section_texts", "summary") assert not any(key in bundle for key in removed_bundle_keys) + note_text = "# 本地 PDF 语言完整性\n\n本笔记验证工件链可安全保存。\n" + lint_path = workdir / "paper_lint.json" + lint_path.write_text( + json.dumps( + { + "output_language": "zh-CN", + "note_sha256": hashlib.sha256(note_text.encode("utf-8")).hexdigest(), + "passes_basic_structure": True, + "passes_style_gate": True, + "passes_math_gate": True, + } + ), + encoding="utf-8", + ) + save_cwd = tmp_path / "save-valid" + save_cwd.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_DISABLE_SHELL_CONFIG"] = "1" + env["DEEPPAPERNOTE_WORKSPACE_OUTPUT_DIR"] = "saved" + saved = subprocess.run( + [ + sys.executable, + str(WRITE_NOTE_SCRIPT), + "--title", + "本地 PDF 语言完整性", + "--content", + note_text, + "--lint-json", + str(lint_path), + "--figure-decisions", + str(decisions_path), + ], + cwd=save_cwd, + env=env, + check=True, + capture_output=True, + text=True, + ) + save_payload = json.loads(saved.stdout) + assert save_payload["output_language"] == "zh-CN" + assert Path(save_payload["note_path"]).read_text(encoding="utf-8") == note_text + assert Path(save_payload["images_dir"]).is_dir() + + for artifact_language, expected_error in ( + (None, "requires output_language"), + ("en", "does not match resolved output_language zh-CN"), + ): + invalid_decisions = dict(decisions) + if artifact_language is None: + invalid_decisions.pop("output_language") + else: + invalid_decisions["output_language"] = artifact_language + invalid_path = workdir / f"invalid-{artifact_language or 'missing'}.json" + invalid_path.write_text(json.dumps(invalid_decisions), encoding="utf-8") + failure_cwd = tmp_path / f"save-{artifact_language or 'missing'}" + failure_cwd.mkdir() + result = subprocess.run( + [ + sys.executable, + str(WRITE_NOTE_SCRIPT), + "--title", + "本地 PDF 语言失败", + "--content", + note_text, + "--lint-json", + str(lint_path), + "--figure-decisions", + str(invalid_path), + ], + cwd=failure_cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert expected_error in result.stderr + assert not (failure_cwd / "saved").exists() + + +def test_extract_source_text_rejects_a_mismatched_acquired_pdf_hash(tmp_path: Path) -> None: + pdf_path = tmp_path / "paper.pdf" + fetch_path = tmp_path / "fetch.json" + manifest_path = tmp_path / "paper_source_manifest.json" + write_test_pdf(pdf_path) + fetch_path.write_text( + json.dumps( + { + "status": "ok", + "script": "fetch_pdf.py", + "paper_id": "paper:hash-mismatch", + "title": "Hash Mismatch", + "pdf_path": str(pdf_path), + "source_sha256": "0" * 64, + "identity_contract": { + "artifact_type": "canonical_identity", + "schema_version": 2, + "paper_id": "paper:hash-mismatch", + "identity_verdict": "accepted", + "work_level_identity": {"title": "Hash Mismatch"}, + }, + } + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(EXTRACT_SOURCE_SCRIPT), + "--input", + str(fetch_path), + "--output", + str(manifest_path), + ], + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "source_sha256 does not match acquired PDF" in result.stderr + assert not manifest_path.exists() + def test_run_pipeline_does_not_materialize_before_final_save( tmp_path: Path, @@ -165,12 +301,81 @@ def fake_run(cmd: list[str], check: bool = True, **kwargs) -> object: assert fetch_call[fetch_call.index("--identity") + 1] == str( (workdir / "paper_identity.json").resolve() ) + assert fetch_call[fetch_call.index("--dest-dir") + 1] == str( + (workdir / "paper_pdfs").resolve() + ) evidence_call = calls[5] assert "--source-manifest" in evidence_call assert evidence_call[evidence_call.index("--source-manifest") + 1] == str( (workdir / "paper_source_manifest.json").resolve() ) + assets_call = calls[6] + assert assets_call[assets_call.index("--assets-dir") + 1] == str( + (workdir / "paper_assets").resolve() + ) + + +def test_run_pipeline_stops_at_configuration_before_identity( + tmp_path: Path, + monkeypatch, + configured_user_home: Path, +) -> None: + configured_user_home.unlink() + workdir = tmp_path / "must-not-exist" + calls: list[list[str]] = [] + monkeypatch.setattr(run_pipeline.subprocess, "run", lambda cmd, **kwargs: calls.append(cmd)) + monkeypatch.setattr( + sys, + "argv", + [ + "run_pipeline.py", + "--input", + "paper.pdf", + "--workdir", + str(workdir), + ], + ) + + with pytest.raises(SystemExit, match="needs_input"): + run_pipeline.main() + + assert calls == [] + assert not workdir.exists() + + +def test_run_pipeline_propagates_run_override_without_persisting_it( + tmp_path: Path, + monkeypatch, + configured_user_home: Path, +) -> None: + original = configured_user_home.read_bytes() + calls: list[tuple[list[str], dict[str, str]]] = [] + + def fake_run(cmd: list[str], check: bool = True, **kwargs) -> object: + calls.append((cmd, kwargs["env"])) + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr(run_pipeline.subprocess, "run", fake_run) + monkeypatch.setattr( + sys, + "argv", + [ + "run_pipeline.py", + "--input", + "paper.pdf", + "--workdir", + str(tmp_path / "run"), + "--language", + "en", + ], + ) + + run_pipeline.main() + + assert calls + assert all(env["DEEPPAPERNOTE_OUTPUT_LANGUAGE"] == "en" for _, env in calls) + assert configured_user_home.read_bytes() == original def test_run_pipeline_stops_before_fetch_when_identity_repair_is_exhausted( diff --git a/tests/test_user_configuration.py b/tests/test_user_configuration.py new file mode 100644 index 0000000..c7d9835 --- /dev/null +++ b/tests/test_user_configuration.py @@ -0,0 +1,476 @@ +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest +import user_configuration +from localization import SUPPORTED_OUTPUT_LANGUAGES, note_schema +from user_configuration import ( + ConfigurationWriteError, + inspect_configuration, + persist_preferences, + resolve_preferences, + user_config_path, +) + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +CONFIG_SCRIPT = PROJECT_ROOT / "skills/deeppapernote/scripts/user_configuration.py" +ENVIRONMENT_SCRIPT = PROJECT_ROOT / "skills/deeppapernote/scripts/check_environment.py" + + +def test_user_config_path_honors_process_isolation_override( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + isolated_path = tmp_path / "isolated" / "config.json" + monkeypatch.setenv("DEEPPAPERNOTE_CONFIG_PATH", str(isolated_path)) + + assert user_config_path() == isolated_path + + +def test_first_use_requests_one_workspace_prompt_batch(tmp_path: Path) -> None: + result = inspect_configuration(config_path=tmp_path / "config.json", environ={}) + + assert result == { + "state": "needs_input", + "config_path": str(tmp_path / "config.json"), + "affected_fields": ["output_language", "save_mode"], + "prompt_fields": ["output_language", "save_mode"], + "migration_candidates": {}, + "warnings": [], + } + + +def test_workspace_configuration_ignores_inactive_obsidian_fields(tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps( + { + "output_language": "zh-CN", + "save_mode": "workspace", + "obsidian_vault": "relative/missing", + "papers_dir": "../unsafe", + } + ), + encoding="utf-8", + ) + + result = inspect_configuration(config_path=path, environ={}) + + assert result["state"] == "ready" + assert result["affected_fields"] == [] + assert result["configuration"]["save_mode"] == "workspace" + + +def test_obsidian_configuration_requires_safe_existing_paths(tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps( + { + "output_language": "de", + "save_mode": "obsidian", + "obsidian_vault": "relative/missing", + "papers_dir": "../unsafe", + } + ), + encoding="utf-8", + ) + + result = inspect_configuration(config_path=path, environ={}) + + assert result["state"] == "invalid" + assert result["affected_fields"] == [ + "output_language", + "obsidian_vault", + "papers_dir", + ] + assert [issue["code"] for issue in result["issues"]] == [ + "invalid_enum", + "missing_vault", + "unsafe_path", + ] + + +def test_inspection_does_not_probe_or_create_the_obsidian_destination(tmp_path: Path) -> None: + vault = tmp_path / "vault" + vault.mkdir() + papers_dir = vault / "Research/Papers" + path = tmp_path / "config.json" + path.write_text( + json.dumps( + { + "output_language": "zh-CN", + "save_mode": "obsidian", + "obsidian_vault": str(vault), + "papers_dir": "Research/Papers", + } + ), + encoding="utf-8", + ) + + result = inspect_configuration(config_path=path, environ={}) + + assert result["state"] == "ready" + assert not papers_dir.exists() + + +def test_precedence_is_explicit_cli_process_then_user_without_persistence(tmp_path: Path) -> None: + path = tmp_path / "config.json" + original = '{"output_language":"zh-CN","save_mode":"workspace"}\n' + path.write_text(original, encoding="utf-8") + + resolved = resolve_preferences( + config_path=path, + explicit_overrides={"output_language": "en"}, + cli_overrides={"output_language": "zh-CN", "save_mode": "obsidian"}, + environ={ + "DEEPPAPERNOTE_OUTPUT_LANGUAGE": "zh-CN", + "DEEPPAPERNOTE_SAVE_MODE": "workspace", + }, + ) + + assert resolved["values"]["output_language"] == "en" + assert resolved["sources"]["output_language"] == "explicit_request" + assert resolved["values"]["save_mode"] == "obsidian" + assert resolved["sources"]["save_mode"] == "cli" + assert path.read_text(encoding="utf-8") == original + + +def test_partial_repair_requests_only_affected_fields(tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "obsidian"}), + encoding="utf-8", + ) + + result = inspect_configuration(config_path=path, environ={}) + + assert result["state"] == "needs_input" + assert result["prompt_fields"] == ["obsidian_vault", "papers_dir"] + + +def test_preference_change_preserves_unknown_fields_and_is_read_back(tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps( + {"output_language": "zh-CN", "save_mode": "workspace", "future_field": 7} + ), + encoding="utf-8", + ) + + result = persist_preferences( + {"output_language": "en"}, config_path=path, environ={} + ) + + assert result["state"] == "ready" + assert result["configuration"]["future_field"] == 7 + assert result["configuration"]["output_language"] == "en" + assert result["warnings"] == ["Preserved unknown configuration fields: future_field"] + + +@pytest.mark.parametrize("payload", ["{broken", "[]"]) +def test_invalid_configuration_requires_confirmed_replacement_and_keeps_backup( + tmp_path: Path, payload: str +) -> None: + path = tmp_path / "config.json" + path.write_text(payload, encoding="utf-8") + + with pytest.raises(ConfigurationWriteError, match="confirmed replacement"): + persist_preferences( + {"output_language": "zh-CN", "save_mode": "workspace"}, + config_path=path, + environ={}, + ) + + result = persist_preferences( + {"output_language": "zh-CN", "save_mode": "workspace"}, + config_path=path, + environ={}, + replace_invalid=True, + ) + + backups = list(tmp_path.glob("config.invalid-*.json")) + assert len(backups) == 1 + assert backups[0].read_text(encoding="utf-8") == payload + assert result["state"] == "ready" + + +def test_absent_configuration_reports_process_and_shell_migration_candidates( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + shell = tmp_path / ".zshrc" + shell.write_text( + 'export DEEPPAPERNOTE_SAVE_MODE="obsidian"\n' + f'export DEEPPAPERNOTE_OBSIDIAN_VAULT="{vault}"\n', + encoding="utf-8", + ) + + result = inspect_configuration( + config_path=tmp_path / "config.json", + environ={"DEEPPAPERNOTE_OUTPUT_LANGUAGE": "en"}, + shell_paths=[shell], + ) + + assert result["state"] == "needs_input" + assert result["migration_candidates"] == { + "output_language": {"value": "en", "source": "process_environment"}, + "save_mode": {"value": "obsidian", "source": str(shell)}, + "obsidian_vault": {"value": str(vault), "source": str(shell)}, + } + + +def test_invalid_legacy_values_are_not_migration_candidates(tmp_path: Path) -> None: + result = inspect_configuration( + config_path=tmp_path / "config.json", + environ={ + "DEEPPAPERNOTE_OUTPUT_LANGUAGE": "de", + "DEEPPAPERNOTE_SAVE_MODE": "banana", + "DEEPPAPERNOTE_OBSIDIAN_VAULT": "relative/missing", + "DEEPPAPERNOTE_PAPERS_DIR": "../escape", + }, + shell_paths=[], + ) + + assert result["migration_candidates"] == {} + + +def test_unknown_home_legacy_vault_is_not_a_migration_candidate(tmp_path: Path) -> None: + shell = tmp_path / ".zshrc" + shell.write_text( + "DEEPPAPERNOTE_OBSIDIAN_VAULT=~definitely_no_such_user_xyz/vault\n", + encoding="utf-8", + ) + + result = inspect_configuration( + config_path=tmp_path / "config.json", + environ={ + "DEEPPAPERNOTE_OBSIDIAN_VAULT": "~definitely_no_such_user_xyz/vault" + }, + shell_paths=[shell], + ) + + assert result["state"] == "needs_input" + assert result["migration_candidates"] == {} + + +def test_existing_configuration_never_reads_shell_candidates(tmp_path: Path) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + encoding="utf-8", + ) + missing_shell = tmp_path / "must-not-be-read" + + result = inspect_configuration( + config_path=path, + environ={"DEEPPAPERNOTE_OUTPUT_LANGUAGE": "en"}, + shell_paths=[missing_shell], + ) + + assert result["state"] == "ready" + assert result["migration_candidates"] == {} + + +@pytest.mark.parametrize( + ("payload", "code"), [("{broken", "malformed"), ("[]", "non_object")] +) +def test_inspector_distinguishes_malformed_and_non_object( + tmp_path: Path, payload: str, code: str +) -> None: + path = tmp_path / "config.json" + path.write_text(payload, encoding="utf-8") + + result = inspect_configuration(config_path=path, environ={}) + + assert result["state"] == "invalid" + assert result["issues"][0]["code"] == code + + +def test_inspector_reports_unreadable_and_unwritable_as_blocked( + tmp_path: Path, monkeypatch +) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + encoding="utf-8", + ) + original_read = user_configuration._read_configuration + monkeypatch.setattr( + user_configuration, + "_read_configuration", + lambda candidate: (None, "unreadable:permission denied"), + ) + unreadable = inspect_configuration(config_path=path, environ={}) + assert unreadable["state"] == "blocked" + assert unreadable["issues"][0]["code"] == "unreadable" + + monkeypatch.setattr(user_configuration, "_read_configuration", original_read) + monkeypatch.setattr(user_configuration, "_path_is_writable", lambda candidate: False) + unwritable = inspect_configuration(config_path=path, environ={}) + assert unwritable["state"] == "blocked" + assert unwritable["issues"][0]["code"] == "unwritable" + + +def test_writability_uses_atomic_replace_parent_directory(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + encoding="utf-8", + ) + checked: list[Path] = [] + + def fake_access(candidate: Path, mode: int) -> bool: + checked.append(Path(candidate)) + return True + + monkeypatch.setattr(user_configuration.os, "access", fake_access) + + assert inspect_configuration(config_path=path, environ={})["state"] == "ready" + assert checked == [tmp_path] + + +def test_readback_mismatch_is_never_reported_as_saved(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + encoding="utf-8", + ) + + def corrupt_write(candidate_path: Path, configuration: object) -> None: + candidate_path.write_text("{}", encoding="utf-8") + + monkeypatch.setattr(user_configuration, "_atomic_write", corrupt_write) + + with pytest.raises(ConfigurationWriteError, match="readback"): + persist_preferences({"output_language": "en"}, config_path=path, environ={}) + + +def test_confirmed_migration_becomes_the_only_durable_source(tmp_path: Path) -> None: + path = tmp_path / "config.json" + shell = tmp_path / ".zshrc" + shell.write_text( + "export DEEPPAPERNOTE_OUTPUT_LANGUAGE=en\n" + "export DEEPPAPERNOTE_SAVE_MODE=workspace\n", + encoding="utf-8", + ) + candidates = inspect_configuration( + config_path=path, environ={}, shell_paths=[shell] + )["migration_candidates"] + + persist_preferences( + {field: candidate["value"] for field, candidate in candidates.items()}, + config_path=path, + environ={}, + ) + shell.write_text( + "export DEEPPAPERNOTE_OUTPUT_LANGUAGE=zh-CN\n", + encoding="utf-8", + ) + result = inspect_configuration(config_path=path, environ={}, shell_paths=[shell]) + + assert result["state"] == "ready" + assert result["configuration"]["output_language"] == "en" + assert result["migration_candidates"] == {} + + +def test_canonical_configuration_reference_matches_machine_contract() -> None: + skill = (PROJECT_ROOT / "skills/deeppapernote/SKILL.md").read_text( + encoding="utf-8" + ) + contract = ( + PROJECT_ROOT / "skills/deeppapernote/references/user-configuration.md" + ).read_text(encoding="utf-8") + + for field in user_configuration.KNOWN_FIELDS: + assert f"`{field}`" in contract + values = ( + *sorted(user_configuration.OUTPUT_LANGUAGES), + *sorted(user_configuration.SAVE_MODES), + ) + for value in values: + assert f"`{value}`" in contract + for name in user_configuration.ENV_FIELDS.values(): + assert f"`{name}`" in contract + for option in ("--language", "--save-mode", "--vault", "--papers-dir"): + assert f"`{option}`" in contract + for state in ("ready", "needs_input", "invalid", "blocked"): + assert f"`{state}`" in contract + assert ( + "explicit request > CLI > current process environment > User Configuration" + in contract + ) + assert "preserve" in contract.lower() and "unknown" in contract.lower() + assert "Workspace mode" in contract and "preserve" in contract + assert "before paper identity resolution" in contract + assert "one canonical Skill" in skill + assert "one pipeline" in skill + assert "paper-local `images/`" in skill + assert "references/user-configuration.md" in skill + assert skill.index("Resolve Run Overrides") < skill.index("inspect User Configuration") + assert skill.index("inspect User Configuration") < skill.index("resolve the paper identity") + assert "without reading User Configuration" in contract + assert "scripts/user_configuration.py" in contract + + +def test_output_language_reference_matches_both_machine_schemas() -> None: + contract = ( + PROJECT_ROOT / "skills/deeppapernote/references/output-language.md" + ).read_text(encoding="utf-8") + + for language in SUPPORTED_OUTPUT_LANGUAGES: + schema = note_schema(language) + section_positions = [] + for section in schema["sections"].values(): + assert f"`{section}`" in contract + section_positions.append(contract.index(f"`{section}`")) + assert section_positions == sorted(section_positions) + for label in schema["figure_labels"].values(): + assert label in contract + assert schema["mechanism_flow"] in contract + + +def test_configuration_cli_keeps_semantic_failures_repairable( + configured_user_home: Path, +) -> None: + configured_user_home.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + encoding="utf-8", + ) + result = subprocess.run( + [sys.executable, str(CONFIG_SCRIPT), "--set-save-mode", "obsidian"], + env=os.environ.copy(), + capture_output=True, + text=True, + check=False, + ) + + payload = json.loads(result.stdout) + assert result.returncode == 2 + assert payload["state"] == "needs_input" + assert payload["affected_fields"] == ["obsidian_vault", "papers_dir"] + + +def test_environment_report_survives_missing_user_configuration( + tmp_path: Path, +) -> None: + home = tmp_path / "empty-home" + home.mkdir() + env = os.environ.copy() + env["DEEPPAPERNOTE_CONFIG_PATH"] = str(home / ".deeppapernote" / "config.json") + result = subprocess.run( + [sys.executable, str(ENVIRONMENT_SCRIPT)], + env=env, + capture_output=True, + text=True, + check=False, + ) + + payload = json.loads(result.stdout) + assert result.returncode == 0 + assert payload["user_configuration"]["state"] == "needs_input" diff --git a/tests/test_windows_path_encoding.py b/tests/test_windows_path_encoding.py index 2684f66..cecc342 100644 --- a/tests/test_windows_path_encoding.py +++ b/tests/test_windows_path_encoding.py @@ -9,8 +9,10 @@ from __future__ import annotations +import hashlib import json import os +import stat import subprocess import sys from pathlib import Path @@ -20,14 +22,47 @@ import lint_grounding import lint_note import plan_figure_table_decisions +import write_obsidian_note from common import load_json_file, resolve_obsidian_note_path - PROJECT_ROOT = Path(__file__).resolve().parents[1] SCRIPTS = PROJECT_ROOT / "skills" / "deeppapernote" / "scripts" BOM = "" +def _formal_save_artifacts(tmp_path: Path, canonical_note_text: str) -> tuple[Path, Path]: + lint_path = tmp_path / "lint.json" + lint_path.write_text( + json.dumps( + { + "output_language": "zh-CN", + "note_sha256": hashlib.sha256( + canonical_note_text.encode("utf-8") + ).hexdigest(), + "passes_basic_structure": True, + "passes_style_gate": True, + "passes_math_gate": True, + } + ), + encoding="utf-8", + ) + decisions_path = tmp_path / "decisions.json" + decisions_path.write_text( + json.dumps({"output_language": "zh-CN", "decisions": []}), + encoding="utf-8", + ) + return lint_path, decisions_path + + +def _source_manifest(tmp_path: Path) -> Path: + path = tmp_path / "source-manifest.json" + path.write_text( + json.dumps({"status": "ok", "source_sha256": "a" * 64}), + encoding="utf-8", + ) + return path + + # --------------------------------------------------------------------------- # # BOM handling (utf-8-sig) # # --------------------------------------------------------------------------- # @@ -139,6 +174,30 @@ def test_source_image_filename_handles_backslash_path() -> None: assert plan_figure_table_decisions.source_image_filename(plan_item) == "page_004_fig.png" +def test_windows_sidecar_hidden_attribute_preserves_existing_attributes( + tmp_path: Path, +) -> None: + sidecar = tmp_path / ".deeppapernote.json" + sidecar.write_text("{}", encoding="utf-8") + attributes = {"value": 0x20} + + def get_attributes(_path: str) -> int: + return attributes["value"] + + def set_attributes(_path: str, value: int) -> bool: + attributes["value"] = value + return True + + write_obsidian_note.ensure_sidecar_hidden( + sidecar, + platform="nt", + get_attributes=get_attributes, + set_attributes=set_attributes, + ) + + assert attributes["value"] == 0x20 | 0x2 + + # --------------------------------------------------------------------------- # # End-to-end (subprocess) — the script main() read paths # # --------------------------------------------------------------------------- # @@ -173,6 +232,8 @@ def test_write_obsidian_note_strips_bom_from_saved_note(tmp_path: Path) -> None: content_file.write_bytes( BOM.encode("utf-8") + "# 标题\n\n正文内容。\n".encode("utf-8") ) + canonical_note_text = "# 标题\n\n正文内容。\n" + lint_path, decisions_path = _formal_save_artifacts(tmp_path, canonical_note_text) result = subprocess.run( [ sys.executable, @@ -180,6 +241,9 @@ def test_write_obsidian_note_strips_bom_from_saved_note(tmp_path: Path) -> None: "--title", "BOM Content Test", "--vault", str(vault), "--content-file", str(content_file), + "--lint-json", str(lint_path), + "--figure-decisions", str(decisions_path), + "--source-manifest", str(_source_manifest(tmp_path)), ], cwd=tmp_path, env=_clean_env(), @@ -193,6 +257,61 @@ def test_write_obsidian_note_strips_bom_from_saved_note(tmp_path: Path) -> None: # frontmatter / the H1 title). assert not saved.startswith(BOM) assert saved.lstrip().startswith("# 标题") + if os.name == "nt": + sidecar = Path(payload["sidecar_path"]) + assert sidecar.stat().st_file_attributes & stat.FILE_ATTRIBUTE_HIDDEN + + +def test_write_obsidian_note_accepts_unchanged_crlf_after_final_lint( + tmp_path: Path, +) -> None: + vault = tmp_path / "vault" + vault.mkdir() + content_file = tmp_path / "note.md" + crlf_text = "# CRLF Note\r\n\r\n正文内容。\r\n" + content_file.write_bytes(crlf_text.encode("utf-8")) + lint_output = tmp_path / "lint-from-script.json" + + subprocess.run( + [ + sys.executable, + str(SCRIPTS / "lint_note.py"), + "--input", str(content_file), + "--output", str(lint_output), + ], + cwd=tmp_path, + env=_clean_env(), + check=True, + ) + lint = json.loads(lint_output.read_text(encoding="utf-8")) + lint.update({key: True for key in lint if key.startswith("passes_")}) + lint_output.write_text(json.dumps(lint), encoding="utf-8") + decisions_path = tmp_path / "decisions.json" + decisions_path.write_text( + json.dumps({"output_language": "zh-CN", "decisions": []}), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(SCRIPTS / "write_obsidian_note.py"), + "--title", "CRLF Note", + "--vault", str(vault), + "--content-file", str(content_file), + "--lint-json", str(lint_output), + "--figure-decisions", str(decisions_path), + "--source-manifest", str(_source_manifest(tmp_path)), + ], + cwd=tmp_path, + env=_clean_env(), + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + saved = Path(json.loads(result.stdout)["note_path"]).read_bytes() + assert b"\r\n" not in saved def test_materialize_figure_asset_embed_uses_forward_slashes(tmp_path: Path) -> None: