From b29b6d970f62df0e7a29911ffcf99addb5f19170 Mon Sep 17 00:00:00 2001 From: mountain Date: Wed, 8 Jul 2026 10:51:40 +0800 Subject: [PATCH 1/8] feat(indexer): expose pageindex_max_concurrency to cap indexing concurrency PageIndex fans out one indexing LLM call per structure node; on a large document that can open a socket per node and exhaust the process file-descriptor limit ("[Errno 24] Too many open files"). This wires an OpenKB config knob through to PageIndex's IndexConfig.max_concurrency cap. - New `pageindex_max_concurrency` KB config key (default None = let PageIndex apply its own default). - indexer forwards it via `_build_index_config` only when set AND the installed PageIndex's IndexConfig declares the field, so OpenKB keeps working against a pinned PageIndex that predates it (IndexConfig forbids unknown kwargs). --- openkb/config.py | 4 ++++ openkb/indexer.py | 27 ++++++++++++++++++++++----- tests/test_config.py | 10 ++++++++++ tests/test_indexer.py | 28 +++++++++++++++++++++++++++- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/openkb/config.py b/openkb/config.py index a5ee010b..bd924b6c 100644 --- a/openkb/config.py +++ b/openkb/config.py @@ -17,6 +17,10 @@ "model": "gpt-5.4", "language": "en", "pageindex_threshold": 20, + # Cap on concurrent indexing LLM calls PageIndex makes for a single long + # document. None = let PageIndex apply its own default. Raise it to index + # faster, lower it if you hit provider rate limits or "too many open files". + "pageindex_max_concurrency": None, } # Default entity-type vocabulary. Overridable per-KB via the optional diff --git a/openkb/indexer.py b/openkb/indexer.py index 0f58bef3..415d61d7 100644 --- a/openkb/indexer.py +++ b/openkb/indexer.py @@ -153,6 +153,27 @@ def _write_long_doc_artifacts( return summary_path +def _build_index_config(config: dict[str, Any]) -> IndexConfig: + """Build the PageIndex ``IndexConfig`` for local indexing. + + Forwards the optional ``pageindex_max_concurrency`` KB setting to PageIndex, + which caps how many indexing LLM calls run at once (guarding against the + "too many open files" fd exhaustion on large documents). The value is only + passed when set *and* the installed PageIndex's ``IndexConfig`` declares the + field, so OpenKB keeps working against a pinned PageIndex that predates it + (``IndexConfig`` forbids unknown kwargs). + """ + kwargs: dict[str, Any] = { + "if_add_node_text": True, + "if_add_node_summary": True, + "if_add_doc_description": True, + } + max_concurrency = config.get("pageindex_max_concurrency") + if max_concurrency is not None and "max_concurrency" in IndexConfig.model_fields: + kwargs["max_concurrency"] = max_concurrency + return IndexConfig(**kwargs) + + def index_long_document(pdf_path: Path, kb_dir: Path, doc_name: str | None = None) -> IndexResult: """Index a long PDF document using PageIndex and write wiki pages. @@ -166,11 +187,7 @@ def index_long_document(pdf_path: Path, kb_dir: Path, doc_name: str | None = Non model: str = config.get("model", "gpt-5.4") pageindex_api_key = os.environ.get("PAGEINDEX_API_KEY", "") - index_config = IndexConfig( - if_add_node_text=True, - if_add_node_summary=True, - if_add_doc_description=True, - ) + index_config = _build_index_config(config) client = PageIndexClient( api_key=pageindex_api_key or None, diff --git a/tests/test_config.py b/tests/test_config.py index be473fcd..dc85c285 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -26,6 +26,16 @@ def test_default_config_values(): assert DEFAULT_CONFIG["pageindex_threshold"] == 20 +def test_pageindex_max_concurrency_defaults_to_none(): + assert DEFAULT_CONFIG["pageindex_max_concurrency"] is None + + +def test_load_pageindex_max_concurrency_override(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text("pageindex_max_concurrency: 12\n", encoding="utf-8") + assert load_config(config_path)["pageindex_max_concurrency"] == 12 + + def test_load_missing_file_returns_defaults(tmp_path): missing = tmp_path / "nonexistent" / "config.yaml" config = load_config(missing) diff --git a/tests/test_indexer.py b/tests/test_indexer.py index 3af85b04..c12b3ccc 100644 --- a/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -5,8 +5,34 @@ from unittest.mock import MagicMock, patch import pytest +from pageindex import IndexConfig + +from openkb.indexer import ( + IndexResult, + _build_index_config, + _normalize_page_content, + index_long_document, +) + + +class TestBuildIndexConfig: + def test_sets_base_flags(self): + cfg = _build_index_config({}) + assert cfg.if_add_node_text is True + assert cfg.if_add_node_summary is True + assert cfg.if_add_doc_description is True + + def test_forwards_max_concurrency_when_supported(self): + cfg = _build_index_config({"pageindex_max_concurrency": 8}) + if "max_concurrency" in IndexConfig.model_fields: + assert cfg.max_concurrency == 8 + else: + # A PageIndex predating the field: forwarded as a no-op, never raised. + assert not hasattr(cfg, "max_concurrency") -from openkb.indexer import IndexResult, _normalize_page_content, index_long_document + def test_none_value_is_left_to_pageindex_default(self): + cfg = _build_index_config({"pageindex_max_concurrency": None}) + assert getattr(cfg, "max_concurrency", None) is None class TestNormalizePageContent: From e12e74ec28b8299b49d2a40010abb122b0ec6320 Mon Sep 17 00:00:00 2001 From: mountain Date: Wed, 8 Jul 2026 11:15:04 +0800 Subject: [PATCH 2/8] feat(config): make compile concurrency configurable (closes #173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Concept/entity generation ran at a hardcoded concurrency of 5 (DEFAULT_COMPILE_CONCURRENCY) with no way to lower it when the LLM provider rate-limits. Add a `compile_concurrency` config key (default 5) and thread it through every add / recompile / cloud-import compile call via a shared `_compile_concurrency` resolver (null / non-positive → the compiler default). Pairs with `pageindex_max_concurrency` (indexing side) from this PR — both concurrency knobs are now KB-configurable. --- openkb/cli.py | 46 +++++++++++++++++++++++++++++++++++---- openkb/config.py | 3 +++ tests/test_add_command.py | 33 ++++++++++++++++++++++++++++ tests/test_config.py | 10 +++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/openkb/cli.py b/openkb/cli.py index 2c27eab2..6c3d4bb0 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -47,7 +47,7 @@ def filter(self, record: logging.LogRecord) -> bool: litellm.suppress_debug_info = True from dotenv import load_dotenv -from openkb.agent.compiler import compile_long_doc +from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY, compile_long_doc from openkb.config import ( DEFAULT_CONFIG, load_config, @@ -439,6 +439,19 @@ def _run_compile_with_retry(coro_factory, label: str) -> None: raise +def _compile_concurrency(config: dict) -> int: + """Concurrency cap for the compile step (concept/entity generation). + + Configurable per KB via ``compile_concurrency`` in config.yaml; a missing, + null, or non-positive value falls back to the compiler's built-in default. + Lower it when the LLM provider rate-limits. + """ + value = config.get("compile_concurrency") + if isinstance(value, int) and value > 0: + return value + return DEFAULT_COMPILE_CONCURRENCY + + def add_single_file( file_path: Path, kb_dir: Path, *, stage: bool = True ) -> Literal["added", "skipped", "failed"]: @@ -564,6 +577,7 @@ def _add_single_file_locked( kb_dir, model, doc_description=index_result.description, + max_concurrency=_compile_concurrency(config), ), label=f"Compiling long doc (doc_id={index_result.doc_id})", ) @@ -572,7 +586,13 @@ def _add_single_file_locked( raise RuntimeError(f"Converted document has no source artifact: {file_path.name}") source_path = result.source_path _run_compile_with_retry( - lambda: compile_short_doc(doc_name, source_path, kb_dir, model), + lambda: compile_short_doc( + doc_name, + source_path, + kb_dir, + model, + max_concurrency=_compile_concurrency(config), + ), label="Compiling short doc", ) @@ -699,6 +719,7 @@ def import_from_pageindex_cloud(doc_id: str, kb_dir: Path) -> Literal["added", " kb_dir, model, doc_description=cloud.description, + max_concurrency=_compile_concurrency(config), ), label=f"Compiling imported doc (doc_id={doc_id})", ) @@ -1679,7 +1700,16 @@ def _classify(meta: dict) -> str: click.echo(f"[{i}/{total}] Recompiling long doc {name}...") start = time.time() try: - asyncio.run(compiler.compile_long_doc(name, summary_path, doc_id, kb_dir, model)) + asyncio.run( + compiler.compile_long_doc( + name, + summary_path, + doc_id, + kb_dir, + model, + max_concurrency=_compile_concurrency(config), + ) + ) except Exception as exc: click.echo(f" [ERROR] Compilation failed: {exc}") logging.getLogger(__name__).debug("Recompile traceback:", exc_info=True) @@ -1699,7 +1729,15 @@ def _classify(meta: dict) -> str: click.echo(f"[{i}/{total}] Recompiling short doc {name}...") start = time.time() try: - asyncio.run(compiler.compile_short_doc(name, source_path, kb_dir, model)) + asyncio.run( + compiler.compile_short_doc( + name, + source_path, + kb_dir, + model, + max_concurrency=_compile_concurrency(config), + ) + ) except Exception as exc: click.echo(f" [ERROR] Compilation failed: {exc}") logging.getLogger(__name__).debug("Recompile traceback:", exc_info=True) diff --git a/openkb/config.py b/openkb/config.py index bd924b6c..f18029df 100644 --- a/openkb/config.py +++ b/openkb/config.py @@ -21,6 +21,9 @@ # document. None = let PageIndex apply its own default. Raise it to index # faster, lower it if you hit provider rate limits or "too many open files". "pageindex_max_concurrency": None, + # Cap on concurrent compile LLM calls OpenKB makes when generating concept + # and entity pages. Lower it if your LLM provider rate-limits. + "compile_concurrency": 5, } # Default entity-type vocabulary. Overridable per-KB via the optional diff --git a/tests/test_add_command.py b/tests/test_add_command.py index b8cdee84..fc2caa4a 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -97,6 +97,39 @@ def test_add_single_file_compile_failure_rolls_back_converted_artifacts(self, tm assert not (kb_dir / "wiki" / "sources" / "notes.md").exists() assert HashRegistry(kb_dir / ".openkb" / "hashes.json").all_entries() == {} + def test_compile_concurrency_resolution(self): + from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY + from openkb.cli import _compile_concurrency + + assert _compile_concurrency({}) == DEFAULT_COMPILE_CONCURRENCY + assert _compile_concurrency({"compile_concurrency": 3}) == 3 + # None / non-positive / non-int fall back to the compiler default. + assert _compile_concurrency({"compile_concurrency": None}) == DEFAULT_COMPILE_CONCURRENCY + assert _compile_concurrency({"compile_concurrency": 0}) == DEFAULT_COMPILE_CONCURRENCY + + def test_add_forwards_compile_concurrency_from_config(self, tmp_path): + from unittest.mock import AsyncMock + + from openkb.cli import add_single_file + + kb_dir = self._setup_kb(tmp_path) + (kb_dir / ".openkb" / "config.yaml").write_text( + "model: gpt-4o-mini\ncompile_concurrency: 3\n", encoding="utf-8" + ) + doc = tmp_path / "notes.md" + doc.write_text("# Notes\n\nBody", encoding="utf-8") + + with ( + patch( + "openkb.agent.compiler.compile_short_doc", new_callable=AsyncMock + ) as mock_compile, + patch("openkb.cli._setup_llm_key"), + ): + outcome = add_single_file(doc, kb_dir) + + assert outcome == "added" + assert mock_compile.call_args.kwargs["max_concurrency"] == 3 + def _long_doc_conv(self, kb_dir, name, file_hash): from openkb.converter import ConvertResult diff --git a/tests/test_config.py b/tests/test_config.py index dc85c285..42e03ef5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -36,6 +36,16 @@ def test_load_pageindex_max_concurrency_override(tmp_path): assert load_config(config_path)["pageindex_max_concurrency"] == 12 +def test_compile_concurrency_defaults_to_5(): + assert DEFAULT_CONFIG["compile_concurrency"] == 5 + + +def test_load_compile_concurrency_override(tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text("compile_concurrency: 3\n", encoding="utf-8") + assert load_config(config_path)["compile_concurrency"] == 3 + + def test_load_missing_file_returns_defaults(tmp_path): missing = tmp_path / "nonexistent" / "config.yaml" config = load_config(missing) From 8a8e65c18e0dea1da407368680a7e174002d00ed Mon Sep 17 00:00:00 2001 From: mountain Date: Wed, 8 Jul 2026 13:12:35 +0800 Subject: [PATCH 3/8] fix(config): address xhigh code-review findings on the concurrency knobs - resolve_compile_concurrency moves to config.py (matching resolve_timeout's convention) and now rejects bool (bool is an int subclass, so `compile_concurrency: true` previously silently became concurrency=1) and logs a warning on any malformed value, same as the other resolvers. - _build_index_config now warns when pageindex_max_concurrency is configured but the installed PageIndex doesn't support the field yet, instead of silently dropping it with no signal to the user. - Replaced the tautological test_forwards_max_concurrency_when_supported (which branched on the same runtime condition as the code under test, so it never exercised the forwarding assertion under CI's pinned PageIndex) with fake IndexConfig doubles that make both branches deterministic regardless of the installed pageindex version. - Added a cross-check test pinning DEFAULT_CONFIG["compile_concurrency"] against the compiler's own DEFAULT_COMPILE_CONCURRENCY so the two literals can't silently drift apart. - Added an end-to-end test proving index_long_document's own loaded config (not just a hand-built dict) reaches PageIndexClient's IndexConfig. - Hoisted the compile-concurrency resolution out of `recompile --all`'s per-document loop (was recomputed every iteration despite being loop-invariant). - Documented both new config.yaml keys in config.yaml.example and examples/configuration/README.md (kept in sync). --- config.yaml.example | 5 ++ examples/configuration/README.md | 7 +++ openkb/cli.py | 27 +++------ openkb/config.py | 21 +++++++ openkb/indexer.py | 10 +++- tests/test_add_command.py | 10 ---- tests/test_config.py | 54 ++++++++++++++++++ tests/test_indexer.py | 95 +++++++++++++++++++++++++++++--- 8 files changed, 190 insertions(+), 39 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index eebf3bf6..caa3c6b5 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -2,6 +2,11 @@ model: gpt-5.4 # LLM model (any LiteLLM-supported provider) language: en # Wiki output language pageindex_threshold: 20 # PDF pages threshold for PageIndex +# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many +# open files" on large PDFs. Omit either to use the default. +# pageindex_max_concurrency: 10 # PageIndex indexing concurrency (null = PageIndex's own default) +# compile_concurrency: 5 # concept/entity page generation concurrency + # Optional: override the entity-type vocabulary used for entity pages. # Omit this key to use the default 7 types # (person, organization, place, product, work, event, other). diff --git a/examples/configuration/README.md b/examples/configuration/README.md index 58cb7d4d..5530e553 100644 --- a/examples/configuration/README.md +++ b/examples/configuration/README.md @@ -70,6 +70,11 @@ model: gpt-5.4 # LLM model (any LiteLLM-supported provider) language: en # Wiki output language pageindex_threshold: 20 # PDF pages threshold for PageIndex +# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many +# open files" on large PDFs. Omit either to use the default. +# pageindex_max_concurrency: 10 # PageIndex indexing concurrency (null = PageIndex's own default) +# compile_concurrency: 5 # concept/entity page generation concurrency + # Optional: override the entity-type vocabulary used for entity pages. # Omit this key to use the default 7 types # (person, organization, place, product, work, event, other). @@ -95,6 +100,8 @@ pageindex_threshold: 20 # PDF pages threshold for PageIndex | `model` | `gpt-5.4` | LLM used for all compile/query/chat work. | | `language` | `en` | Language the wiki is written in. | | `pageindex_threshold` | `20` | PDFs with this many pages **or more** take the long-doc (PageIndex) path; shorter ones go through the short-doc path. See [`pageindex-cloud/`](../pageindex-cloud/). | +| `pageindex_max_concurrency` | `null` | Caps concurrent indexing LLM calls PageIndex makes for a single long document. Lower it if you hit provider rate limits or "too many open files" on large PDFs. `null` lets PageIndex apply its own default. | +| `compile_concurrency` | `5` | Caps concurrent LLM calls OpenKB makes while generating concept/entity pages. Lower it if your LLM provider rate-limits. | | `entity_types` | 7 defaults | Custom vocabulary for entity pages. `other` is always kept. | | `litellm:` | – | A pass-through block for LiteLLM. See below. | diff --git a/openkb/cli.py b/openkb/cli.py index 6c3d4bb0..7b7e1e64 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -47,13 +47,14 @@ def filter(self, record: logging.LogRecord) -> bool: litellm.suppress_debug_info = True from dotenv import load_dotenv -from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY, compile_long_doc +from openkb.agent.compiler import compile_long_doc from openkb.config import ( DEFAULT_CONFIG, load_config, save_config, load_global_config, register_kb, + resolve_compile_concurrency, resolve_extra_headers, set_extra_headers, resolve_timeout, @@ -439,19 +440,6 @@ def _run_compile_with_retry(coro_factory, label: str) -> None: raise -def _compile_concurrency(config: dict) -> int: - """Concurrency cap for the compile step (concept/entity generation). - - Configurable per KB via ``compile_concurrency`` in config.yaml; a missing, - null, or non-positive value falls back to the compiler's built-in default. - Lower it when the LLM provider rate-limits. - """ - value = config.get("compile_concurrency") - if isinstance(value, int) and value > 0: - return value - return DEFAULT_COMPILE_CONCURRENCY - - def add_single_file( file_path: Path, kb_dir: Path, *, stage: bool = True ) -> Literal["added", "skipped", "failed"]: @@ -577,7 +565,7 @@ def _add_single_file_locked( kb_dir, model, doc_description=index_result.description, - max_concurrency=_compile_concurrency(config), + max_concurrency=resolve_compile_concurrency(config), ), label=f"Compiling long doc (doc_id={index_result.doc_id})", ) @@ -591,7 +579,7 @@ def _add_single_file_locked( source_path, kb_dir, model, - max_concurrency=_compile_concurrency(config), + max_concurrency=resolve_compile_concurrency(config), ), label="Compiling short doc", ) @@ -719,7 +707,7 @@ def import_from_pageindex_cloud(doc_id: str, kb_dir: Path) -> Literal["added", " kb_dir, model, doc_description=cloud.description, - max_concurrency=_compile_concurrency(config), + max_concurrency=resolve_compile_concurrency(config), ), label=f"Compiling imported doc (doc_id={doc_id})", ) @@ -1665,6 +1653,7 @@ def _classify(meta: dict) -> str: _setup_llm_key(kb_dir) config = load_config(openkb_dir / "config.yaml") model: str = config.get("model", DEFAULT_CONFIG["model"]) + max_concurrency = resolve_compile_concurrency(config) # Import lazily and reference via the module so tests can patch # ``openkb.agent.compiler.compile_*`` and see the call. @@ -1707,7 +1696,7 @@ def _classify(meta: dict) -> str: doc_id, kb_dir, model, - max_concurrency=_compile_concurrency(config), + max_concurrency=max_concurrency, ) ) except Exception as exc: @@ -1735,7 +1724,7 @@ def _classify(meta: dict) -> str: source_path, kb_dir, model, - max_concurrency=_compile_concurrency(config), + max_concurrency=max_concurrency, ) ) except Exception as exc: diff --git a/openkb/config.py b/openkb/config.py index f18029df..45edea05 100644 --- a/openkb/config.py +++ b/openkb/config.py @@ -183,6 +183,27 @@ def resolve_timeout(config: dict) -> float | None: return value +def resolve_compile_concurrency(config: dict) -> int: + """Resolve the optional ``compile_concurrency:`` key for the compile step + (concept/entity generation). + + Returns ``DEFAULT_CONFIG["compile_concurrency"]`` when absent, ``None``, or + invalid; rejects bools and non-positive values, warning when present but + unusable (an explicit ``null`` is the normal "use the default" case and + warns silently, matching ``resolve_timeout``). + """ + value = config.get("compile_concurrency") + if value is None: + return DEFAULT_CONFIG["compile_concurrency"] + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + logger.warning( + "config: 'compile_concurrency' must be a positive integer, got %r — using default.", + value, + ) + return DEFAULT_CONFIG["compile_concurrency"] + return value + + def resolve_litellm_settings(config: dict) -> dict[str, Any]: """Resolve the optional ``litellm:`` mapping of LiteLLM module settings. diff --git a/openkb/indexer.py b/openkb/indexer.py index 415d61d7..25b528db 100644 --- a/openkb/indexer.py +++ b/openkb/indexer.py @@ -169,8 +169,14 @@ def _build_index_config(config: dict[str, Any]) -> IndexConfig: "if_add_doc_description": True, } max_concurrency = config.get("pageindex_max_concurrency") - if max_concurrency is not None and "max_concurrency" in IndexConfig.model_fields: - kwargs["max_concurrency"] = max_concurrency + if max_concurrency is not None: + if "max_concurrency" in IndexConfig.model_fields: + kwargs["max_concurrency"] = max_concurrency + else: + logger.warning( + "config: 'pageindex_max_concurrency' is set but the installed " + "PageIndex version does not support it yet — ignoring it." + ) return IndexConfig(**kwargs) diff --git a/tests/test_add_command.py b/tests/test_add_command.py index fc2caa4a..2a415a27 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -97,16 +97,6 @@ def test_add_single_file_compile_failure_rolls_back_converted_artifacts(self, tm assert not (kb_dir / "wiki" / "sources" / "notes.md").exists() assert HashRegistry(kb_dir / ".openkb" / "hashes.json").all_entries() == {} - def test_compile_concurrency_resolution(self): - from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY - from openkb.cli import _compile_concurrency - - assert _compile_concurrency({}) == DEFAULT_COMPILE_CONCURRENCY - assert _compile_concurrency({"compile_concurrency": 3}) == 3 - # None / non-positive / non-int fall back to the compiler default. - assert _compile_concurrency({"compile_concurrency": None}) == DEFAULT_COMPILE_CONCURRENCY - assert _compile_concurrency({"compile_concurrency": 0}) == DEFAULT_COMPILE_CONCURRENCY - def test_add_forwards_compile_concurrency_from_config(self, tmp_path): from unittest.mock import AsyncMock diff --git a/tests/test_config.py b/tests/test_config.py index 42e03ef5..71274faf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,6 +5,7 @@ get_extra_headers, get_timeout, load_config, + resolve_compile_concurrency, resolve_extra_headers, resolve_litellm_settings, resolve_timeout, @@ -46,6 +47,59 @@ def test_load_compile_concurrency_override(tmp_path): assert load_config(config_path)["compile_concurrency"] == 3 +def test_resolve_compile_concurrency_absent_uses_default(): + assert resolve_compile_concurrency({}) == DEFAULT_CONFIG["compile_concurrency"] + + +def test_resolve_compile_concurrency_valid_value(): + assert resolve_compile_concurrency({"compile_concurrency": 3}) == 3 + + +def test_resolve_compile_concurrency_rejects_bool(caplog): + with caplog.at_level(logging.WARNING, logger="openkb.config"): + result = resolve_compile_concurrency({"compile_concurrency": True}) + assert result == DEFAULT_CONFIG["compile_concurrency"] + assert "compile_concurrency" in caplog.text + + +def test_resolve_compile_concurrency_rejects_non_positive(caplog): + with caplog.at_level(logging.WARNING, logger="openkb.config"): + assert ( + resolve_compile_concurrency({"compile_concurrency": 0}) + == DEFAULT_CONFIG["compile_concurrency"] + ) + assert "compile_concurrency" in caplog.text + caplog.clear() + with caplog.at_level(logging.WARNING, logger="openkb.config"): + assert ( + resolve_compile_concurrency({"compile_concurrency": -1}) + == DEFAULT_CONFIG["compile_concurrency"] + ) + assert "compile_concurrency" in caplog.text + + +def test_resolve_compile_concurrency_rejects_non_int(): + assert ( + resolve_compile_concurrency({"compile_concurrency": "3"}) + == DEFAULT_CONFIG["compile_concurrency"] + ) + + +def test_resolve_compile_concurrency_none_is_silent(caplog): + # Explicit null / absent is the normal "use the default" case — no warning. + with caplog.at_level(logging.WARNING, logger="openkb.config"): + resolve_compile_concurrency({"compile_concurrency": None}) + assert caplog.text == "" + + +def test_compile_concurrency_defaults_match_compiler_default(): + """DEFAULT_CONFIG and the compiler's own DEFAULT_COMPILE_CONCURRENCY are two + independent literals; catch drift immediately if only one is ever updated.""" + from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY + + assert DEFAULT_CONFIG["compile_concurrency"] == DEFAULT_COMPILE_CONCURRENCY + + def test_load_missing_file_returns_defaults(tmp_path): missing = tmp_path / "nonexistent" / "config.yaml" config = load_config(missing) diff --git a/tests/test_indexer.py b/tests/test_indexer.py index c12b3ccc..857b1bf4 100644 --- a/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -2,10 +2,10 @@ from __future__ import annotations +import logging from unittest.mock import MagicMock, patch import pytest -from pageindex import IndexConfig from openkb.indexer import ( IndexResult, @@ -15,6 +15,38 @@ ) +class _FakeIndexConfigWithConcurrency: + """Stand-in for a PageIndex ``IndexConfig`` that declares ``max_concurrency``. + + Used instead of relying on whatever ``pageindex`` happens to be installed in + this environment, so the forwarding tests are deterministic regardless of + the currently-pinned PageIndex version (see ``test_forwards_...`` below). + """ + + model_fields = { + "if_add_node_text": None, + "if_add_node_summary": None, + "if_add_doc_description": None, + "max_concurrency": None, + } + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +class _FakeIndexConfigWithoutConcurrency: + """Stand-in for a PageIndex ``IndexConfig`` predating ``max_concurrency``.""" + + model_fields = { + "if_add_node_text": None, + "if_add_node_summary": None, + "if_add_doc_description": None, + } + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + class TestBuildIndexConfig: def test_sets_base_flags(self): cfg = _build_index_config({}) @@ -22,18 +54,39 @@ def test_sets_base_flags(self): assert cfg.if_add_node_summary is True assert cfg.if_add_doc_description is True - def test_forwards_max_concurrency_when_supported(self): + def test_forwards_max_concurrency_when_supported(self, monkeypatch): + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) cfg = _build_index_config({"pageindex_max_concurrency": 8}) - if "max_concurrency" in IndexConfig.model_fields: - assert cfg.max_concurrency == 8 - else: - # A PageIndex predating the field: forwarded as a no-op, never raised. - assert not hasattr(cfg, "max_concurrency") + assert cfg.max_concurrency == 8 - def test_none_value_is_left_to_pageindex_default(self): + def test_does_not_forward_when_unsupported(self, monkeypatch): + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency) + cfg = _build_index_config({"pageindex_max_concurrency": 8}) + assert not hasattr(cfg, "max_concurrency") + + def test_none_value_is_left_to_pageindex_default(self, monkeypatch): + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) cfg = _build_index_config({"pageindex_max_concurrency": None}) assert getattr(cfg, "max_concurrency", None) is None + def test_warns_when_configured_but_unsupported(self, monkeypatch, caplog): + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency) + with caplog.at_level(logging.WARNING, logger="openkb.indexer"): + _build_index_config({"pageindex_max_concurrency": 8}) + assert "pageindex_max_concurrency" in caplog.text + + def test_no_warning_when_unset(self, monkeypatch, caplog): + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency) + with caplog.at_level(logging.WARNING, logger="openkb.indexer"): + _build_index_config({}) + assert caplog.text == "" + + def test_no_warning_when_supported(self, monkeypatch, caplog): + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) + with caplog.at_level(logging.WARNING, logger="openkb.indexer"): + _build_index_config({"pageindex_max_concurrency": 8}) + assert caplog.text == "" + class TestNormalizePageContent: def test_normalizes_pageindex_dicts(self): @@ -204,6 +257,32 @@ def test_localclient_called_with_index_config(self, kb_dir, sample_tree, tmp_pat assert ic.if_add_node_summary is True assert ic.if_add_doc_description is True + def test_pageindex_max_concurrency_flows_from_kb_config(self, kb_dir, sample_tree, tmp_path): + """The KB's real config.yaml, loaded by index_long_document itself, must + reach the IndexConfig passed to PageIndexClient — not just the isolated + _build_index_config unit tested directly with a hand-built dict.""" + (kb_dir / ".openkb" / "config.yaml").write_text( + "model: gpt-4o-mini\npageindex_max_concurrency: 7\n", encoding="utf-8" + ) + + doc_id = "conc-789" + fake_col = self._make_fake_collection(doc_id, sample_tree) + fake_client = MagicMock() + fake_client.collection.return_value = fake_col + + pdf_path = tmp_path / "report.pdf" + pdf_path.write_bytes(b"%PDF-1.4 fake") + + with ( + patch("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency), + patch("openkb.indexer.PageIndexClient", return_value=fake_client) as mock_cls, + patch("openkb.images.convert_pdf_to_pages", return_value=self._fake_pages()), + ): + index_long_document(pdf_path, kb_dir) + + _, kwargs = mock_cls.call_args + assert kwargs["index_config"].max_concurrency == 7 + def test_cloud_page_content_is_normalized(self, kb_dir, sample_tree, tmp_path, monkeypatch): doc_id = "cloud-123" fake_col = self._make_fake_collection(doc_id, sample_tree) From 4719a85ccf8aca37dd34e80d1024d87e5ccae3fb Mon Sep 17 00:00:00 2001 From: mountain Date: Wed, 8 Jul 2026 15:18:25 +0800 Subject: [PATCH 4/8] refactor(config): unify pageindex_max_concurrency + compile_concurrency into `concurrency` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PageIndex indexing and OpenKB's own concept/entity compilation never run concurrently with each other for the same document (they're sequential phases of one add), and both knobs exist for the exact same reason — the user hit a provider rate limit or an fd ceiling. Splitting them by internal subsystem leaked OpenKB's architecture into the config surface for no practical benefit, since a user tuning one for a rate limit would set the other to the same value anyway. - Single `concurrency` config key (default null = each stage keeps its own built-in default). - config.resolve_concurrency(config) -> int | None: validates (rejects bool and non-positive values with a warning), returns None on unset/invalid — no default substitution inside; callers decide what None means for them. - cli.py's compile call sites: `resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY`. - indexer.py's _build_index_config now routes through resolve_concurrency (openkb validates before forwarding to PageIndex, rather than relying solely on PageIndex's own future validator) instead of raw config.get. - This also removes the prior dual-literal-default drift risk entirely: there's no second numeric default to keep in sync, since an unset `concurrency` simply forwards nothing to PageIndex and lets the compiler use its own DEFAULT_COMPILE_CONCURRENCY. - Updated config.yaml.example and examples/configuration/README.md (kept in sync) to the single key. --- config.yaml.example | 9 ++-- examples/configuration/README.md | 12 ++--- openkb/cli.py | 12 ++--- openkb/config.py | 39 ++++++++-------- openkb/indexer.py | 24 +++++----- tests/test_add_command.py | 4 +- tests/test_config.py | 76 +++++++++++--------------------- tests/test_indexer.py | 25 +++++++---- 8 files changed, 93 insertions(+), 108 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index caa3c6b5..93fad530 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -2,10 +2,11 @@ model: gpt-5.4 # LLM model (any LiteLLM-supported provider) language: en # Wiki output language pageindex_threshold: 20 # PDF pages threshold for PageIndex -# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many -# open files" on large PDFs. Omit either to use the default. -# pageindex_max_concurrency: 10 # PageIndex indexing concurrency (null = PageIndex's own default) -# compile_concurrency: 5 # concept/entity page generation concurrency +# Optional: cap concurrent LLM calls during ingest (PageIndex indexing and +# concept/entity compilation — they never overlap, so one setting covers +# both). Lower it if you hit provider rate limits or "too many open files" on +# large PDFs. Omit to let each stage apply its own default. +# concurrency: 5 # Optional: override the entity-type vocabulary used for entity pages. # Omit this key to use the default 7 types diff --git a/examples/configuration/README.md b/examples/configuration/README.md index 5530e553..6790cc80 100644 --- a/examples/configuration/README.md +++ b/examples/configuration/README.md @@ -70,10 +70,11 @@ model: gpt-5.4 # LLM model (any LiteLLM-supported provider) language: en # Wiki output language pageindex_threshold: 20 # PDF pages threshold for PageIndex -# Optional: cap concurrent LLM calls to avoid provider rate limits or "too many -# open files" on large PDFs. Omit either to use the default. -# pageindex_max_concurrency: 10 # PageIndex indexing concurrency (null = PageIndex's own default) -# compile_concurrency: 5 # concept/entity page generation concurrency +# Optional: cap concurrent LLM calls during ingest (PageIndex indexing and +# concept/entity compilation — they never overlap, so one setting covers +# both). Lower it if you hit provider rate limits or "too many open files" on +# large PDFs. Omit to let each stage apply its own default. +# concurrency: 5 # Optional: override the entity-type vocabulary used for entity pages. # Omit this key to use the default 7 types @@ -100,8 +101,7 @@ pageindex_threshold: 20 # PDF pages threshold for PageIndex | `model` | `gpt-5.4` | LLM used for all compile/query/chat work. | | `language` | `en` | Language the wiki is written in. | | `pageindex_threshold` | `20` | PDFs with this many pages **or more** take the long-doc (PageIndex) path; shorter ones go through the short-doc path. See [`pageindex-cloud/`](../pageindex-cloud/). | -| `pageindex_max_concurrency` | `null` | Caps concurrent indexing LLM calls PageIndex makes for a single long document. Lower it if you hit provider rate limits or "too many open files" on large PDFs. `null` lets PageIndex apply its own default. | -| `compile_concurrency` | `5` | Caps concurrent LLM calls OpenKB makes while generating concept/entity pages. Lower it if your LLM provider rate-limits. | +| `concurrency` | `null` | Caps concurrent LLM calls OpenKB makes during ingest — both PageIndex's indexing of a long document and OpenKB's own concept/entity compilation. The two never run at once for the same document, so one setting covers both. Lower it if you hit provider rate limits or "too many open files" on large PDFs. `null` lets each stage apply its own default. | | `entity_types` | 7 defaults | Custom vocabulary for entity pages. `other` is always kept. | | `litellm:` | – | A pass-through block for LiteLLM. See below. | diff --git a/openkb/cli.py b/openkb/cli.py index 61c2fa67..56ca1031 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -47,14 +47,14 @@ def filter(self, record: logging.LogRecord) -> bool: litellm.suppress_debug_info = True from dotenv import load_dotenv -from openkb.agent.compiler import compile_long_doc +from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY, compile_long_doc from openkb.config import ( DEFAULT_CONFIG, load_config, save_config, load_global_config, register_kb, - resolve_compile_concurrency, + resolve_concurrency, resolve_extra_headers, set_extra_headers, resolve_timeout, @@ -556,7 +556,7 @@ def commit_body(snapshot) -> None: kb_dir, model, doc_description=index_result.description, - max_concurrency=resolve_compile_concurrency(config), + max_concurrency=resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY, ), label=f"Compiling long doc (doc_id={index_result.doc_id})", ) @@ -570,7 +570,7 @@ def commit_body(snapshot) -> None: source_path, kb_dir, model, - max_concurrency=resolve_compile_concurrency(config), + max_concurrency=resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY, ), label="Compiling short doc", ) @@ -692,7 +692,7 @@ def commit_body(_snapshot) -> None: kb_dir, model, doc_description=cloud.description, - max_concurrency=resolve_compile_concurrency(config), + max_concurrency=resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY, ), label=f"Compiling imported doc (doc_id={doc_id})", ) @@ -1645,7 +1645,7 @@ def _classify(meta: dict) -> str: _setup_llm_key(kb_dir) config = load_config(openkb_dir / "config.yaml") model: str = config.get("model", DEFAULT_CONFIG["model"]) - max_concurrency = resolve_compile_concurrency(config) + max_concurrency = resolve_concurrency(config) or DEFAULT_COMPILE_CONCURRENCY # Import lazily and reference via the module so tests can patch # ``openkb.agent.compiler.compile_*`` and see the call. diff --git a/openkb/config.py b/openkb/config.py index 45edea05..99b69864 100644 --- a/openkb/config.py +++ b/openkb/config.py @@ -17,13 +17,13 @@ "model": "gpt-5.4", "language": "en", "pageindex_threshold": 20, - # Cap on concurrent indexing LLM calls PageIndex makes for a single long - # document. None = let PageIndex apply its own default. Raise it to index - # faster, lower it if you hit provider rate limits or "too many open files". - "pageindex_max_concurrency": None, - # Cap on concurrent compile LLM calls OpenKB makes when generating concept - # and entity pages. Lower it if your LLM provider rate-limits. - "compile_concurrency": 5, + # Cap on concurrent LLM calls OpenKB makes during ingest — both PageIndex's + # indexing of a long document and OpenKB's own concept/entity compilation. + # These never run concurrently with each other for the same document, so + # one knob covers both. None = each stage applies its own built-in default. + # Lower it if you hit provider rate limits or "too many open files"; raise + # it to go faster. + "concurrency": None, } # Default entity-type vocabulary. Overridable per-KB via the optional @@ -183,24 +183,27 @@ def resolve_timeout(config: dict) -> float | None: return value -def resolve_compile_concurrency(config: dict) -> int: - """Resolve the optional ``compile_concurrency:`` key for the compile step - (concept/entity generation). +def resolve_concurrency(config: dict) -> int | None: + """Resolve the optional ``concurrency:`` key — the cap on concurrent LLM + calls OpenKB makes during ingest (PageIndex indexing and concept/entity + compilation alike; they never run at the same time for one document, so + one setting covers both). - Returns ``DEFAULT_CONFIG["compile_concurrency"]`` when absent, ``None``, or - invalid; rejects bools and non-positive values, warning when present but - unusable (an explicit ``null`` is the normal "use the default" case and - warns silently, matching ``resolve_timeout``). + Returns ``None`` when absent, explicitly ``null``, or invalid (rejecting + bools and non-positive values with a warning when present but unusable — + an explicit ``null``/absent key is the normal "unset" case and stays + silent, matching ``resolve_timeout``). Callers apply their own default (or + omit the setting entirely) when this returns ``None``. """ - value = config.get("compile_concurrency") + value = config.get("concurrency") if value is None: - return DEFAULT_CONFIG["compile_concurrency"] + return None if isinstance(value, bool) or not isinstance(value, int) or value <= 0: logger.warning( - "config: 'compile_concurrency' must be a positive integer, got %r — using default.", + "config: 'concurrency' must be a positive integer, got %r — ignoring it.", value, ) - return DEFAULT_CONFIG["compile_concurrency"] + return None return value diff --git a/openkb/indexer.py b/openkb/indexer.py index 03420989..3b55fc42 100644 --- a/openkb/indexer.py +++ b/openkb/indexer.py @@ -11,7 +11,7 @@ from pageindex import IndexConfig, PageIndexClient -from openkb.config import load_config +from openkb.config import load_config, resolve_concurrency from openkb.tree_renderer import render_summary_md logger = logging.getLogger(__name__) @@ -156,26 +156,26 @@ def _write_long_doc_artifacts( def _build_index_config(config: dict[str, Any]) -> IndexConfig: """Build the PageIndex ``IndexConfig`` for local indexing. - Forwards the optional ``pageindex_max_concurrency`` KB setting to PageIndex, - which caps how many indexing LLM calls run at once (guarding against the - "too many open files" fd exhaustion on large documents). The value is only - passed when set *and* the installed PageIndex's ``IndexConfig`` declares the - field, so OpenKB keeps working against a pinned PageIndex that predates it - (``IndexConfig`` forbids unknown kwargs). + Forwards the KB's ``concurrency`` setting to PageIndex, which caps how many + indexing LLM calls run at once (guarding against "too many open files" fd + exhaustion on large documents). The value is only passed when set *and* the + installed PageIndex's ``IndexConfig`` declares the field, so OpenKB keeps + working against a pinned PageIndex that predates it (``IndexConfig`` + forbids unknown kwargs). """ kwargs: dict[str, Any] = { "if_add_node_text": True, "if_add_node_summary": True, "if_add_doc_description": True, } - max_concurrency = config.get("pageindex_max_concurrency") - if max_concurrency is not None: + concurrency = resolve_concurrency(config) + if concurrency is not None: if "max_concurrency" in IndexConfig.model_fields: - kwargs["max_concurrency"] = max_concurrency + kwargs["max_concurrency"] = concurrency else: logger.warning( - "config: 'pageindex_max_concurrency' is set but the installed " - "PageIndex version does not support it yet — ignoring it." + "config: 'concurrency' is set but the installed PageIndex " + "version does not support it yet — ignoring it." ) return IndexConfig(**kwargs) diff --git a/tests/test_add_command.py b/tests/test_add_command.py index 2447d189..3f51788e 100644 --- a/tests/test_add_command.py +++ b/tests/test_add_command.py @@ -97,14 +97,14 @@ def test_add_single_file_compile_failure_rolls_back_converted_artifacts(self, tm assert not (kb_dir / "wiki" / "sources" / "notes.md").exists() assert HashRegistry(kb_dir / ".openkb" / "hashes.json").all_entries() == {} - def test_add_forwards_compile_concurrency_from_config(self, tmp_path): + def test_add_forwards_concurrency_from_config(self, tmp_path): from unittest.mock import AsyncMock from openkb.cli import add_single_file kb_dir = self._setup_kb(tmp_path) (kb_dir / ".openkb" / "config.yaml").write_text( - "model: gpt-4o-mini\ncompile_concurrency: 3\n", encoding="utf-8" + "model: gpt-4o-mini\nconcurrency: 3\n", encoding="utf-8" ) doc = tmp_path / "notes.md" doc.write_text("# Notes\n\nBody", encoding="utf-8") diff --git a/tests/test_config.py b/tests/test_config.py index 71274faf..90bbbcbb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,7 @@ get_extra_headers, get_timeout, load_config, - resolve_compile_concurrency, + resolve_concurrency, resolve_extra_headers, resolve_litellm_settings, resolve_timeout, @@ -27,79 +27,53 @@ def test_default_config_values(): assert DEFAULT_CONFIG["pageindex_threshold"] == 20 -def test_pageindex_max_concurrency_defaults_to_none(): - assert DEFAULT_CONFIG["pageindex_max_concurrency"] is None +def test_concurrency_defaults_to_none(): + assert DEFAULT_CONFIG["concurrency"] is None -def test_load_pageindex_max_concurrency_override(tmp_path): +def test_load_concurrency_override(tmp_path): config_path = tmp_path / "config.yaml" - config_path.write_text("pageindex_max_concurrency: 12\n", encoding="utf-8") - assert load_config(config_path)["pageindex_max_concurrency"] == 12 + config_path.write_text("concurrency: 12\n", encoding="utf-8") + assert load_config(config_path)["concurrency"] == 12 -def test_compile_concurrency_defaults_to_5(): - assert DEFAULT_CONFIG["compile_concurrency"] == 5 +def test_resolve_concurrency_absent_is_none(): + assert resolve_concurrency({}) is None -def test_load_compile_concurrency_override(tmp_path): - config_path = tmp_path / "config.yaml" - config_path.write_text("compile_concurrency: 3\n", encoding="utf-8") - assert load_config(config_path)["compile_concurrency"] == 3 - - -def test_resolve_compile_concurrency_absent_uses_default(): - assert resolve_compile_concurrency({}) == DEFAULT_CONFIG["compile_concurrency"] - +def test_resolve_concurrency_valid_value(): + assert resolve_concurrency({"concurrency": 3}) == 3 -def test_resolve_compile_concurrency_valid_value(): - assert resolve_compile_concurrency({"compile_concurrency": 3}) == 3 - -def test_resolve_compile_concurrency_rejects_bool(caplog): +def test_resolve_concurrency_rejects_bool(caplog): with caplog.at_level(logging.WARNING, logger="openkb.config"): - result = resolve_compile_concurrency({"compile_concurrency": True}) - assert result == DEFAULT_CONFIG["compile_concurrency"] - assert "compile_concurrency" in caplog.text + result = resolve_concurrency({"concurrency": True}) + assert result is None + assert "concurrency" in caplog.text -def test_resolve_compile_concurrency_rejects_non_positive(caplog): +def test_resolve_concurrency_rejects_non_positive(caplog): with caplog.at_level(logging.WARNING, logger="openkb.config"): - assert ( - resolve_compile_concurrency({"compile_concurrency": 0}) - == DEFAULT_CONFIG["compile_concurrency"] - ) - assert "compile_concurrency" in caplog.text + assert resolve_concurrency({"concurrency": 0}) is None + assert "concurrency" in caplog.text caplog.clear() with caplog.at_level(logging.WARNING, logger="openkb.config"): - assert ( - resolve_compile_concurrency({"compile_concurrency": -1}) - == DEFAULT_CONFIG["compile_concurrency"] - ) - assert "compile_concurrency" in caplog.text + assert resolve_concurrency({"concurrency": -1}) is None + assert "concurrency" in caplog.text -def test_resolve_compile_concurrency_rejects_non_int(): - assert ( - resolve_compile_concurrency({"compile_concurrency": "3"}) - == DEFAULT_CONFIG["compile_concurrency"] - ) +def test_resolve_concurrency_rejects_non_int(): + assert resolve_concurrency({"concurrency": "3"}) is None -def test_resolve_compile_concurrency_none_is_silent(caplog): - # Explicit null / absent is the normal "use the default" case — no warning. +def test_resolve_concurrency_none_is_silent(caplog): + # Explicit null / absent is the normal "unset — caller applies its own + # default, or omits the setting entirely" case — no warning. with caplog.at_level(logging.WARNING, logger="openkb.config"): - resolve_compile_concurrency({"compile_concurrency": None}) + assert resolve_concurrency({"concurrency": None}) is None assert caplog.text == "" -def test_compile_concurrency_defaults_match_compiler_default(): - """DEFAULT_CONFIG and the compiler's own DEFAULT_COMPILE_CONCURRENCY are two - independent literals; catch drift immediately if only one is ever updated.""" - from openkb.agent.compiler import DEFAULT_COMPILE_CONCURRENCY - - assert DEFAULT_CONFIG["compile_concurrency"] == DEFAULT_COMPILE_CONCURRENCY - - def test_load_missing_file_returns_defaults(tmp_path): missing = tmp_path / "nonexistent" / "config.yaml" config = load_config(missing) diff --git a/tests/test_indexer.py b/tests/test_indexer.py index 4da2a1ef..e0843fa8 100644 --- a/tests/test_indexer.py +++ b/tests/test_indexer.py @@ -54,26 +54,33 @@ def test_sets_base_flags(self): assert cfg.if_add_node_summary is True assert cfg.if_add_doc_description is True - def test_forwards_max_concurrency_when_supported(self, monkeypatch): + def test_forwards_concurrency_when_supported(self, monkeypatch): monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) - cfg = _build_index_config({"pageindex_max_concurrency": 8}) + cfg = _build_index_config({"concurrency": 8}) assert cfg.max_concurrency == 8 def test_does_not_forward_when_unsupported(self, monkeypatch): monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency) - cfg = _build_index_config({"pageindex_max_concurrency": 8}) + cfg = _build_index_config({"concurrency": 8}) assert not hasattr(cfg, "max_concurrency") def test_none_value_is_left_to_pageindex_default(self, monkeypatch): monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) - cfg = _build_index_config({"pageindex_max_concurrency": None}) + cfg = _build_index_config({"concurrency": None}) + assert getattr(cfg, "max_concurrency", None) is None + + def test_invalid_value_is_left_to_pageindex_default(self, monkeypatch): + # resolve_concurrency() rejects bools/non-positive values — same as an + # unset key, just via the shared config-level validation. + monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) + cfg = _build_index_config({"concurrency": 0}) assert getattr(cfg, "max_concurrency", None) is None def test_warns_when_configured_but_unsupported(self, monkeypatch, caplog): monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency) with caplog.at_level(logging.WARNING, logger="openkb.indexer"): - _build_index_config({"pageindex_max_concurrency": 8}) - assert "pageindex_max_concurrency" in caplog.text + _build_index_config({"concurrency": 8}) + assert "concurrency" in caplog.text def test_no_warning_when_unset(self, monkeypatch, caplog): monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithoutConcurrency) @@ -84,7 +91,7 @@ def test_no_warning_when_unset(self, monkeypatch, caplog): def test_no_warning_when_supported(self, monkeypatch, caplog): monkeypatch.setattr("openkb.indexer.IndexConfig", _FakeIndexConfigWithConcurrency) with caplog.at_level(logging.WARNING, logger="openkb.indexer"): - _build_index_config({"pageindex_max_concurrency": 8}) + _build_index_config({"concurrency": 8}) assert caplog.text == "" @@ -280,12 +287,12 @@ def test_localclient_called_with_index_config(self, kb_dir, sample_tree, tmp_pat assert ic.if_add_node_summary is True assert ic.if_add_doc_description is True - def test_pageindex_max_concurrency_flows_from_kb_config(self, kb_dir, sample_tree, tmp_path): + def test_concurrency_flows_from_kb_config(self, kb_dir, sample_tree, tmp_path): """The KB's real config.yaml, loaded by index_long_document itself, must reach the IndexConfig passed to PageIndexClient — not just the isolated _build_index_config unit tested directly with a hand-built dict.""" (kb_dir / ".openkb" / "config.yaml").write_text( - "model: gpt-4o-mini\npageindex_max_concurrency: 7\n", encoding="utf-8" + "model: gpt-4o-mini\nconcurrency: 7\n", encoding="utf-8" ) doc_id = "conc-789" From 792e0b803ea0ab18646684c6e74fe2e27791e014 Mon Sep 17 00:00:00 2001 From: mountain Date: Wed, 8 Jul 2026 18:10:46 +0800 Subject: [PATCH 5/8] build(deps): bump pageindex to 0.3.0.dev2 (ships IndexConfig.max_concurrency) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.3.0.dev2 is the first published release that declares IndexConfig.max_concurrency, so `concurrency` in config.yaml now takes effect for the indexing stage in a normal `pip install` / `uv sync`, not just against a local editable checkout. Vetted: downloaded the wheel and confirmed its sha256 matches PyPI (b0cb1f6e…) and that IndexConfig declares `max_concurrency: int | None` with a positivity field_validator. uv.lock regenerated via `uv lock --upgrade-package pageindex`. The runtime `"max_concurrency" in IndexConfig.model_fields` guard in indexer._build_index_config is now always-true under the pinned dependency, but is kept as graceful degradation for mismatched local installs. --- pyproject.toml | 2 +- uv.lock | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c9c79b4e..c59d948b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ keywords = ["ai", "rag", "retrieval", "knowledge-base", "llm", "pageindex", "age # returning empty Responses output (BerriAI/litellm#25429) and # auto-injects GitHub Copilot IDE-auth headers. dependencies = [ - "pageindex==0.3.0.dev1", + "pageindex==0.3.0.dev2", "markitdown[docx,pptx,xlsx,xls]==0.1.5", "trafilatura==2.0.0", "click==8.4.0", diff --git a/uv.lock b/uv.lock index bd4157c6..10f2a8b1 100644 --- a/uv.lock +++ b/uv.lock @@ -1975,7 +1975,7 @@ requires-dist = [ { name = "markitdown", extras = ["docx", "pptx", "xls", "xlsx"], specifier = "==0.1.5" }, { name = "mypy", marker = "extra == 'dev'", specifier = "==1.15.0" }, { name = "openai-agents", specifier = "==0.17.3" }, - { name = "pageindex", specifier = "==0.3.0.dev1" }, + { name = "pageindex", specifier = "==0.3.0.dev2" }, { name = "portalocker", specifier = "==3.2.0" }, { name = "prompt-toolkit", specifier = "==3.0.52" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==9.0.3" }, @@ -2013,21 +2013,24 @@ wheels = [ [[package]] name = "pageindex" -version = "0.3.0.dev1" +version = "0.3.0.dev2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx", extra = ["socks"] }, { name = "litellm" }, + { name = "openai" }, { name = "openai-agents" }, + { name = "pydantic" }, { name = "pymupdf" }, { name = "pypdf2" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/27/318e73fe1ed2194b19abeb53f57f30b022c9ea4a3ebfe8c2b293f239d537/pageindex-0.3.0.dev1.tar.gz", hash = "sha256:77f02a85622a180918a11d4c4e4e97c10dc0ba142c6418ab1c501d6c1301a409", size = 61010, upload-time = "2026-04-10T17:20:58.524Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/87/9bf0e64e04509a453b110b09638eee297d8e046e8f709d13dc88d25a269b/pageindex-0.3.0.dev2.tar.gz", hash = "sha256:8d36899e5755710d82729f18bdb70f85de71cc2fcde0491749188ae6c8f3065f", size = 66284, upload-time = "2026-07-08T10:04:30.528Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/7b/a2d37a28475605b715f9fe28320989aec753c023b272fbf9c4bf9f693a55/pageindex-0.3.0.dev1-py3-none-any.whl", hash = "sha256:570c7fd7e54d62782030154dfe02b2f6b9e88407a293f96ea824203716ef28d3", size = 66954, upload-time = "2026-04-10T17:20:56.738Z" }, + { url = "https://files.pythonhosted.org/packages/93/99/7b24f5b7f36b6827f9c94d540248e79444103b5e2cb06ffe5ab0e8e8fe7f/pageindex-0.3.0.dev2-py3-none-any.whl", hash = "sha256:b0cb1f6e83e1a81d81f00b5b7bf229e1850fe54591fbb06d209fec0b0e63cf3b", size = 71889, upload-time = "2026-07-08T10:04:29.092Z" }, ] [[package]] From a6ae2bdb529d6db3627c8e8a34fc2d7eb2690ebc Mon Sep 17 00:00:00 2001 From: mountain Date: Fri, 10 Jul 2026 15:17:10 +0800 Subject: [PATCH 6/8] refactor(config): keep concurrency out of DEFAULT_CONFIG (align with other optional knobs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other optional tuning knob (timeout, extra_headers, litellm, entity_types, parallel_tool_calls) is resolved via its resolver's .get() and lives only in config.yaml.example, not DEFAULT_CONFIG — which holds just the core keys openkb init writes (model, language, pageindex_threshold). concurrency was the lone exception; resolve_concurrency already reads it via .get(), so the DEFAULT_CONFIG entry was redundant. Remove it for consistency. --- openkb/config.py | 7 ------- tests/test_config.py | 8 ++++++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openkb/config.py b/openkb/config.py index 72fd8e23..d4e7e557 100644 --- a/openkb/config.py +++ b/openkb/config.py @@ -17,13 +17,6 @@ "model": "gpt-5.4", "language": "en", "pageindex_threshold": 20, - # Cap on concurrent LLM calls OpenKB makes during ingest — both PageIndex's - # indexing of a long document and OpenKB's own concept/entity compilation. - # These never run concurrently with each other for the same document, so - # one knob covers both. None = each stage applies its own built-in default. - # Lower it if you hit provider rate limits or "too many open files"; raise - # it to go faster. - "concurrency": None, } # Default entity-type vocabulary. Overridable per-KB via the optional diff --git a/tests/test_config.py b/tests/test_config.py index 3d6bbf0f..ace8a4e9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -136,8 +136,12 @@ def test_default_config_values(): assert DEFAULT_CONFIG["pageindex_threshold"] == 20 -def test_concurrency_defaults_to_none(): - assert DEFAULT_CONFIG["concurrency"] is None +def test_concurrency_not_in_default_config(): + # Like the other optional tuning knobs (timeout, extra_headers, + # parallel_tool_calls), concurrency stays out of DEFAULT_CONFIG — + # resolve_concurrency reads it via .get(), so an absent key resolves to + # None without relying on load_config's merge. + assert "concurrency" not in DEFAULT_CONFIG def test_load_concurrency_override(tmp_path): From d9531ecabc6f52b76751e5dfea2dc8b4a873094a Mon Sep 17 00:00:00 2001 From: mountain Date: Fri, 10 Jul 2026 15:26:03 +0800 Subject: [PATCH 7/8] build(deps): bump pageindex 0.3.0.dev2 -> 0.3.0.dev3 Vetted: identical Requires-Dist/Requires-Python to dev2; IndexConfig, PageIndexClient, and IndexConfig.max_concurrency (used by #174) all still present; wheel/sdist hashes verified against PyPI. uv.lock regenerated via `uv lock` (change scoped to the pageindex entry). Also aligns the stale dev1 reference in the config README. --- examples/configuration/README.md | 2 +- pyproject.toml | 2 +- uv.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/configuration/README.md b/examples/configuration/README.md index 4ab20568..25e5b3a8 100644 --- a/examples/configuration/README.md +++ b/examples/configuration/README.md @@ -12,7 +12,7 @@ pip install openkb ``` OpenKB pins a **pre-release** of its PageIndex dependency -(`pageindex==0.3.0.dev1`), which some installers skip by default. If an install +(`pageindex==0.3.0.dev3`), which some installers skip by default. If an install can't resolve `pageindex`, allow pre-releases: ```bash diff --git a/pyproject.toml b/pyproject.toml index c59d948b..d246bec2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ keywords = ["ai", "rag", "retrieval", "knowledge-base", "llm", "pageindex", "age # returning empty Responses output (BerriAI/litellm#25429) and # auto-injects GitHub Copilot IDE-auth headers. dependencies = [ - "pageindex==0.3.0.dev2", + "pageindex==0.3.0.dev3", "markitdown[docx,pptx,xlsx,xls]==0.1.5", "trafilatura==2.0.0", "click==8.4.0", diff --git a/uv.lock b/uv.lock index 10f2a8b1..9d39b59b 100644 --- a/uv.lock +++ b/uv.lock @@ -1975,7 +1975,7 @@ requires-dist = [ { name = "markitdown", extras = ["docx", "pptx", "xls", "xlsx"], specifier = "==0.1.5" }, { name = "mypy", marker = "extra == 'dev'", specifier = "==1.15.0" }, { name = "openai-agents", specifier = "==0.17.3" }, - { name = "pageindex", specifier = "==0.3.0.dev2" }, + { name = "pageindex", specifier = "==0.3.0.dev3" }, { name = "portalocker", specifier = "==3.2.0" }, { name = "prompt-toolkit", specifier = "==3.0.52" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==9.0.3" }, @@ -2013,7 +2013,7 @@ wheels = [ [[package]] name = "pageindex" -version = "0.3.0.dev2" +version = "0.3.0.dev3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx", extra = ["socks"] }, @@ -2028,9 +2028,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/87/9bf0e64e04509a453b110b09638eee297d8e046e8f709d13dc88d25a269b/pageindex-0.3.0.dev2.tar.gz", hash = "sha256:8d36899e5755710d82729f18bdb70f85de71cc2fcde0491749188ae6c8f3065f", size = 66284, upload-time = "2026-07-08T10:04:30.528Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/58/92b159e86e73af3a61aedea05e9390181d83e4a6c717e37845a7cf1eedd5/pageindex-0.3.0.dev3.tar.gz", hash = "sha256:93174bbfcd261c5e7d6c7d47f1be371ae21c1fe977ce706cf3c021f4932693bd", size = 76083, upload-time = "2026-07-10T07:00:30.522Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/99/7b24f5b7f36b6827f9c94d540248e79444103b5e2cb06ffe5ab0e8e8fe7f/pageindex-0.3.0.dev2-py3-none-any.whl", hash = "sha256:b0cb1f6e83e1a81d81f00b5b7bf229e1850fe54591fbb06d209fec0b0e63cf3b", size = 71889, upload-time = "2026-07-08T10:04:29.092Z" }, + { url = "https://files.pythonhosted.org/packages/3c/29/34a2a1eaed83be5aeb879bf85bff7be9f6faa76fe70718479d6426ce21aa/pageindex-0.3.0.dev3-py3-none-any.whl", hash = "sha256:5056969108785f5c9c31e03ce60f5fa1043be96ecffb5a06353375212027b258", size = 81956, upload-time = "2026-07-10T07:00:29.163Z" }, ] [[package]] From 8aa37554d67ee11e65382d408d11dbb05200c481 Mon Sep 17 00:00:00 2001 From: mountain Date: Fri, 10 Jul 2026 16:41:57 +0800 Subject: [PATCH 8/8] feat(cli): add --version flag The CLI group had no way to report its version (`openkb version` / `openkb -v` both fail). Add `@click.version_option(package_name="openkb")` so `openkb --version` prints "openkb ", read from installed package metadata (tracks the hatch-vcs-derived version, no hardcoded string). --- openkb/cli.py | 1 + tests/test_cli.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/openkb/cli.py b/openkb/cli.py index 2c301f8e..6a37350a 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -757,6 +757,7 @@ def append_cloud_log() -> None: @click.group() +@click.version_option(package_name="openkb", prog_name="openkb", message="%(prog)s %(version)s") @click.option("-v", "--verbose", is_flag=True, default=False, help="Enable verbose logging.") @click.option( "--kb-dir", diff --git a/tests/test_cli.py b/tests/test_cli.py index 1589379b..9d7e9191 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,6 +9,17 @@ from openkb.schema import AGENTS_MD +def test_version_flag_reports_installed_version(): + import importlib.metadata + + runner = CliRunner() + result = runner.invoke(cli, ["--version"]) + assert result.exit_code == 0 + # Reports the package's installed version (via importlib.metadata), so it + # tracks the hatch-vcs-derived version without a hardcoded string. + assert importlib.metadata.version("openkb") in result.output + + def test_init_creates_structure(tmp_path): runner = CliRunner() with runner.isolated_filesystem(temp_dir=tmp_path), patch("openkb.cli.register_kb"):