feat(oie): full-pipeline runner, docs, and experimental-flag notes

.env.example

@@ -52,14 +52,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret
 GOOGLE_SECRET_JSON=path/to/secret.json
 LOGIN_ALLOWED_DOMAINS=example.com
 
-# GCP
+# GCP / Gemini (optional — any LiteLLM provider works for Module B)
 
 GCP_NATIVE=false
 GEMINI_API_KEY=your-gemini-api-key
 VERTEX_CHAT_MODEL=gemini-2.5-flash
 VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001
 
-# Noise / Relevance Filter (Module B)
+# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GROQ_API_KEY=gsk_...
+# MISTRAL_API_KEY=...
+# OPENROUTER_API_KEY=...
+
+# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model)
+# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini |
+#           anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile
+# Any chat model LiteLLM supports; mid-size models usually work best
+# (cheap + reliable for this classify gate — avoid tiny completion stubs).
 
 CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite
 CRE_NOISE_FILTER_BATCH_SIZE=10

.gitignore

@@ -79,6 +79,8 @@ standards_cache.sqlite
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md
 !docs/gsoc_2026_module_b/final_metrics.md
+!docs/oie/*.md
+!scripts/oie_owasp_eval/experiments/*.md
 
 ### Dev DBDumps
 *.sql

Makefile

@@ -141,8 +141,12 @@ install-python:
 	make install-deps-python &&\
 	playwright install  # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress
 
-install-typescript:
-	yarn add webpack && cd application/frontend && yarn build
+# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is
+# already in package.json, and a local build rewrites
+# application/frontend/www/bundle.js (tracked so Heroku can serve the UI
+# without a Node compile step). That was dirtying contributor trees after
+# `make install`. Rebuild with `make frontend` when you change the UI.
+install-typescript: install-deps-typescript
 
 install: install-typescript install-python migrate-upgrade
 
@@ -185,6 +189,22 @@ oie-pipeline:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
 
+# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing.
+# Example:
+#   make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone
+oie-tag-base:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \
+		--cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \
+		$(OIE_TAG_ARGS)
+
+# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream).
+# Examples:
+#   make setup-oie
+#   make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate'
+setup-oie:
+	bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS)
+
 oie-e2e-smoke:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py

application/prompt_client/litellm_router.py

@@ -8,15 +8,17 @@
 
 from __future__ import annotations
 
-from cre_logging import get_logger
-
-logger = get_logger(__name__)
-
+import logging
 import os
 import time
 from typing import Any, Callable, List, Optional, Sequence
 
-from application.prompt_client.llm_error_utils import is_rate_limit_error
+from application.prompt_client.llm_error_utils import (
+    is_rate_limit_error,
+    is_transient_llm_error,
+)
+
+logger = logging.getLogger(__name__)
 
 LlmFn = Callable[[str, str], str]
 
@@ -47,21 +49,26 @@ def with_rate_limit_retry(
     max_retries: Optional[int] = None,
     retry_sleep_seconds: Optional[int] = None,
 ) -> Any:
+    """Retry rate limits and transient connection/timeout blips."""
     default_retries, default_sleep = retry_policy()
     retries = default_retries if max_retries is None else max_retries
     sleep_s = default_sleep if retry_sleep_seconds is None else retry_sleep_seconds
     for attempt in range(retries + 1):
         try:
             return fn()
         except Exception as err:
-            if not is_rate_limit_error(err) or attempt >= retries:
+            retriable = is_transient_llm_error(err) or is_rate_limit_error(err)
+            if not retriable or attempt >= retries:
                 raise
+            kind = "rate/quota" if is_rate_limit_error(err) else "transient"
             logger.info(
-                "rate/quota limited during %s; sleeping %ss (attempt %s/%s)",
+                "%s error during %s; sleeping %ss (attempt %s/%s): %s",
+                kind,
                 context,
                 sleep_s,
                 attempt + 1,
                 retries + 1,
+                type(err).__name__,
             )
             time.sleep(sleep_s)
     raise RuntimeError("unreachable: retry loop exited unexpectedly")

application/prompt_client/llm_error_utils.py

@@ -31,3 +31,38 @@ def is_rate_limit_error(err: BaseException) -> bool:
             if code == 429:
                 return True
     return False
+
+
+def is_transient_llm_error(err: BaseException) -> bool:
+    """Connection / timeout blips that are safe to retry (not model refusal)."""
+    if is_rate_limit_error(err):
+        return True
+    name = type(err).__name__.lower()
+    if any(
+        tok in name
+        for tok in (
+            "connect",
+            "timeout",
+            "apiconnection",
+            "remoteprotocol",
+            "network",
+        )
+    ):
+        return True
+    msg = str(err).lower()
+    return any(
+        tok in msg
+        for tok in (
+            "connection reset",
+            "connection aborted",
+            "connection refused",
+            "temporarily unavailable",
+            "broken pipe",
+            "timed out",
+            "timeout",
+            "errno 54",
+            "errno 104",
+            "server disconnected",
+            "remote end closed",
+        )
+    )