Fix OWASP agent probe gaps (chapters, people, membership, CRE facts)

.env.example

@@ -41,6 +41,18 @@ CRE_ENABLE_MYOPENCRE=false
 
 CRE_ENABLE_LOGIN=false
 
+# Experimental OIE OWASP metadata agent (chat + MCP only — never CRE graph UI).
+# Off by default. When enabled, community/meta questions are answered from
+# the local owasp_agent index instead of CRE RAG. Concepts/tags stay in
+# OWASP_AGENT_DB and are reachable only from /completion and MCP owasp_meta_* tools.
+OWASP_AGENT_ENABLED=false
+# Nest REST API key (https://nest.owasp.org — create via GraphQL while logged in)
+NEST_API_KEY=
+NEST_API_BASE=https://nest.owasp.org/api/v0
+OWASP_AGENT_DB=tmp/owasp_agent.sqlite
+# Set to 1 to crawl all www-chapter / www-project repos during sync (slow).
+OWASP_AGENT_CRAWL_REPOS=0
+
 # Embeddings
 
 NO_GEN_EMBEDDINGS=false
@@ -52,14 +64,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret
 GOOGLE_SECRET_JSON=path/to/secret.json
 LOGIN_ALLOWED_DOMAINS=example.com
 
-# GCP
+# GCP / Gemini (optional — any LiteLLM provider works for Module B)
 
 GCP_NATIVE=false
 GEMINI_API_KEY=your-gemini-api-key
 VERTEX_CHAT_MODEL=gemini-2.5-flash
 VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001
 
-# Noise / Relevance Filter (Module B)
+# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GROQ_API_KEY=gsk_...
+# MISTRAL_API_KEY=...
+# OPENROUTER_API_KEY=...
+
+# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model)
+# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini |
+#           anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile
+# Any chat model LiteLLM supports; mid-size models usually work best
+# (cheap + reliable for this classify gate — avoid tiny completion stubs).
 
 CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite
 CRE_NOISE_FILTER_BATCH_SIZE=10

.gitignore

@@ -72,13 +72,16 @@ standards_cache.sqlite
 !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md
 !docs/Final_eval_blog_gsoc2026/module_B_final_blog.md
 !application/utils/librarian/README.md
+!application/utils/owasp_agent/README.md
 !docs/gsoc_2026_module_c/*.md
 !docs/gsoc_2026_module_a/*.md
 !docs/Final_eval_blog_gsoc2026/*.md
 !docs/gsoc_2026_module_b/module_a_contract.md
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md
 !docs/gsoc_2026_module_b/final_metrics.md
+!docs/oie/*.md
+!scripts/oie_owasp_eval/experiments/*.md
 
 ### Dev DBDumps
 *.sql

Makefile

@@ -1,6 +1,6 @@
 .ONESHELL:
 
-.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db
+.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval
 
 prod-run:
 	gunicorn cre:app --log-file=-
@@ -141,8 +141,12 @@ install-python:
 	make install-deps-python &&\
 	playwright install  # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress
 
-install-typescript:
-	yarn add webpack && cd application/frontend && yarn build
+# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is
+# already in package.json, and a local build rewrites
+# application/frontend/www/bundle.js (tracked so Heroku can serve the UI
+# without a Node compile step). That was dirtying contributor trees after
+# `make install`. Rebuild with `make frontend` when you change the UI.
+install-typescript: install-deps-typescript
 
 install: install-typescript install-python migrate-upgrade
 
@@ -185,10 +189,39 @@ oie-pipeline:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
 
+# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing.
+# Example:
+#   make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone
+oie-tag-base:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \
+		--cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \
+		$(OIE_TAG_ARGS)
+
+# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream).
+# Examples:
+#   make setup-oie
+#   make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate'
+setup-oie:
+	bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS)
+
 oie-e2e-smoke:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py
 
+owasp-agent-sync:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_sync.py $(OWASP_AGENT_SYNC_ARGS)
+
+owasp-agent-merge-concepts:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_merge_concepts.py $(OWASP_AGENT_MERGE_ARGS)
+
+# Score fixtures/owasp_agent_synth_v2.jsonl against OWASP_AGENT_DB (see application/utils/owasp_agent/README.md).
+owasp-agent-synth-eval:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	OWASP_AGENT_ENABLED=1 PYTHONPATH=. python scripts/owasp_agent_synth_eval.py
+
 openapi-generate:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	python scripts/generate_openapi.py

application/database/db.py

@@ -200,6 +200,9 @@ class Embeddings(BaseModel):  # type: ignore
     # vectors (legacy CSV ``embeddings`` column is dropped in that migration).
     # SQLite/tests: Text holding a pgvector literal ``[1.0,2.0,...]``.
     embedding_vec = sqla.Column(sqla.Text, nullable=False)
+    # Multi-lane chat routing: cre_graph | corpus | owasp_agent_meta
+    # (Alembic c9d0e1f2a3b4). Nullable for legacy rows until backfilled.
+    source_channel = sqla.Column(sqla.String, nullable=True, default=None)
 
 
 class GapAnalysisResults(BaseModel):
@@ -3112,6 +3115,7 @@ def add_embedding(
                     embedding_model_id=embedding_model_id,
                     embedding_dim=embedding_dim,
                     embedding_vec=embedding_vec_literal,
+                    source_channel="cre_graph",
                 )
             else:
                 emb = Embeddings(
@@ -3122,6 +3126,7 @@ def add_embedding(
                     embedding_model_id=embedding_model_id,
                     embedding_dim=embedding_dim,
                     embedding_vec=embedding_vec_literal,
+                    source_channel="corpus",
                 )
             self.session.add(emb)
             self.session.commit()

application/frontend/src/pages/chatbot/chatbot.scss

@@ -70,6 +70,13 @@ h1.ui.header.chatbot-title {
   margin-bottom: 1rem !important;
 }
 
+.chatbot-toolbar {
+  display: flex;
+  justify-content: center;
+  margin: 0 auto 0.5rem;
+  max-width: 960px;
+}
+
 /* Tablet & below */
 @media (max-width: 768px) {
   h1.ui.header.chatbot-title {
@@ -270,7 +277,7 @@ h1.ui.header.chatbot-title {
 
   margin-top: 1rem;
   background-color: #daebdb;
-/* background: transparent; */
+  /* background: transparent; */
   border: 1px solid rgb(190, 214, 232);
   padding: 0.75rem;
   border-radius: 12px;
@@ -399,7 +406,7 @@ h1.ui.header.chatbot-title {
   flex-direction: column;
   height: 100%;
 
- background: #fcfcfa; /* same palette as input */
+  background: #fcfcfa; /* same palette as input */
   backdrop-filter: blur(6px);
 
   border-radius: 18px;
@@ -434,7 +441,6 @@ h1.ui.header.chatbot-title {
    ========================= */
 
 @keyframes typingBounce {
-
   0%,
   80%,
   100% {
@@ -470,4 +476,4 @@ h1.ui.header.chatbot-title {
     opacity: 0.85;
     transform: translateX(-50%) translateY(0);
   }
-}
+}

application/frontend/src/pages/chatbot/chatbot.tsx

@@ -24,6 +24,7 @@ export const Chatbot = () => {
     message: string;
     data: Document[] | null;
     accurate: boolean;
+    owaspCitations?: string[];
   };
 
   interface ChatState {
@@ -135,6 +136,16 @@ export const Chatbot = () => {
     return res;
   }
 
+  const HISTORY_CAP = 20;
+
+  function startNewChat() {
+    setChatMessages([]);
+    setError('');
+    setChat(DEFAULT_CHAT_STATE);
+    setModelName('');
+    setShowScrollToBottom(false);
+  }
+
   async function onSubmit() {
     if (!chat.term.trim()) return;
     shouldForceScrollRef.current = true;
@@ -143,6 +154,12 @@ export const Chatbot = () => {
     setChat({ ...chat, term: '' });
     setLoading(true);
 
+    const prior = chatMessages.slice(-(HISTORY_CAP - 1)).map((m) => ({
+      role: m.role === 'assistant' ? 'assistant' : 'user',
+      content: m.message,
+    }));
+    const messages = [...prior, { role: 'user', content: currentTerm }];
+
     setChatMessages((prev) => [
       ...prev,
       {
@@ -157,7 +174,7 @@ export const Chatbot = () => {
     fetch(`${apiUrl}/completion`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
-      body: JSON.stringify({ prompt: currentTerm }),
+      body: JSON.stringify({ prompt: currentTerm, messages }),
     })
       .then(async (response) => {
         if (!response.ok) {
@@ -187,8 +204,17 @@ export const Chatbot = () => {
             timestamp: new Date().toLocaleTimeString(),
             role: 'assistant',
             message: data.response,
-            data: data.table,
+            // CRE RAG citations only — never treat owasp_agent hits as graph nodes.
+            data: Array.isArray(data.table)
+              ? data.table.filter(
+                  (d: Document) =>
+                    d &&
+                    d.doctype &&
+                    !['owaspmeta', 'owasp_meta', 'concept'].includes(String(d.doctype).toLowerCase())
+                )
+              : data.table,
             accurate: data.accurate,
+            owaspCitations: Array.isArray(data?.owasp_agent?.citations) ? data.owasp_agent.citations : [],
           },
         ]);
       })
@@ -201,6 +227,11 @@ export const Chatbot = () => {
 
   function displayDocument(d: Document) {
     if (!d || !d.doctype) return null;
+    // Never present non-CRE / meta-shaped payloads as OpenCRE graph nodes.
+    const dt = String(d.doctype).toLowerCase();
+    if (dt === 'owaspmeta' || dt === 'owasp_meta' || dt === 'concept') {
+      return null;
+    }
 
     let link = `/node/${d.doctype.toLowerCase()}/${d.name}`;
     link += d.section ? `/section/${d.section}` : `/sectionid/${d.sectionID}`;
@@ -224,6 +255,30 @@ export const Chatbot = () => {
     );
   }
 
+  function displayOwaspCitations(urls: string[]) {
+    const safe = urls.filter((url) => {
+      try {
+        const parsed = new URL(url);
+        return parsed.protocol === 'http:' || parsed.protocol === 'https:';
+      } catch {
+        return false;
+      }
+    });
+    if (!safe.length) return null;
+    return (
+      <div className="references">
+        <div className="references-title">OWASP community sources</div>
+        {safe.map((url, i) => (
+          <div className="reference-card" key={i}>
+            <a href={url} target="_blank" rel="noopener noreferrer">
+              {url}
+            </a>
+          </div>
+        ))}
+      </div>
+    );
+  }
+
   return (
     <>
       {/* user login check moved to useEffect */}
@@ -233,6 +288,13 @@ export const Chatbot = () => {
           <Header as="h1" className="chatbot-title">
             OWASP OpenCRE Chat
           </Header>
+          {hasMessages ? (
+            <div className="chatbot-toolbar">
+              <Button type="button" size="small" basic onClick={startNewChat}>
+                <Icon name="plus" /> New chat
+              </Button>
+            </div>
+          ) : null}
 
           <Container>
             <div className={`chat-container ${hasMessages ? 'chat-active' : 'chat-landing'}`}>
@@ -264,6 +326,10 @@ export const Chatbot = () => {
                           </div>
                         )}
 
+                        {m.owaspCitations && m.owaspCitations.length > 0
+                          ? displayOwaspCitations(m.owaspCitations)
+                          : null}
+
                         {!m.accurate && (
                           <div className="accuracy-warning">
                             This answer could not be fully verified against OpenCRE sources. Please validate