Admin showcase panel for signed-in users

.env.example

@@ -5,6 +5,10 @@
 # Never commit your real `.env` file to version control.
 # ------------------------------------------------------------------------------
 # Database
+# Local Docker (make docker-postgres / make admin-local):
+#   postgresql://cre:password@127.0.0.1:5432/cre
+# `make admin-local` starts that container when the URL is unreachable and
+# never falls back to SQLite for the admin showcase.
 
 DEV_DATABASE_URL=postgresql://user:password@localhost:5432/opencre
 
@@ -41,6 +45,21 @@ CRE_ENABLE_MYOPENCRE=false
 
 CRE_ENABLE_LOGIN=false
 
+# Experimental OIE OWASP metadata agent (chat + MCP only — never CRE graph UI).
+# Off by default. Uses the main app Postgres (DEV_DATABASE_URL / DATABASE_URL)
+# for the agent index + pgvector. Reachable from /completion and MCP owasp_meta_* tools.
+OWASP_AGENT_ENABLED=false
+
+# Module C.1 graph filing (Automatically linked to). Off by default — leave Module C
+# decisions in decision_queue for review; set true to auto-add high-confidence links.
+OIE_GRAPH_FILING_ENABLED=false
+
+# Nest REST API key (https://nest.owasp.org — create via GraphQL while logged in)
+NEST_API_KEY=
+NEST_API_BASE=https://nest.owasp.org/api/v0
+# Set to 1 to crawl all www-chapter / www-project repos during sync (slow).
+OWASP_AGENT_CRAWL_REPOS=0
+
 # Embeddings
 
 NO_GEN_EMBEDDINGS=false
@@ -52,14 +71,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret
 GOOGLE_SECRET_JSON=path/to/secret.json
 LOGIN_ALLOWED_DOMAINS=example.com
 
-# GCP
+# GCP / Gemini (optional — any LiteLLM provider works for Module B)
 
 GCP_NATIVE=false
 GEMINI_API_KEY=your-gemini-api-key
 VERTEX_CHAT_MODEL=gemini-2.5-flash
 VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001
 
-# Noise / Relevance Filter (Module B)
+# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GROQ_API_KEY=gsk_...
+# MISTRAL_API_KEY=...
+# OPENROUTER_API_KEY=...
+
+# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model)
+# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini |
+#           anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile
+# Any chat model LiteLLM supports; mid-size models usually work best
+# (cheap + reliable for this classify gate — avoid tiny completion stubs).
 
 CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite
 CRE_NOISE_FILTER_BATCH_SIZE=10

.gitignore

@@ -72,13 +72,16 @@ standards_cache.sqlite
 !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md
 !docs/Final_eval_blog_gsoc2026/module_B_final_blog.md
 !application/utils/librarian/README.md
+!application/utils/owasp_agent/README.md
 !docs/gsoc_2026_module_c/*.md
 !docs/gsoc_2026_module_a/*.md
 !docs/Final_eval_blog_gsoc2026/*.md
 !docs/gsoc_2026_module_b/module_a_contract.md
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md
 !docs/gsoc_2026_module_b/final_metrics.md
+!docs/oie/*.md
+!scripts/oie_owasp_eval/experiments/*.md
 
 ### Dev DBDumps
 *.sql
@@ -91,6 +94,8 @@ neo4j/
 .mypy_cache
 tmp/
 
+import_telemetry.json
+
 ### CREs dir
 cres/*
 ### Local project management tooling

Makefile

@@ -1,6 +1,6 @@
 .ONESHELL:
 
-.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db
+.PHONY: run test covers install-deps dev dev-flask docker lint frontend clean all e2e e2e-db admin-local owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval
 
 prod-run:
 	gunicorn cre:app --log-file=-
@@ -74,11 +74,19 @@ start-worker:
 	. ./venv/bin/activate && FLASK_APP=`pwd`/cre.py python cre.py --start_worker
 
 upstream-sync:
-	. ./venv/bin/activate && python cre.py --upstream_sync
+	. ./venv/bin/activate &&\
+	python cre.py --upstream_sync --cache_file "$(or $(DEV_DATABASE_URL),$(CRE_CACHE_FILE),$(CURDIR)/standards_cache.sqlite)"
 
 PORT?=5000
 
-dev-flask:
+# Local app against Docker Postgres: migrate, upstream-sync if cre/node empty, Flask.
+# Same entrypoint as admin showcase / OIE local (no SQLite). macOS AirPlay often
+# holds :5000 — use PORT=5001 make dev if bind fails.
+dev dev-flask admin-local:
+	bash ./scripts/run_admin_local.sh --port $(PORT) $(ADMIN_LOCAL_ARGS)
+
+# Bare Flask without Postgres bootstrap (legacy / CI helpers). Prefer `make dev`.
+dev-flask-sqlite:
 	. ./venv/bin/activate && INSECURE_REQUESTS=1 FLASK_APP=`pwd`/cre.py  FLASK_CONFIG=development flask run --port $(PORT)
 
 dev-flask-docker:
@@ -136,15 +144,28 @@ install-deps-typescript:
 install-deps: install-deps-python install-deps-typescript
 
 install-python:
-	virtualenv -p python3 venv
+	@if [ ! -d "./venv" ]; then \
+		if command -v virtualenv >/dev/null 2>&1; then \
+			virtualenv -p python3 venv; \
+		else \
+			echo "virtualenv not on PATH — using python3 -m venv"; \
+			python3 -m venv venv; \
+		fi; \
+	fi
 	. ./venv/bin/activate &&\
 	make install-deps-python &&\
 	playwright install  # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress
 
-install-typescript:
-	yarn add webpack && cd application/frontend && yarn build
+# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is
+# already in package.json, and a local build rewrites
+# application/frontend/www/bundle.js (tracked so Heroku can serve the UI
+# without a Node compile step). That was dirtying contributor trees after
+# `make install`. Rebuild with `make frontend` when you change the UI.
+install-typescript: install-deps-typescript
 
-install: install-typescript install-python migrate-upgrade
+# Deps + Docker Postgres schema + CRE upstream graph when empty (see run_admin_local.sh).
+install: install-typescript install-python
+	bash ./scripts/run_admin_local.sh --migrate-only $(ADMIN_LOCAL_ARGS)
 
 docker-dev:
 	docker build -f Dockerfile-dev -t opencre-dev:$(shell git rev-parse HEAD) .
@@ -185,10 +206,39 @@ oie-pipeline:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
 
+# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing.
+# Example:
+#   make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone
+oie-tag-base:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \
+		--cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \
+		$(OIE_TAG_ARGS)
+
+# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream).
+# Examples:
+#   make setup-oie
+#   make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate'
+setup-oie:
+	bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS)
+
 oie-e2e-smoke:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py
 
+owasp-agent-sync:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_sync.py $(OWASP_AGENT_SYNC_ARGS)
+
+owasp-agent-merge-concepts:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_merge_concepts.py $(OWASP_AGENT_MERGE_ARGS)
+
+# Score fixtures/owasp_agent_synth_v2.jsonl against OWASP_AGENT_DB (see application/utils/owasp_agent/README.md).
+owasp-agent-synth-eval:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	OWASP_AGENT_ENABLED=1 PYTHONPATH=. python scripts/owasp_agent_synth_eval.py
+
 openapi-generate:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	python scripts/generate_openapi.py
@@ -240,6 +290,21 @@ monitor-chatbot-health-prod:
 		--base-url "https://opencre.org" \
 		--output-json "$(CURDIR)/tmp/prod-chatbot-health.json"
 
+# Scheduled OIE jobs (Postgres only). Cron `make oie-tick` every 10 minutes: each
+# job runs once per slot (owasp every 10 min, cre_expansion daily) and is a no-op
+# otherwise. Overrides: OIE_JOB=owasp|cre_expansion|all, OIE_FORCE=1, OIE_DRY_RUN=1.
+oie-tick:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --run_scheduled "$${OIE_JOB:-all}" \
+		$${OIE_FORCE:+--scheduled_force} $${OIE_DRY_RUN:+--scheduled_dry_run}
+
+monitor-oie-health:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export PYTHONPATH="$(CURDIR)"; \
+	python scripts/monitor_oie_health.py --output-json "$(CURDIR)/tmp/oie-health.json"
+
 verify-ga-parity-local:
 	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
 	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \

application/cmd/cre_main.py

@@ -880,6 +880,39 @@ def backfill_gap_analysis_only(
     logger.info("GA backfill complete: final_missing_pairs=%s", final_missing)
 
 
+def run_scheduled_cli(args: argparse.Namespace, collection: db.Node_collection) -> int:
+    """``--run_scheduled``: run the selected OIE job(s); return a process exit code.
+
+    Postgres only -- the advisory-lock lease and the run table assume it, and the
+    default ``standards_cache.sqlite`` must not silently become the scheduler's
+    database. Exit 1 when a job errored, 2 for a non-Postgres database.
+    """
+    import json
+
+    from application.utils.oie_scheduler.runner import run_due_jobs
+
+    dialect = collection.session.get_bind().dialect.name
+    if dialect != "postgresql":
+        logger.error(
+            "--run_scheduled is Postgres-only (got %s); point --cache_file at "
+            "a postgresql:// URL (make docker-postgres for a local one)",
+            dialect,
+        )
+        return 2
+    selected = args.run_scheduled
+    force = bool(getattr(args, "scheduled_force", False))
+    outcomes = run_due_jobs(
+        collection,
+        cache_file=args.cache_file,
+        jobs=None if selected == "all" else [selected],
+        force=force,
+        dry_run=bool(getattr(args, "scheduled_dry_run", False)),
+        trigger="manual" if force else "scheduled",
+    )
+    print(json.dumps([o.to_dict() for o in outcomes], indent=2))
+    return 1 if any(o.failed for o in outcomes) else 0
+
+
 def run(args: argparse.Namespace) -> None:  # pragma: no cover
     script_path = os.path.dirname(os.path.realpath(__file__))
     os.path.join(script_path, "../cres")
@@ -892,6 +925,14 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         logger.info("Exported %s rows to %s", rows, csv_out)
         return
 
+    if getattr(args, "run_scheduled", ""):
+        import sys
+
+        code = run_scheduled_cli(args, db_connect(args.cache_file))
+        if code:
+            sys.exit(code)
+        return
+
     if getattr(args, "run_harvester", False):
         import sys
 
@@ -905,6 +946,12 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
             args.run_id.strip(),
             repos_yaml=repos_yaml,
             dry_run=getattr(args, "harvester_dry_run", False),
+            kinds=[
+                k.strip()
+                for k in (getattr(args, "harvester_kinds", "") or "").split(",")
+                if k.strip()
+            ]
+            or None,
         )
         print(summary.to_json())
         if summary.status == "degraded":

application/config.py

@@ -4,9 +4,30 @@
 
 import os
 
+from application.utils.postgres_url import (
+    is_postgres_url,
+    main_db_env_keys,
+    sqlalchemy_postgres_url,
+)
+
 basedir = os.path.abspath(os.path.dirname(__file__))
 
 
+def _sqlalchemy_uri_from_env(raw: str) -> str:
+    """Bind Postgres env URLs to psycopg2; leave sqlite and other schemes alone."""
+    if is_postgres_url(raw):
+        return sqlalchemy_postgres_url(raw)
+    return raw
+
+
+def _first_env_url(*keys: str) -> str:
+    for key in keys:
+        raw = (os.environ.get(key) or "").strip()
+        if raw:
+            return raw
+    return ""
+
+
 class Config:
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SQLALCHEMY_RECORD_QUERIES = False
@@ -18,8 +39,8 @@ class DevelopmentConfig(Config):
     DEBUG = True
     ENVIRONMENT = "DEVELOPMENT"
     CACHE_TYPE = "SimpleCache"
-    SQLALCHEMY_DATABASE_URI = (
-        os.environ.get("DEV_DATABASE_URL")
+    SQLALCHEMY_DATABASE_URI = _sqlalchemy_uri_from_env(
+        _first_env_url(*main_db_env_keys("development"))
         or f'sqlite:///{os.path.join(basedir, "../standards_cache.sqlite")}'
     )
 
@@ -35,8 +56,8 @@ class ProductionConfig(Config):
     ENVIRONMENT = "PRODUCTION"
     CACHE_TYPE = "SimpleCache"
     CACHE_DEFAULT_TIMEOUT = 3000
-    SQLALCHEMY_DATABASE_URI = (
-        os.environ.get("PROD_DATABASE_URL")
+    SQLALCHEMY_DATABASE_URI = _sqlalchemy_uri_from_env(
+        _first_env_url(*main_db_env_keys("production"))
         or f'sqlite:///{os.path.join(basedir, "../standards_cache.sqlite")}'
     )
 
@@ -46,11 +67,12 @@ class CMDConfig(Config):
 
     def __init__(self, db_uri: str):
         if "://" in db_uri:
-            # Heroku and some tools still emit ``postgres://``; SQLAlchemy 2 expects
-            # ``postgresql://`` for the psycopg dialect name.
-            if db_uri.startswith("postgres://"):
-                db_uri = "postgresql://" + db_uri[len("postgres://") :]
-            self.SQLALCHEMY_DATABASE_URI = db_uri
+            # Heroku still emits ``postgres://``; SQLAlchemy 2.1 maps bare
+            # ``postgresql://`` to psycopg v3 — pin psycopg2 (requirements.txt).
+            if is_postgres_url(db_uri):
+                self.SQLALCHEMY_DATABASE_URI = sqlalchemy_postgres_url(db_uri)
+            else:
+                self.SQLALCHEMY_DATABASE_URI = db_uri
         else:
             # Flask-SQLAlchemy 3+ resolves non-absolute sqlite URLs against
             # app.instance_path, not the shell cwd. CLI --cache_file should