fix: make MCP credential failures visible and diagnosable

.claude/hooks/session-start.sh

@@ -35,10 +35,16 @@ if [ -n "${OP_SERVICE_ACCOUNT_TOKEN:-}" ]; then
     OP_VAULT="${CLAUDE_OP_VAULT:-Claude}"
     OP_ITEM="${CLAUDE_OP_ITEM:-cloud-session-env}"
 
-    # Seed the Nullify CLI credentials file so its built-in refresh flow
-    # mints fresh access tokens (an env NULLIFY_TOKEN would override stored
-    # credentials in the CLI, so that var is deliberately NOT resolved here).
-    if [ ! -s "$HOME/.nullify/credentials.json" ]; then
+    # Nullify accepts either a long-lived NULLIFY_TOKEN or a seeded
+    # credentials file. Prefer the token: it needs no refresh cycle and no
+    # browser, so it survives an ephemeral container, whereas a seeded refresh
+    # token ages out and its renewals die with the container. Seed the file
+    # only when no token is configured, since the CLI would ignore it anyway.
+    nullify_token="$(op read "op://${OP_VAULT}/${OP_ITEM}/NULLIFY_TOKEN" 2>/dev/null || true)"
+    if [ -n "$nullify_token" ]; then
+      printf 'export %s=%q\n' "NULLIFY_TOKEN" "$nullify_token" >> "$CLAUDE_ENV_FILE"
+      unset nullify_token
+    elif [ ! -s "$HOME/.nullify/credentials.json" ]; then
       nullify_creds="$(op read "op://${OP_VAULT}/${OP_ITEM}/NULLIFY_CREDENTIALS_JSON" 2>/dev/null || true)"
       if [ -n "$nullify_creds" ]; then
         mkdir -p "$HOME/.nullify"
@@ -96,4 +102,21 @@ if ! golangci-lint version 2>/dev/null | grep -q "version ${GOLANGCI_LINT_VERSIO
     go install "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}"
 fi
 
+# ============================================================
+# 3. MCP credential preflight — report-only
+# ============================================================
+
+# Expired credentials make a server expose no tools, or list every tool and
+# fail each call. Neither is visible without looking, so surface it here
+# instead of leaving it to be noticed when a report comes back empty.
+# Never fails the hook: the session is still usable with a degraded server.
+echo "--- MCP preflight ---"
+if ! .claude/mcp/preflight.sh; then
+  echo
+  echo "!!  One or more MCP servers are UNUSABLE - see the table above."
+  echo "!!  Their tools may still be listed and will return empty results."
+  echo "!!  Do not read an empty result from those servers as 'nothing found'."
+  echo "!!  Machine-readable status: .claude/mcp/health.json"
+fi
+
 echo "Session start hook completed: secrets resolved, MCP servers installed, Go toolchain ready."

.claude/mcp/health-probes.json

@@ -0,0 +1,51 @@
+{
+  "_comment": [
+    "Authorization probes for the MCP servers declared in .mcp.json.",
+    "",
+    "A handshake only proves a server started. Several of these servers will",
+    "start, advertise their whole tool set, and then answer every call with an",
+    "empty result or a 403 - which reads as 'healthy but quiet' rather than as",
+    "an outage. Each server therefore needs a probe that proves the credential",
+    "is actually accepted.",
+    "",
+    "preflight.sh reads the server list from .mcp.json, not from this file. A",
+    "server present there but missing here is reported UNCHECKED rather than",
+    "skipped, so adding a server cannot quietly add a blind spot.",
+    "",
+    "kinds:",
+    "  http        GET url_env + path with the key in an Authorization header;",
+    "              ok_status means authorized. Pick an endpoint that returns",
+    "              data, and verify it answers differently without the key.",
+    "  jwt-env     Inspect a JWT in var: reject expired, and reject any",
+    "              token_use in reject_token_use. Opaque tokens pass through.",
+    "  env-present Assert vars are non-empty. Weakest kind - only for servers",
+    "              with no cheap authenticated endpoint."
+  ],
+
+  "probes": {
+    "nullify": {
+      "kind": "jwt-env",
+      "var": "NULLIFY_TOKEN",
+      "reject_token_use": ["id"],
+      "fallback_file": "~/.nullify/credentials.json",
+      "hint": "set NULLIFY_TOKEN to an API token that does not expire, or seed NULLIFY_CREDENTIALS_JSON"
+    },
+
+    "latent-defense": {
+      "kind": "http",
+      "url_env": "LATENT_DEFENSE_URL",
+      "path": "/api/infra/records",
+      "key_env": "LATENT_DEFENSE_API_KEY",
+      "ok_status": "200",
+      "hint": "rotate LATENT_DEFENSE_API_KEY in the secret store",
+      "_note": "Deliberately NOT /auth/me: that endpoint serves device-flow sessions and answers 401 to a valid API key, which once produced a six-day false outage."
+    },
+
+    "sentinelone": {
+      "kind": "env-present",
+      "vars": ["PURPLEMCP_CONSOLE_TOKEN", "PURPLEMCP_CONSOLE_BASE_URL"],
+      "hint": "set PURPLEMCP_CONSOLE_TOKEN and PURPLEMCP_CONSOLE_BASE_URL",
+      "_note": "The console exposes no unauthenticated health endpoint, so presence is the strongest cheap check available."
+    }
+  }
+}

.claude/mcp/nullify-serve.sh

@@ -1,22 +1,57 @@
 #!/bin/bash
-# Start the Nullify MCP server with auto-refreshing credentials.
+# Start the Nullify MCP server, accepting either credential model.
 #
-# NULLIFY_CREDENTIALS_JSON (resolved from 1Password by with-op-env.sh) holds
-# the contents of ~/.nullify/credentials.json from a machine where
-# 'nullify auth login' was run. Seeding that file lets the CLI's built-in
-# refresh flow mint fresh access tokens for the refresh token's lifetime
-# (~30 days), instead of relying on a static hourly token.
+# NULLIFY_TOKEN (preferred for cloud sessions)
+#   A long-lived API token. The CLI uses it directly, so there is no refresh
+#   cycle to go stale and nothing to mint through a browser. This is the right
+#   model for an ephemeral container: the token in the secret store is the only
+#   state, and it stays valid until it is deliberately rotated.
 #
-# NOTE: an env NULLIFY_TOKEN overrides stored credentials in the CLI - do not
-# set it anywhere (environment settings or 1Password) when using this flow.
+# NULLIFY_CREDENTIALS_JSON (fallback)
+#   The contents of ~/.nullify/credentials.json from a machine where
+#   'nullify auth login' was run. Seeding it lets the CLI's refresh flow mint
+#   access tokens for the refresh token's lifetime (~30 days). The catch is
+#   that refreshes are written to the container's filesystem and lost when it
+#   exits, so the stored copy ages out and eventually stops working - at which
+#   point a human has to re-run an interactive browser login.
+#
+# NULLIFY_TOKEN wins when both are present: the CLI would prefer it anyway
+# (it outranks stored credentials), so seeding a file we would not read only
+# creates a second thing to keep in sync.
 set -uo pipefail
 
-if [ -n "${NULLIFY_CREDENTIALS_JSON:-}" ] && [ ! -s "$HOME/.nullify/credentials.json" ]; then
+if [ -n "${NULLIFY_TOKEN:-}" ]; then
+  echo "nullify-serve: authenticating with NULLIFY_TOKEN" >&2
+else
+  if [ -n "${NULLIFY_CREDENTIALS_JSON:-}" ] && [ ! -s "$HOME/.nullify/credentials.json" ]; then
+    mkdir -p "$HOME/.nullify"
+    chmod 700 "$HOME/.nullify"
+    printf '%s' "$NULLIFY_CREDENTIALS_JSON" > "$HOME/.nullify/credentials.json"
+    chmod 600 "$HOME/.nullify/credentials.json"
+  fi
+
+  if [ -s "$HOME/.nullify/credentials.json" ]; then
+    echo "nullify-serve: authenticating with seeded credentials file" >&2
+  else
+    {
+      echo "nullify-serve: WARNING no Nullify credentials available."
+      echo "nullify-serve:   set NULLIFY_TOKEN (preferred) or NULLIFY_CREDENTIALS_JSON"
+      echo "nullify-serve:   in the 1Password item named by CLAUDE_OP_VAULT/CLAUDE_OP_ITEM."
+      echo "nullify-serve:   the server will start but expose no tools."
+    } >&2
+  fi
+fi
+unset NULLIFY_CREDENTIALS_JSON
+
+# Seed config.json too. 'mcp serve' reads NULLIFY_HOST directly, but leaving
+# config.json absent means anyone debugging inside the container gets
+# "Not configured" from 'nullify auth status' even though the credentials are
+# right there - which is exactly the wrong signal when auth is what's broken.
+if [ -n "${NULLIFY_HOST:-}" ] && [ ! -s "$HOME/.nullify/config.json" ]; then
   mkdir -p "$HOME/.nullify"
   chmod 700 "$HOME/.nullify"
-  printf '%s' "$NULLIFY_CREDENTIALS_JSON" > "$HOME/.nullify/credentials.json"
-  chmod 600 "$HOME/.nullify/credentials.json"
+  printf '{\n  "host": "%s"\n}\n' "$NULLIFY_HOST" > "$HOME/.nullify/config.json"
+  chmod 600 "$HOME/.nullify/config.json"
 fi
-unset NULLIFY_CREDENTIALS_JSON
 
 exec go run ./cmd/cli mcp serve

.claude/mcp/preflight.sh

@@ -0,0 +1,222 @@
+#!/bin/bash
+# Verify every MCP server in .mcp.json is not just running but usable, and
+# leave a machine-readable record of the answer.
+#
+#   .claude/mcp/preflight.sh          human table + health.json
+#   make preflight                    same
+#
+# Why this exists: an MCP server almost never fails loudly. It either dies
+# during startup and shows up in the client as a server with no tools, or it
+# starts, advertises its whole tool set, and answers every call with an empty
+# result because its credential is rejected. Both look like "nothing to report"
+# rather than "outage", and the second is worse - work gets published against
+# data that was never there.
+#
+# So each server gets two checks:
+#   1. handshake     does it start and list tools?
+#   2. authorization is its credential actually accepted?
+#
+# The server list comes from .mcp.json, never from a hand-kept list here. A
+# server with no probe in health-probes.json is reported UNCHECKED, so adding
+# one cannot quietly add a blind spot.
+#
+# Exit status: 0 only when every server passes, so this can gate a check.
+set -uo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.." || exit 1
+
+MCP_CONFIG="${PREFLIGHT_MCP_CONFIG:-.mcp.json}"
+PROBE_CONFIG="${PREFLIGHT_PROBE_CONFIG:-.claude/mcp/health-probes.json}"
+HEALTH_OUT="${PREFLIGHT_OUT:-.claude/mcp/health.json}"
+TIMEOUT="${PREFLIGHT_TIMEOUT:-120}"
+
+STDERR_FILE="$(mktemp)"
+trap 'rm -f "$STDERR_FILE"' EXIT
+
+for dep in jq curl; do
+  command -v "$dep" >/dev/null 2>&1 || { echo "preflight: $dep is required" >&2; exit 2; }
+done
+[ -f "$MCP_CONFIG" ] || { echo "preflight: $MCP_CONFIG not found" >&2; exit 2; }
+
+failures=0
+declare -a ROWS=()      # status|server|detail
+declare -a JSON_ROWS=()
+
+record() {
+  ROWS+=("$1|$2|$3")
+  JSON_ROWS+=("$(jq -nc --arg s "$1" --arg n "$2" --arg d "$3" --argjson t "${4:-null}" \
+    '{server:$n, status:$s, detail:$d, tools:$t}')")
+  [ "$1" = "FAIL" ] && failures=$((failures + 1))
+  return 0
+}
+
+# ---------------------------------------------------------------- handshake
+
+# Expand the ${VAR:-default} form that MCP client env blocks use.
+expand_env_value() {
+  local raw="$1"
+  if [[ "$raw" =~ ^\$\{([A-Za-z_][A-Za-z0-9_]*):-(.*)\}$ ]]; then
+    local name="${BASH_REMATCH[1]}" fallback="${BASH_REMATCH[2]}"
+    printf '%s' "${!name:-$fallback}"
+  elif [[ "$raw" =~ ^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$ ]]; then
+    printf '%s' "${!BASH_REMATCH[1]:-}"
+  else
+    printf '%s' "$raw"
+  fi
+}
+
+# Start the server exactly as the client would - through the same wrapper and
+# arguments - so the wiring itself is under test, not just the binary.
+handshake_tool_count() {
+  local server="$1"
+  local -a cmd=() envs=()
+  local key val
+
+  mapfile -t cmd < <(jq -r --arg s "$server" \
+    '.mcpServers[$s] | ([.command] + (.args // []))[]' "$MCP_CONFIG" 2>/dev/null)
+  [ "${#cmd[@]}" -gt 0 ] || { echo ""; return; }
+
+  while IFS=$'\t' read -r key val; do
+    [ -n "$key" ] && envs+=("$key=$(expand_env_value "$val")")
+  done < <(jq -r --arg s "$server" \
+    '(.mcpServers[$s].env // {}) | to_entries[] | "\(.key)\t\(.value)"' "$MCP_CONFIG" 2>/dev/null)
+
+  {
+    echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"preflight","version":"1"}}}'
+    echo '{"jsonrpc":"2.0","method":"notifications/initialized"}'
+    echo '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}'
+    sleep 2
+  } | timeout "$TIMEOUT" env "${envs[@]}" "${cmd[@]}" 2>"$STDERR_FILE" \
+    | jq -R 'fromjson? // empty' 2>/dev/null \
+    | jq -s 'map(select(.id == 2)) | (.[0].result.tools // []) | length' 2>/dev/null
+}
+
+# Last stderr line that looks like a cause rather than a banner.
+stderr_reason() {
+  grep -v '^[[:space:]]*$' "$STDERR_FILE" 2>/dev/null \
+    | grep -viE 'warning:|IncompleteFieldDefinition|warnings\.warn|INFO |Processing request|Starting MCP server|^ +' \
+    | tail -1 | cut -c1-150
+}
+
+# ------------------------------------------------------------------- probes
+
+probe_field() {
+  jq -r --arg s "$1" --arg f "$2" '.probes[$s][$f] // empty' "$PROBE_CONFIG" 2>/dev/null
+}
+
+# echoes "<verdict>|<detail>"; verdict is ok, bad or unknown
+run_probe() {
+  local server="$1" kind
+  kind="$(probe_field "$server" kind)"
+
+  case "$kind" in
+    http)
+      local url_env key_env path ok base key code
+      url_env="$(probe_field "$server" url_env)"; key_env="$(probe_field "$server" key_env)"
+      path="$(probe_field "$server" path)";       ok="$(probe_field "$server" ok_status)"
+      base="${!url_env:-}"; key="${!key_env:-}"
+      [ -n "$base" ] || { echo "unknown|$url_env is unset"; return; }
+      [ -n "$key" ]  || { echo "bad|$key_env is unset"; return; }
+      code="$(curl -sS -o /dev/null -w '%{http_code}' -m 25 \
+        -H "Authorization: Bearer ${key}" -H "Accept: application/json" \
+        "${base%/}${path}" 2>/dev/null)"
+      if [ "$code" = "$ok" ]; then echo "ok|authorized"
+      elif [ -z "$code" ] || [ "$code" = "000" ]; then echo "unknown|${base} unreachable"
+      else echo "bad|$key_env rejected (HTTP $code)"; fi
+      ;;
+
+    jwt-env)
+      # A JWT states its own validity, so read it instead of waiting for the
+      # API to answer 403 on every call.
+      local var tok claims exp use reject file
+      var="$(probe_field "$server" var)"; tok="${!var:-}"
+      if [ -z "$tok" ]; then
+        file="$(probe_field "$server" fallback_file)"; file="${file/#\~/$HOME}"
+        if [ -n "$file" ] && [ -s "$file" ]; then echo "unknown|$var unset; using $(basename "$file")"
+        else echo "bad|$var unset and no fallback credentials"; fi
+        return
+      fi
+      claims="$(printf '%s' "$tok" | jq -R 'split(".") | if length == 3 then .[1] | @base64d | fromjson else empty end' 2>/dev/null)"
+      if [ -z "$claims" ]; then echo "ok|opaque token present"; return; fi
+      exp="$(printf '%s' "$claims" | jq -r '.exp // empty')"
+      use="$(printf '%s' "$claims" | jq -r '.token_use // empty')"
+      if [ -n "$exp" ] && [ "$exp" -lt "$(date +%s)" ] 2>/dev/null; then
+        echo "bad|$var expired $(date -u -d "@$exp" '+%Y-%m-%d %H:%M UTC' 2>/dev/null)"; return
+      fi
+      reject="$(jq -r --arg s "$server" '(.probes[$s].reject_token_use // [])[]' "$PROBE_CONFIG" 2>/dev/null | tr '\n' ' ')"
+      if [ -n "$use" ] && [[ " $reject " == *" $use "* ]]; then
+        echo "bad|$var is a ${use} token, which the API rejects"; return
+      fi
+      echo "ok|token valid"
+      ;;
+
+    env-present)
+      local missing=() v
+      while read -r v; do
+        [ -n "$v" ] && [ -z "${!v:-}" ] && missing+=("$v")
+      done < <(jq -r --arg s "$server" '(.probes[$s].vars // [])[]' "$PROBE_CONFIG" 2>/dev/null)
+      if [ "${#missing[@]}" -gt 0 ]; then echo "bad|unset: ${missing[*]}"
+      else echo "ok|credentials present"; fi
+      ;;
+
+    *) echo "unknown|no probe defined" ;;
+  esac
+}
+
+# -------------------------------------------------------------------- main
+
+echo "MCP preflight - $(jq -r '.mcpServers | keys | length' "$MCP_CONFIG") server(s) in $MCP_CONFIG"
+echo
+
+while read -r server; do
+  [ -n "$server" ] || continue
+
+  count="$(handshake_tool_count "$server")"
+  count="${count:-0}"
+  hint="$(probe_field "$server" hint)"
+
+  if [ "$count" -eq 0 ]; then
+    reason="$(stderr_reason)"
+    # A dead server usually explains itself on stderr; if the probe knows why,
+    # say that instead of the raw message.
+    verdict="$(run_probe "$server")"
+    if [ "${verdict%%|*}" = "bad" ]; then
+      record "FAIL" "$server" "no tools - ${verdict#*|}${hint:+; $hint}"
+    elif [ -n "$reason" ]; then
+      record "FAIL" "$server" "no tools - $reason"
+    else
+      record "FAIL" "$server" "no tools and no error reported"
+    fi
+    continue
+  fi
+
+  verdict="$(run_probe "$server")"
+  case "${verdict%%|*}" in
+    ok)      record "PASS" "$server" "${verdict#*|}" "$count" ;;
+    bad)     record "FAIL" "$server" "${count} tools but ${verdict#*|}${hint:+; $hint} - calls return empty results, not errors" "$count" ;;
+    unknown) record "WARN" "$server" "${count} tools; ${verdict#*|}" "$count" ;;
+  esac
+done < <(jq -r '.mcpServers | keys[]' "$MCP_CONFIG")
+
+printf '%-8s %-16s %s\n' "RESULT" "SERVER" "DETAIL"
+printf '%-8s %-16s %s\n' "------" "------" "------"
+for row in "${ROWS[@]}"; do
+  printf '%-8s %-16s %s\n' "${row%%|*}" "$(cut -d'|' -f2 <<<"$row")" "${row##*|}"
+done
+echo
+
+mkdir -p "$(dirname "$HEALTH_OUT")"
+printf '%s\n' "${JSON_ROWS[@]}" | jq -s \
+  --arg ts "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
+  --argjson failures "$failures" \
+  '{checked_at:$ts, healthy:($failures == 0), failures:$failures, servers:.}' \
+  > "$HEALTH_OUT" 2>/dev/null && echo "Wrote $HEALTH_OUT"
+
+if [ "$failures" -gt 0 ]; then
+  echo
+  echo "$failures server(s) unusable. Treat their data as ABSENT, not as empty:"
+  echo "a rejected credential returns no findings, which looks identical to good news."
+  exit 1
+fi
+
+echo "All MCP servers reachable and authorized."