Add automatic knowledge discovery, readiness checkpoints and integrated answering

ts/packages/agents/browser/README.md

@@ -36,14 +36,21 @@ resurfaces decisions, and Questions can restore dismissed decisions. Source
 mentions are not independent votes.
 
 Assisted authoring requires explicit approval to generate a proposal and an
-explicit maintenance scope; generated slots still require approval. Discovery is
-opt-in, bounded and explicitly invoked. Neither operation authorizes tools.
+explicit maintenance scope; per-view generated slots still require approval.
+Per-view discovery is opt-in, bounded and explicitly invoked. Corpus-level
+discovery can separately admit informational contracts under the
+`sourceObserved` policy. Neither operation authorizes tools.
 `memoryGetViewPage` returns deterministic Markdown and a `memory-view:` URI;
 Markdown links preserve exact revisions, slots and collection item keys when
 opened in the browser. Optional service capabilities and operation errors are
 reported explicitly, not interpreted as empty inventories. Question lists use
 bounded summary pages; selecting a question separately loads its full evidence.
-There is no corpus-wide discovery policy or knowledge-status control.
+Questions also mounts knowledge preparation status, discovery policy controls,
+bounded failed-discovery retries and captured readiness checkpoints. Existing
+corpora remain off until enabled; new corpora default to source observed.
+Policy admission is not human confirmation, and source indexing alone never
+implies knowledge readiness. Integrated search answers retain original-source
+citations and separately render maintained-View and user-assertion evidence.
 
 Questions also shows acquisition progress and explicit run-once/cancel controls
 for existing host-authorized requests. Its RPC projection excludes staged

ts/packages/agents/browser/package.json

@@ -41,7 +41,7 @@
     "test:local": "jest && npm run test:crossword && npm run test:views:gateway && npm run test:views:client",
     "test:pdf": "node --test test/pdf*.spec.mjs test/pdfExtraction/*.spec.mjs test/memoryImportHandlers.spec.mjs test/agentConnectionBoundary.spec.mjs",
     "test:views:client": "jest --runInBand --testMatch \"**/views/client/**/*.test.ts\"",
-    "test:views:gateway": "node --test test/viewGateway.test.mjs test/viewDraftGateway.test.mjs test/viewLifecycle.test.mjs test/memoryHub.test.mjs test/memoryHubQuery.test.mjs test/memoryHubSourceDependencies.test.mjs test/memoryHubCapture.test.mjs test/browserCaptureRpc.test.mjs test/memoryHubRunbookImports.test.mjs test/memoryHubWebAnalytics.test.mjs test/schemaQuestionRpc.test.mjs",
+    "test:views:gateway": "node --test test/viewGateway.test.mjs test/viewDraftGateway.test.mjs test/viewLifecycle.test.mjs test/memoryHub.test.mjs test/memoryHubQuery.test.mjs test/memoryHubSourceDependencies.test.mjs test/memoryHubCapture.test.mjs test/browserCaptureRpc.test.mjs test/memoryHubRunbookImports.test.mjs test/memoryHubWebAnalytics.test.mjs test/schemaQuestionRpc.test.mjs test/knowledgeRpc.test.mjs",
     "tsc:agent": "tsc -p src/agent/tsconfig.json",
     "tsc:puppeteer": "tsc -p src/puppeteer/tsconfig.json",
     "tsc:views:client:typecheck": "tsc -p src/views/client/pdf/tsconfig.json && tsc -p src/views/client/library/tsconfig.json",

ts/packages/agents/browser/src/agent/memoryHubIntegratedAnswer.mts

@@ -0,0 +1,113 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { conversationCorpusName } from "@typeagent/memory-service";
+import type { MemoryCenterCorpus } from "@typeagent/browser-control-rpc/serviceTypes";
+import type {
+    MemoryHubSearchRequest,
+    MemoryHubSearchResult,
+} from "@typeagent/browser-control-rpc/viewRpc";
+import type { MemoryHubReadService } from "./memoryHubQuery.mjs";
+
+export async function tryMemoryHubIntegratedAnswer(
+    service: MemoryHubReadService,
+    corpora: MemoryCenterCorpus[],
+    request: MemoryHubSearchRequest,
+    result: MemoryHubSearchResult,
+): Promise<boolean> {
+    if (!request.generateAnswer) return false;
+    const capability = (await service.getCapabilities()).derivedViews
+        ?.integratedAnswering;
+    const eligible =
+        request.corpusId !== undefined &&
+        corpora.length === 1 &&
+        corpora[0].name !== conversationCorpusName &&
+        !request.sourceTypes?.length &&
+        !request.tags?.length &&
+        !request.dateFrom &&
+        !request.dateTo &&
+        !request.viewKinds?.length &&
+        (!request.conversationScope || request.conversationScope === "none");
+    if (!capability || !eligible) {
+        if (
+            request.retrievalPolicy === "sourcesAndViews" ||
+            request.knowledgeCheckpointId
+        )
+            throw new Error(
+                "Strict sources-and-views answering requires a supporting host and one selected document collection without filters or conversations.",
+            );
+        if (capability)
+            result.warnings.push(
+                "Integrated knowledge answering requires one selected document collection without filters or conversations. This scope uses the bounded legacy evidence reader instead; derived knowledge coverage is not qualified.",
+            );
+        return false;
+    }
+    const corpus = corpora[0];
+    const answer = await service.answer({
+        corpusId: corpus.corpusId,
+        question: result.query,
+        ...(request.retrievalPolicy === undefined
+            ? {}
+            : { retrievalPolicy: request.retrievalPolicy }),
+        ...(request.knowledgeCheckpointId === undefined
+            ? {}
+            : { knowledgeCheckpointId: request.knowledgeCheckpointId }),
+        limit: 12,
+        maxResponseChars: 24000,
+    });
+    result.matches = answer.citations.map((source, index) => ({
+        id: source.evidenceId,
+        kind: "source",
+        corpusId: corpus.corpusId,
+        corpusName: corpus.name,
+        objectId: source.sourceId,
+        title: source.title,
+        snippet: source.snippet,
+        score: source.score,
+        rank: index + 1,
+        sourceId: source.sourceId,
+        revisionId: source.revisionId,
+        sourceType: source.sourceType,
+        ...(source.locator === undefined ? {} : { locator: source.locator }),
+        ...(source.canonicalUri === undefined
+            ? {}
+            : { canonicalUri: source.canonicalUri }),
+        ...(source.synopsis === undefined ? {} : { synopsis: source.synopsis }),
+        ...(source.capturedAt === undefined
+            ? {}
+            : { eventTime: source.capturedAt }),
+    }));
+    result.warnings.push(...answer.warnings);
+    result.warnings.push(
+        request.retrievalPolicy === "sourcesOnly"
+            ? "Sources-only diagnostic answering excludes maintained views; original evidence selection is bounded, not exhaustive."
+            : "Knowledge answering uses bounded original evidence and eligible maintained views. Ready knowledge is relative to captured coverage, not exhaustive memory.",
+    );
+    result.answer = {
+        text: answer.answer,
+        mode: answer.mode,
+        citationIds: result.matches.map((source) => source.id),
+        followUps: [],
+        ...(answer.derivedReferences === undefined
+            ? {}
+            : {
+                  derivedReferences: answer.derivedReferences.map(
+                      (reference) => ({
+                          corpusId: corpus.corpusId,
+                          ...reference,
+                      }),
+                  ),
+              }),
+        ...(answer.assertionReferences === undefined
+            ? {}
+            : {
+                  assertionReferences: answer.assertionReferences.map(
+                      (reference) => ({
+                          corpusId: corpus.corpusId,
+                          ...reference,
+                      }),
+                  ),
+              }),
+    };
+    return true;
+}

ts/packages/agents/browser/src/agent/memoryHubQuery.mts

@@ -33,6 +33,7 @@ import {
 import { loadMemoryHubCorpora, mapMemoryHubCorpora } from "./memoryHub.mjs";
 import { timed } from "./memoryHubQuerySupport.mjs";
 import { memoryHubSearchInsights } from "./memoryHubSearchInsights.mjs";
+import { tryMemoryHubIntegratedAnswer } from "./memoryHubIntegratedAnswer.mjs";
 export { timed } from "./memoryHubQuerySupport.mjs";
 
 export type MemoryHubReadService = MemoryService & Partial<MemoryViewService>;
@@ -224,6 +225,7 @@ async function viewResults(
     corpus: MemoryCenterCorpus,
     request: MemoryHubSearchRequest,
 ): Promise<MemoryHubEvidence[]> {
+    if (request.retrievalPolicy === "sourcesOnly") return [];
     const capabilities = await timed(service.getCapabilities());
     if (!capabilities.derivedViews?.search) return [];
     if (!service.searchViews)
@@ -684,6 +686,36 @@ export function createMemoryHubQueryFunctions(
                                 warnings: [],
                                 errors: [],
                             };
+                            try {
+                                if (
+                                    await tryMemoryHubIntegratedAnswer(
+                                        service,
+                                        corpora,
+                                        request,
+                                        result,
+                                    )
+                                ) {
+                                    result.insights = {
+                                        provider: "canonical",
+                                        status: "unsupported",
+                                        ...(request.corpusId === undefined
+                                            ? {}
+                                            : { corpusId: request.corpusId }),
+                                        topTopics: [],
+                                        relatedEntities: [],
+                                        message:
+                                            "Integrated answering returns exact evidence and maintained-knowledge references; separate query insights are not generated for this answer path.",
+                                    };
+                                    return result;
+                                }
+                            } catch (error) {
+                                result.errors.push({
+                                    corpusId: request.corpusId ?? "*",
+                                    operation: "answer",
+                                    message: message(error),
+                                });
+                                return result;
+                            }
                             const lists = await timeSearchStage(
                                 "hub.retrieval",
                                 () =>

ts/packages/agents/browser/src/agent/memoryViews.mts

@@ -8,6 +8,13 @@ import type {
 } from "@typeagent/memory-service";
 import { createMemoryServiceRpcFacade } from "@typeagent/memory-service/rpc";
 import type { MemoryAcquisitionProgressService } from "@typeagent/memory-client";
+import {
+    knowledgeCorpusRequestSchema,
+    knowledgeDiscoveryUpdateSchema,
+    knowledgeCheckpointRequestSchema,
+    knowledgeCheckpointReadSchema,
+    knowledgeCheckpointWaitSchema,
+} from "@typeagent/memory-client";
 import {
     acquisitionProgress,
     acquisitionOperation,
@@ -22,6 +29,11 @@ type ViewMethods = Pick<
     | "memoryListQuestions"
     | "memoryListQuestionSummaries"
     | "memoryGetQuestion"
+    | "memoryGetKnowledgeStatus"
+    | "memoryUpdateKnowledgeDiscovery"
+    | "memoryCreateKnowledgeCheckpoint"
+    | "memoryGetKnowledgeCheckpoint"
+    | "memoryWaitForKnowledgeCheckpoint"
     | "memoryListAcquisitions"
     | "memoryRunAcquisitions"
     | "memoryCancelAcquisition"
@@ -71,6 +83,48 @@ export function createMemoryViewFunctions(
         return method.bind(value) as NonNullable<MemoryViewService[K]>;
     }
     return {
+        memoryGetKnowledgeStatus: (request) =>
+            views().getKnowledgeStatus(
+                knowledgeCorpusRequestSchema.parse(request).corpusId,
+            ),
+        memoryUpdateKnowledgeDiscovery: (request) => {
+            const { corpusId, policy, retryFailed } =
+                knowledgeDiscoveryUpdateSchema.parse(request);
+            return views().updateKnowledgeDiscovery({
+                corpusId,
+                policy,
+                ...(retryFailed === undefined ? {} : { retryFailed }),
+            });
+        },
+        memoryCreateKnowledgeCheckpoint: (request) => {
+            const {
+                corpusId,
+                sourceIds,
+                families,
+                producerComplete,
+                timeoutMs,
+            } = knowledgeCheckpointRequestSchema.parse(request);
+            return views().createKnowledgeCheckpoint({
+                corpusId,
+                ...(sourceIds === undefined ? {} : { sourceIds }),
+                ...(families === undefined ? {} : { families }),
+                ...(producerComplete === undefined ? {} : { producerComplete }),
+                ...(timeoutMs === undefined ? {} : { timeoutMs }),
+            });
+        },
+        memoryGetKnowledgeCheckpoint: (request) =>
+            views().getKnowledgeCheckpoint(
+                knowledgeCheckpointReadSchema.parse(request),
+            ),
+        memoryWaitForKnowledgeCheckpoint: (request) => {
+            const { corpusId, checkpointId, timeoutMs } =
+                knowledgeCheckpointWaitSchema.parse(request);
+            return views().waitForKnowledgeCheckpoint({
+                corpusId,
+                checkpointId,
+                ...(timeoutMs === undefined ? {} : { timeoutMs }),
+            });
+        },
         memoryViewCapabilities: () => service().getCapabilities(),
         memoryQueryTopics: (request) => views().queryTopics(request),
         memoryRebuildTopics: ({ corpusId }) => views().rebuildTopics(corpusId),