diff --git a/CHANGELOG.md b/CHANGELOG.md index 94a5aff..4cc844b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) - **Embedding — message origins:** the JavaScript SDK accepted responses and events from any document loaded in the DataLab-Web iframe, and DataLab-Web sent its events to host windows whatever document they showed. The SDK now only accepts messages from the configured `targetOrigin`, and DataLab-Web sends its responses and events to the origin from which each host window made its requests. - **Image viewer — pixel positions:** images were drawn half a pixel off their coordinates: the first pixel started at the image origin instead of being centered on it, as in Sigima and DataLab desktop. Regions of interest, analysis results, the profile crosshair and the statistics rectangle therefore appeared shifted by half a pixel relative to the computed pixels, and the value shown when hovering the image could be that of the neighbouring pixel. Each pixel is now centered on its coordinates in the image viewer, the multi-image views and the processing preview, so overlays, read-outs, cross-sections and statistics match the computations exactly. The default regions of interest and areas to erase are centered on the image, and the preview of _Create ROI grid…_ now shows the same whole-pixel cells as the created regions of interest. Cross-sections and statistics also work on images with non-uniform coordinates. - **Image viewer — blurred pixels when zooming in:** since version 0.6.1, zooming in on an image showed blurred pixels, each one fading into its neighbours, so that close values could not be told apart. Zoomed-in pixels are now drawn as sharp squares (nearest pixel, as in DataLab desktop) in the image viewer, the _Spatial_ multi-image view and the processing preview, in every browser and in the images exported from the plot. Zoomed-out images are displayed as before. +- **AI Assistant — Save to Macros:** since version 0.10.0, clicking **Save to Macros** under a macro proposed by the AI Assistant showed "Saved ✓", but the macro did not appear in the Macros panel; each click added another hidden macro to the workspace. The saved macro now opens in the Macros panel, which is brought to the front, including from a conversation reopened from the history. +- **AI Assistant — macros written as text:** asked to write a macro, the assistant could answer with the code in its reply, without any **Save to Macros** button, in particular for curves its creation tools cannot produce (super-Gaussian, Lorentzian, polynomial…), which it was told to create without a macro. The assistant now proposes such macros through its macro tool, without running them unless asked (the approval dialog says so, and approving such a macro never auto-approves a macro that runs), and a **Save to Macros** button is also offered under each Python code block of its replies. +- **AI Assistant — macro output:** the assistant received an empty result when reading back the output of the last macro it ran. ### Changed diff --git a/src/App.tsx b/src/App.tsx index 1415448..57db29c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -565,6 +565,18 @@ export default function App() { setCentralView("macro"); } }, [macroFloating, setMacroFloating]); + // "Save to Macros" in the AI Assistant: the Macros panel saves the + // macro (whatever the runtime mode) and shows it. A floating Macros + // panel is already visible, so only the docked one gets focus. + const saveMacroFromAssistant = useCallback( + async (title: string, code: string) => { + const panel = macroPanelRef.current; + if (!panel) throw new Error("Macros panel is not available yet"); + await panel.importMacro(title, code); + if (!macroFloating) setCentralView("macro"); + }, + [macroFloating], + ); // Stable DOM containers for the Notebook / Macro panels. They are // moved between the central area and the FloatingDockStack via // ``appendChild`` rather than being unmounted / remounted, so the @@ -5296,6 +5308,7 @@ export default function App() { > setAIPanelCollapsed(true)} onClose={() => setAIPanelVisible(false)} /> diff --git a/src/aiassistant/macroCodeBlocks.ts b/src/aiassistant/macroCodeBlocks.ts new file mode 100644 index 0000000..3b0bf4a --- /dev/null +++ b/src/aiassistant/macroCodeBlocks.ts @@ -0,0 +1,50 @@ +/** + * Python code blocks of assistant replies that can be saved as macros. + * + * The model is asked to propose macros through ``create_and_run_macro`` + * (with ``autorun: false`` when the user only wants the code), but it + * may still paste the code in its reply. The AI Assistant panel offers + * a "Save to Macros" button under each such block; this module finds + * them and derives the title of the saved macro. + */ + +import { marked, type Token } from "marked"; +import { deriveTitle } from "./conversationStore"; + +/** Title used when no user request is available to name the macro. */ +export const DEFAULT_MACRO_TITLE = "AI Assistant macro"; + +const PYTHON_LANGS = new Set(["python", "py", "python3", "py3"]); + +/** Heuristic for fences without a language: a macro imports something + * or drives DataLab through ``proxy``. */ +const PYTHON_HINT = /^\s*(import\s+\w|from\s+[\w.]+\s+import\s)|\bproxy\./m; + +function isMacroCode(lang: string, code: string): boolean { + const normalized = lang.trim().split(/\s+/)[0]?.toLowerCase() ?? ""; + if (PYTHON_LANGS.has(normalized)) return true; + return normalized === "" && PYTHON_HINT.test(code); +} + +/** Return the source of every Python code block of *markdown*, in + * document order (blocks nested in lists or quotes included). */ +export function extractMacroCodeBlocks(markdown: string): string[] { + if (!markdown.trim()) return []; + const blocks: string[] = []; + const tokens = marked.lexer(markdown); + marked.walkTokens(tokens, (token: Token) => { + if (token.type !== "code") return; + const code = String(token.text ?? ""); + if (code.trim() && isMacroCode(String(token.lang ?? ""), code)) { + blocks.push(code); + } + }); + return blocks; +} + +/** Title of a macro saved from a reply: the user request it answers, + * shortened, or :data:`DEFAULT_MACRO_TITLE`. */ +export function deriveMacroTitle(userText: string | null | undefined): string { + if (!userText || !userText.trim()) return DEFAULT_MACRO_TITLE; + return deriveTitle(userText); +} diff --git a/src/aiassistant/systemPrompt.ts b/src/aiassistant/systemPrompt.ts index f68a23c..537d5ee 100644 --- a/src/aiassistant/systemPrompt.ts +++ b/src/aiassistant/systemPrompt.ts @@ -1,9 +1,9 @@ /** * System prompt for the DataLab-Web AI Assistant. * - * Adapted from ``DataLab/datalab/aiassistant/controller.py`` — - * stripped of macro/plugin guidance (those tools are not exposed in v1) - * and rephrased for the browser-native context. + * Adapted from ``DataLab/datalab/aiassistant/controller.py`` and + * rephrased for the browser-native context. Keep the guidance in line + * with what the tools of ``tools.ts`` actually support. */ export const SYSTEM_PROMPT = `You are an AI assistant integrated in DataLab-Web, the browser-native @@ -38,24 +38,28 @@ You help the user by: user can opt to save the macro to the Macros panel via a 'Save to Macros' button on the result, so always pick a short, descriptive 'name' (e.g. 'Generate super-Gaussian') that would - make a good saved title. Use 'get_macro_console_output' to - retrieve the buffered stdout/stderr if a previous result was - truncated. + make a good saved title. With 'autorun' set to false, the macro is + only checked and offered for saving, without being executed. Use + 'get_macro_console_output' to retrieve the buffered stdout/stderr + if a previous result was truncated. Guidelines: - 'create_synthetic_signal' / 'create_synthetic_image' are the ONLY - way to produce a new object from scratch. Do NOT try to use - 'apply_processing' for creation — it only transforms an existing - source object and will fail when nothing is selected. -- Reach for 'create_and_run_macro' only as a last resort, when no - combination of the other tools can express what the user wants - (custom NumPy code, multi-step orchestration, …). For straightforward - creation requests — including parametric curves like a Gaussian, a - super-Gaussian, a Lorentzian or a polynomial — the right pattern is - to call 'create_synthetic_signal' (or run the relevant 'apply_processing' - step on top of an existing signal); never write a macro just to plot - a curve. + way to produce a new object from scratch without a macro. Do NOT try + to use 'apply_processing' for creation — it only transforms an + existing source object and will fail when nothing is selected. +- Use 'create_synthetic_signal' only for the kinds it supports (sine, + cosine, gauss, noise) and 'create_synthetic_image' only for the + kinds it supports (gauss, ramp, random). For any other shape (e.g. a + super-Gaussian, a Lorentzian, a polynomial), write a macro with + 'create_and_run_macro'. Otherwise, prefer the dedicated tools over a + macro. +- When the user asks you to write, create or give a macro (or a + script), call 'create_and_run_macro' with 'autorun' set to false, so + that the user gets the 'Save to Macros' button; set 'autorun' to true + (or omit it) only if the user also asks to run it. NEVER paste the + code of a macro in your reply instead of calling the tool. - The macro environment is a real Pyodide bundle: NumPy, SciPy, Sigima and guidata are all importable (parity with DataLab desktop). matplotlib, plotly, pandas display helpers and any GUI library are diff --git a/src/aiassistant/tools.ts b/src/aiassistant/tools.ts index 141e430..3f5374f 100644 --- a/src/aiassistant/tools.ts +++ b/src/aiassistant/tools.ts @@ -436,8 +436,33 @@ const CAPTURE_VIEW_TOOL: Tool = { }, }; +/** Name of the macro tool — shared with the UI, which surfaces its + * ``_macro`` echo as a "Save to Macros" button. */ +export const MACRO_TOOL_NAME = "create_and_run_macro"; + +/** True when *args* request a macro that is offered for saving but not + * executed (``autorun: false``). */ +export function isNonExecutingMacroCall( + toolName: string, + args: Record, +): boolean { + return toolName === MACRO_TOOL_NAME && args.autorun === false; +} + +/** Key under which the "Auto-approve" choice of a tool call is + * remembered. A macro offered without being executed gets its own key, + * so approving it never auto-approves a macro that actually runs. */ +export function approvalKey( + toolName: string, + args: Record, +): string { + return isNonExecutingMacroCall(toolName, args) + ? `${toolName}:noexec` + : toolName; +} + const CREATE_AND_RUN_MACRO_TOOL: Tool = { - name: "create_and_run_macro", + name: MACRO_TOOL_NAME, description: "Run an arbitrary Python script (a 'macro') against the workspace. " + "The script executes in a sandboxed Pyodide worker and has access " + @@ -447,10 +472,14 @@ const CREATE_AND_RUN_MACRO_TOOL: Tool = { "'Save to Macros' button on the result \u2014 always pick a short, " + "descriptive title (passed via the 'name' field, e.g. 'Generate " + "super-Gaussian') so the saved macro is easy to recognise. " + - "Returns the captured stdout/stderr. Use this only when the " + - "existing tools cannot express the required operation \u2014 e.g. " + - "complex multi-step workflows, custom NumPy code, or features not " + - "yet exposed as a dedicated tool.", + "Returns the captured stdout/stderr. Set 'autorun' to false when " + + "the user asks you to write (not run) a macro: the code is then " + + "only checked and offered to the user through the 'Save to Macros' " + + "button, without being executed. Use this tool for workflows that " + + "the other tools cannot express \u2014 e.g. complex multi-step " + + "workflows, custom NumPy code, signal shapes not supported by " + + "'create_synthetic_signal', or features not yet exposed as a " + + "dedicated tool.", parameters: { type: "object", properties: { @@ -467,6 +496,13 @@ const CREATE_AND_RUN_MACRO_TOOL: Tool = { "to 'AI Assistant macro' but you should always provide a " + "specific name.", }, + autorun: { + type: "boolean", + description: + "Run the macro (default true). Set to false to only check " + + "the code and offer it to the user for saving to the Macros " + + "panel, without executing it.", + }, }, required: ["code"], additionalProperties: false, @@ -478,12 +514,26 @@ const CREATE_AND_RUN_MACRO_TOOL: Tool = { typeof args.name === "string" && args.name.trim() ? args.name.trim() : "AI Assistant macro"; - const out = await runtime.runMacroCode(code, name); // Echo the source under ``_macro`` so the AI Assistant transcript // can offer a "Save to Macros" button on user demand. The leading // underscore is a hint that the field is UI-only — the LLM should // ignore it (it's not part of the macro's actual stdout/stderr). - return { ...out, _macro: { name, code } }; + const macro = { name, code }; + if (args.autorun === false) { + const lint = await runtime.lintMacro(code); + return { + autorun: false, + note: + "Macro not executed. The user can save it to the Macros " + + "panel with the 'Save to Macros' button.", + syntax_error: lint.syntax_error, + unknown_methods: lint.unknown_methods, + missing_await: lint.missing_await, + _macro: macro, + }; + } + const out = await runtime.runMacroCode(code, name); + return { ...out, _macro: macro }; }, }; @@ -495,7 +545,7 @@ const GET_MACRO_CONSOLE_OUTPUT_TOOL: Tool = { "run yet). Useful when the previous tool result was truncated.", parameters: { type: "object", properties: {}, additionalProperties: false }, readonly: true, - handler: async (runtime) => runtime.lastMacroOutput, + handler: async (runtime) => runtime.getLastMacroOutput(), }; /** Default tool registry shipped in v1. */ diff --git a/src/components/AIAssistant/AIAssistantPanel.tsx b/src/components/AIAssistant/AIAssistantPanel.tsx index fd960af..79dec28 100644 --- a/src/components/AIAssistant/AIAssistantPanel.tsx +++ b/src/components/AIAssistant/AIAssistantPanel.tsx @@ -19,7 +19,15 @@ import { saveSettings, } from "../../aiassistant/settings"; import { SYSTEM_PROMPT } from "../../aiassistant/systemPrompt"; -import { BUILTIN_TOOLS } from "../../aiassistant/tools"; +import { + approvalKey, + BUILTIN_TOOLS, + MACRO_TOOL_NAME, +} from "../../aiassistant/tools"; +import { + deriveMacroTitle, + extractMacroCodeBlocks, +} from "../../aiassistant/macroCodeBlocks"; import type { ChatMessage, ProviderSettings, @@ -48,6 +56,10 @@ import { t } from "../../i18n/translate"; interface Props { runtime: RuntimeApi; + /** Save a macro proposed by the assistant to the Macros panel (and + * show it). Called by the "Save to Macros" buttons; a rejection is + * reported on the button so the user can retry. */ + onSaveMacro: (title: string, code: string) => Promise; /** Optional callback to collapse the panel into the floating pill. */ onMinimize?: () => void; /** Optional callback to fully hide the panel (same as the View menu @@ -126,7 +138,12 @@ function CloseIcon() { ); } -export function AIAssistantPanel({ runtime, onMinimize, onClose }: Props) { +export function AIAssistantPanel({ + runtime, + onSaveMacro, + onMinimize, + onClose, +}: Props) { const [settings, setSettings] = useState(() => loadSettings(), ); @@ -158,7 +175,8 @@ export function AIAssistantPanel({ runtime, onMinimize, onClose }: Props) { inputHistoryRef.current = new InputHistory(); } - // Per-conversation auto-approve memory keyed by tool name. + // Per-conversation auto-approve memory, keyed by :func:`approvalKey` + // (the tool name, refined for macros offered without being run). const autoApprovedRef = useRef>(new Set()); // ``settings`` getter for the controller — wrapped through a ref so a // mid-conversation settings update is picked up by the next request @@ -203,7 +221,8 @@ export function AIAssistantPanel({ runtime, onMinimize, onClose }: Props) { const confirmTool = useCallback( (tool: Tool, args: Record): Promise => new Promise((resolve) => { - if (autoApprovedRef.current.has(tool.name)) { + const key = approvalKey(tool.name, args); + if (autoApprovedRef.current.has(key)) { resolve(true); return; } @@ -212,7 +231,7 @@ export function AIAssistantPanel({ runtime, onMinimize, onClose }: Props) { args, resolve: ({ approve, remember }) => { if (approve && remember) { - autoApprovedRef.current.add(tool.name); + autoApprovedRef.current.add(key); } setPendingConfirm(null); resolve(approve); @@ -266,6 +285,17 @@ export function AIAssistantPanel({ runtime, onMinimize, onClose }: Props) { el.scrollTop = el.scrollHeight; }, [transcript, busy, pendingConfirm, streamingText]); + // For each transcript entry, the text of the user request it answers + // (latest user message at or before it) — names the macros saved from + // assistant replies. + const requestTexts = useMemo(() => { + let last: string | null = null; + return transcript.map(({ message }) => { + if (message.role === "user") last = messageText(message); + return last; + }); + }, [transcript]); + // On mount: try to restore the most-recent conversation. Runs at most // once even under React 18 strict-mode double invocation. const restoredRef = useRef(false); @@ -532,7 +562,12 @@ export function AIAssistantPanel({ runtime, onMinimize, onClose }: Props) { )} {transcript.map((entry, idx) => ( - + ))} {streamingText && (
(p.type === "text" ? p.text : "")).join(" "); +} + function TranscriptItem({ entry, - runtime, + requestText, + onSaveMacro, }: { entry: TranscriptEntry; - runtime: RuntimeApi; + requestText: string | null; + onSaveMacro: (title: string, code: string) => Promise; }) { const { message } = entry; if (message.role === "user") { @@ -793,7 +838,14 @@ function TranscriptItem({ }} > {message.content && message.content.trim() ? ( - + <> + + + ) : null} {message.tool_calls && message.tool_calls.length > 0 && (
); } return null; } -function ToolTranscriptBubble({ - name, - preview, - fullText, - macroToSave, - runtime, +/** "Save to Macros" buttons under the Python code blocks of an + * assistant reply (fallback when the model writes the macro in its + * reply instead of proposing it through ``create_and_run_macro``). */ +function CodeBlockSaveButtons({ + text, + requestText, + onSaveMacro, }: { - name: string; - preview: string; - fullText: string; - macroToSave: { name: string; code: string } | null; - runtime: RuntimeApi; + text: string; + requestText: string | null; + onSaveMacro: (title: string, code: string) => Promise; +}) { + const blocks = useMemo(() => extractMacroCodeBlocks(text), [text]); + if (blocks.length === 0) return null; + const title = deriveMacroTitle(requestText); + return ( +
+ {blocks.map((code, i) => ( + 1 + ? t("💾 Save code block {index} to Macros", { index: i + 1 }) + : undefined + } + /> + ))} +
+ ); +} + +/** Button saving one macro to the Macros panel, with its progress. */ +function SaveMacroButton({ + title, + code, + onSaveMacro, + label, +}: { + title: string; + code: string; + onSaveMacro: (title: string, code: string) => Promise; + /** Idle label; defaults to "💾 Save to Macros". */ + label?: string; }) { const [saveState, setSaveState] = useState< "idle" | "saving" | "saved" | "error" >("idle"); - const [savedTitle, setSavedTitle] = useState(null); const handleSave = useCallback(async () => { - if (!macroToSave || saveState === "saving" || saveState === "saved") return; + if (saveState === "saving" || saveState === "saved") return; setSaveState("saving"); try { - const rec = await runtime.saveMacro(macroToSave.name, macroToSave.code); - setSavedTitle(rec.title); + await onSaveMacro(title, code); setSaveState("saved"); } catch (err) { console.error("Failed to save macro:", err); setSaveState("error"); } - }, [macroToSave, runtime, saveState]); + }, [code, onSaveMacro, saveState, title]); + return ( +
+ + {saveState === "saved" && ( + + {t("as “{title}”", { title })} + + )} + {saveState === "error" && ( + + {t("save failed (see console)")} + + )} +
+ ); +} + +function ToolTranscriptBubble({ + name, + preview, + fullText, + macroToSave, + onSaveMacro, +}: { + name: string; + preview: string; + fullText: string; + macroToSave: { name: string; code: string } | null; + onSaveMacro: (title: string, code: string) => Promise; +}) { return (
{macroToSave && ( -
- - {saveState === "saved" && savedTitle && ( - - {t("as “{title}”", { title: savedTitle })} - - )} - {saveState === "error" && ( - - {t("save failed (see console)")} - - )} +
+
)}
diff --git a/src/components/AIAssistant/ToolConfirmDialog.tsx b/src/components/AIAssistant/ToolConfirmDialog.tsx index 9cf9970..ec5af9c 100644 --- a/src/components/AIAssistant/ToolConfirmDialog.tsx +++ b/src/components/AIAssistant/ToolConfirmDialog.tsx @@ -8,6 +8,7 @@ */ import { lazy, Suspense, useEffect, useState } from "react"; +import { isNonExecutingMacroCall } from "../../aiassistant/tools"; import type { Tool } from "../../aiassistant/types"; import { t } from "../../i18n/translate"; @@ -59,6 +60,7 @@ export function ToolConfirmDialog({ request }: Props) { } const hasCompactArgs = Object.keys(compactArgs).length > 0; const compactArgsText = JSON.stringify(compactArgs, null, 2); + const noExec = isNonExecutingMacroCall(request.tool.name, request.args); const handleApprove = () => request.resolve({ approve: true, remember }); const handleReject = () => request.resolve({ approve: false, remember: false }); @@ -82,6 +84,16 @@ export function ToolConfirmDialog({ request }: Props) { "The assistant wants to run the following mutating tool. Review the arguments and approve or reject.", )}

+ {noExec && ( +

+ {t( + "This macro will not be executed: approving only offers it for saving to the Macros panel.", + )} +

+ )}
setRemember(e.target.checked)} /> - {t("Auto-approve")} {request.tool.name}{" "} - {t("for the rest of this conversation")} + {noExec ? ( + t( + "Auto-approve macros offered without being run for the rest of this conversation", + ) + ) : ( + <> + {t("Auto-approve")} {request.tool.name}{" "} + {t("for the rest of this conversation")} + + )}