Refine composer tools, effort selection, and Copilot cursor

scripts/computer-use-runtime.py

@@ -190,18 +190,53 @@ def source_checkout(source):
 
 
 def draw_cursor(path):
-    # Original blue arrow geometry; no upstream/reference artwork is copied.
     width = height = 126
-    polygon = [(60, 56), (60, 80), (66, 74), (72, 86), (77, 83), (71, 71), (83, 71)]
+    # AppKit's (60, 70) tip anchor is (60, 56) in this top-origin PNG.
+    polygon = [(60, 56), (60.7, 77.8), (66.1, 73.1), (69.1, 78.2),
+               (72.4, 76.2), (69.4, 71.2), (77.8, 70.4)]
+    fill = (42, 37, 32)
+    edge = (244, 240, 230)
+    edge_width = 0.75
+    samples = 4
+    sides = list(zip(polygon, polygon[1:] + polygon[:1]))
+    left, right = min(int(x) for x, _ in polygon), max(int(x) for x, _ in polygon)
+    top, bottom = min(int(y) for _, y in polygon), max(int(y) for _, y in polygon)
+
     def inside(x, y):
         crossings = 0
-        for i, (ax, ay) in enumerate(polygon):
-            bx, by = polygon[(i + 1) % len(polygon)]
+        for (ax, ay), (bx, by) in sides:
             if (ay > y) != (by > y) and x < (bx - ax) * (y - ay) / (by - ay) + ax:
                 crossings += 1
         return crossings % 2
-    raw = b"".join(b"\0" + b"".join(bytes((30, 112, 220, 255)) if inside(x, y) else bytes(4)
-                                    for x in range(width)) for y in range(height))
+
+    def near_edge(x, y):
+        for (ax, ay), (bx, by) in sides:
+            dx, dy = bx - ax, by - ay
+            along = max(0, min(1, ((x - ax) * dx + (y - ay) * dy) / (dx * dx + dy * dy)))
+            if (x - ax - along * dx) ** 2 + (y - ay - along * dy) ** 2 < edge_width ** 2:
+                return True
+        return False
+
+    rows = []
+    for y in range(height):
+        row = bytearray(b"\0")
+        for x in range(width):
+            colors = []
+            if left <= x <= right and top <= y <= bottom:
+                for sy in range(samples):
+                    for sx in range(samples):
+                        px, py = x + (sx + 0.5) / samples, y + (sy + 0.5) / samples
+                        if inside(px, py):
+                            colors.append(edge if near_edge(px, py) else fill)
+            if colors:
+                row.extend(round(sum(color[channel] for color in colors) / len(colors))
+                           for channel in range(3))
+                row.append(round(255 * len(colors) / (samples * samples)))
+            else:
+                row.extend(bytes(4))
+        rows.append(row)
+    raw = b"".join(rows)
+
     def chunk(kind, data):
         return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
     path.write_bytes(b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0))

scripts/test-computer-use-packaging.py

@@ -3,11 +3,13 @@
 import importlib.util
 import json
 from pathlib import Path
+import re
 import struct
 import subprocess
 import tempfile
 import unittest
 import zipfile
+import zlib
 
 spec = importlib.util.spec_from_file_location("runtime", Path(__file__).with_name("computer-use-runtime.py"))
 runtime = importlib.util.module_from_spec(spec)
@@ -164,6 +166,56 @@ def test_boolean_is_not_a_numeric_schema_bound(self):
                     protocol.validate_tools(tools)
 
 
+class CursorArtworkTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        path = Path(self.temp.name) / "biorouter-cursor.png"
+        runtime.draw_cursor(path)
+        data = path.read_bytes()
+        self.assertEqual(data[:8], b"\x89PNG\r\n\x1a\n")
+        offset, compressed = 8, b""
+        while offset < len(data):
+            length = struct.unpack_from(">I", data, offset)[0]
+            kind, body = data[offset + 4:offset + 8], data[offset + 8:offset + 8 + length]
+            if kind == b"IHDR":
+                self.width, self.height, depth, color, _, _, _ = struct.unpack(">IIBBBBB", body)
+                self.assertEqual((depth, color), (8, 6))
+            elif kind == b"IDAT":
+                compressed += body
+            offset += length + 12
+        raw = zlib.decompress(compressed)
+        self.pixels = {}
+        for y in range(self.height):
+            row = raw[y * (self.width * 4 + 1):(y + 1) * (self.width * 4 + 1)]
+            self.assertEqual(row[0], 0)
+            for x in range(self.width):
+                self.pixels[x, y] = tuple(row[1 + x * 4:5 + x * 4])
+        self.visible = {point: color for point, color in self.pixels.items() if color[3] > 0}
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def test_compact_artwork_keeps_the_native_tip_anchor(self):
+        self.assertEqual((self.width, self.height), (126, 126))
+        branding = (runtime.VENDOR / "patches/0001-branding.patch").read_text()
+        anchor = re.search(r"\+    static let tipAnchor = CGPoint\(x: ([\d.]+), y: ([\d.]+)\)", branding)
+        tip_x, tip_y = float(anchor[1]), self.height - float(anchor[2])
+        self.assertEqual(min(x for x, _ in self.visible), tip_x)
+        self.assertEqual(min(y for _, y in self.visible), tip_y)
+        self.assertGreater(self.pixels[int(tip_x), int(tip_y)][3], 0)
+        self.assertLessEqual(max(x for x, _ in self.visible) - tip_x + 1, 20)
+        self.assertLessEqual(max(y for _, y in self.visible) - tip_y + 1, 24)
+
+    def test_neutral_pointer_has_a_light_edge_and_antialiased_transparency(self):
+        opaque = [color for color in self.visible.values() if color[3] == 255]
+        self.assertGreater(sum(max(color[:3]) < 65 for color in opaque), len(opaque) / 2)
+        self.assertTrue(any(min(color[:3]) > 220 for color in self.visible.values()))
+        self.assertTrue(all(max(color[:3]) - min(color[:3]) <= 20 for color in self.visible.values()))
+        self.assertTrue(any(0 < color[3] < 255 for color in self.visible.values()))
+        self.assertEqual(self.pixels[0, 0], (0, 0, 0, 0))
+        self.assertEqual(self.pixels[125, 125], (0, 0, 0, 0))
+
+
 class PayloadTests(unittest.TestCase):
     def setUp(self):
         self.temp = tempfile.TemporaryDirectory()

ui/desktop/src/components/ChatInput.firstSendRemount.test.tsx

@@ -253,13 +253,17 @@ function ChatPane({ clean, sessionId, composerKey = TAB }: PaneProps) {
   );
 }
 
+const toolCounts = (chip: HTMLElement | null) =>
+  chip?.getAttribute('aria-label')?.match(/\d+/g) ?? [];
+
 /** What the composer states in one frame: the Tools chip, its name, the padlock, the folder. */
 function frame(): string {
   const tools = screen.queryByTestId('composer-tools-chip');
   const model = screen.queryByTestId('model-chip');
   const lock = model?.querySelector('[data-testid="privacy-badge"]') ? '+lock' : 'no lock';
   const folder = screen.queryByTestId('footer-folder')?.textContent ?? '-';
-  return `${tools?.textContent ?? '-'} | ${tools?.getAttribute('aria-label') ?? '-'} | ${lock} | ${folder}`;
+  const counts = toolCounts(tools).join(', ');
+  return `${tools?.textContent ?? '-'} | ${counts || '-'} | ${tools?.getAttribute('aria-label') ?? '-'} | ${lock} | ${folder}`;
 }
 
 /** Every distinct committed state from now on (the observer reports per commit). */
@@ -283,7 +287,7 @@ const settle = (ms: number) =>
     await new Promise((resolve) => setTimeout(resolve, ms));
   });
 
-const SETTLED = `Tools5 | Tools: 2 extensions, 2 skills, 1 knowledge base | +lock | ${CHOSEN_DIR}`;
+const SETTLED = `Tools | 2, 2, 1 | Tools: 2 extensions, 2 skills, 1 knowledge base | +lock | ${CHOSEN_DIR}`;
 
 /** A new tab, answered, with a folder picked before the first message. */
 async function openNewChatWithPickedFolder() {
@@ -294,7 +298,7 @@ async function openNewChatWithPickedFolder() {
 }
 
 describe('a new chat’s first send keeps what the composer states (V2-R8-3)', () => {
-  it('keeps "Tools 5", its name, the padlock and the picked folder in every commit', async () => {
+  it('keeps all three tool counts, their name, the padlock and the picked folder in every commit', async () => {
     const view = await openNewChatWithPickedFolder();
     const fieldBefore = screen.getByTestId('chat-input');
     const record = recordFrames();
@@ -321,6 +325,7 @@ describe('a new chat’s first send keeps what the composer states (V2-R8-3)', (
 
     // The same tab, opened again: it asks for itself.
     const again = render(<ChatPane clean sessionId={null} />);
+    expect(toolCounts(screen.getByTestId('composer-tools-chip'))).toEqual([]);
     expect(screen.getByTestId('composer-tools-chip')).toHaveTextContent(/^Tools$/);
     expect(screen.getByTestId('composer-tools-chip')).toHaveAccessibleName('Tools');
     expect(screen.getByTestId('footer-folder')).toHaveTextContent(DEFAULT_DIR);
@@ -331,13 +336,14 @@ describe('a new chat’s first send keeps what the composer states (V2-R8-3)', (
     const chatA = render(<ChatPane clean={false} sessionId={NEW_CHAT} />);
     await waitFor(() => expect(frame()).toBe(SETTLED));
     chatA.rerender(<ChatPane clean sessionId={OTHER_CHAT} />);
+    expect(toolCounts(screen.getByTestId('composer-tools-chip'))).toEqual([]);
     expect(screen.getByTestId('composer-tools-chip')).toHaveTextContent(/^Tools$/);
     expect(screen.getByTestId('composer-tools-chip')).toHaveAccessibleName('Tools');
     expect(screen.getByTestId('footer-folder')).toHaveTextContent(/^$/);
     // Then its own figures and folder, once they are read.
     await waitFor(() =>
       expect(frame()).toBe(
-        `Tools5 | Tools: 2 extensions, 2 skills, 1 knowledge base | +lock | ${OTHER_DIR}`
+        `Tools | 2, 2, 1 | Tools: 2 extensions, 2 skills, 1 knowledge base | +lock | ${OTHER_DIR}`
       )
     );
     chatA.unmount();

ui/desktop/src/components/bottom_menu/EffortSection.tsx

@@ -1,7 +1,7 @@
 import './pickers.css';
 import * as React from 'react';
 import { InfoTip, useInfoTipId } from '../ui/info-tip';
-import { Slider, type SliderStep } from '../ui/slider';
+import { SegmentedControl, type SegmentedOption } from '../ui/segmented-control';
 import {
   REASONING_EFFORT_HELP,
   REASONING_EFFORT_LABELS,
@@ -11,42 +11,43 @@ import {
 } from '../../store/reasoningEffort';
 import { MODEL_COPY } from './copy';
 
-const STEPS: ReadonlyArray<SliderStep<ReasoningEffort>> = REASONING_EFFORTS.map((effort) => ({
-  value: effort,
-  label: REASONING_EFFORT_LABELS[effort],
-}));
+const OPTIONS: ReadonlyArray<SegmentedOption<ReasoningEffort>> = REASONING_EFFORTS.map(
+  (effort) => ({
+    value: effort,
+    label: REASONING_EFFORT_LABELS[effort],
+  })
+);
 
 export interface EffortSectionProps {
   /** Where the effort is kept (`store/reasoningEffort.ts`): a chat's scope, or a draft's. */
   scope: string;
   /** The effort the store holds for `scope`. */
   effort: ReasoningEffort;
-  /** The slider's thumb, so the menu around it can move focus there (`moveFocusAcrossEffort`). */
-  thumbRef?: React.Ref<HTMLSpanElement>;
+  /** Lets the surrounding menu reach the selected segment with Up and Down. */
+  controlRef?: React.Ref<HTMLDivElement>;
 }
 
 /**
- * The model menu's Effort section (R-26, owner message 13): ONE slider with three stops,
- * Quick, Normal and Deep, where the menu used to list three radio rows.
- *
- * The head row says "Effort" with an InfoTip on the left and the current stop on the right; the
- * slider under it fills the menu's width. A change applies at once, through the same store and
- * the same scope the radio rows wrote (`setReasoningEffort(scope, …)`), and the menu stays open
- * so the person sees what they set.
- *
- * The InfoTip carries what the three levels mean, and the thumb points at the InfoTip's hidden
- * description, so a screen reader hears it on focus without hovering: the per-level descriptions
- * the radio rows had in tooltips are now one place.
- *
- * The section is a `group` named "Effort" (a `menu` may own groups; a bare slider among its rows
- * would not be one of its allowed children). How the keyboard reaches it inside the menu is
- * `moveFocusAcrossEffort`, below.
+ * A choice applies immediately and keeps the menu open. The surrounding menu's vertical keys
+ * visit this control as one stop; horizontal keys stay within its radio group.
  */
-export function EffortSection({ scope, effort, thumbRef }: EffortSectionProps) {
+export function EffortSection({ scope, effort, controlRef }: EffortSectionProps) {
   const labelId = React.useId();
   const helpId = useInfoTipId();
   return (
-    <div role="group" aria-labelledby={labelId} className="br-effort" data-testid="effort-section">
+    <div
+      role="group"
+      aria-labelledby={labelId}
+      className="br-effort"
+      data-testid="effort-section"
+      onKeyDown={(event) => {
+        if (
+          ['ArrowLeft', 'ArrowRight', 'Home', 'End', ' '].includes(event.key) &&
+          (event.target as HTMLElement).closest('[role="radiogroup"]')
+        )
+          event.stopPropagation();
+      }}
+    >
       <div className="br-effort__head">
         <span id={labelId} className="br-effort__label">
           {MODEL_COPY.effortGroup}
@@ -57,20 +58,17 @@ export function EffortSection({ scope, effort, thumbRef }: EffortSectionProps) {
           help={REASONING_EFFORT_HELP}
           data-testid="effort-info"
         />
-        {/* The slider speaks its own value; this is the same word for the eye. */}
-        <span className="br-effort__value" aria-hidden="true" data-testid="effort-value">
-          {REASONING_EFFORT_LABELS[effort]}
-        </span>
       </div>
-      <Slider
-        ref={thumbRef}
-        steps={STEPS}
+      <SegmentedControl
+        ref={controlRef}
+        options={OPTIONS}
+        fill
         value={effort}
         onValueChange={(next) => setReasoningEffort(scope, next)}
         aria-labelledby={labelId}
         aria-describedby={helpId}
-        className="br-effort__slider"
-        data-testid="effort-slider"
+        className="br-effort__control"
+        data-testid="effort-control"
       />
     </div>
   );
@@ -80,53 +78,42 @@ const MENU_ROW_SELECTOR =
   '[role="menuitem"]:not([data-disabled]), [role="menuitemradio"]:not([data-disabled]), [role="menuitemcheckbox"]:not([data-disabled])';
 
 /**
- * Lets the menu's Up and Down keys reach the Effort slider, as `onKeyDownCapture` on the menu.
- *
- * ⚠ **A Radix menu's arrow keys visit menu ROWS only** (its roving focus group holds
- * `MenuItem`s), and its Tab leaves the menu (`useTabLeavesMenu`), so a slider among the rows is
- * reached by neither: the keyboard could not set the effort at all. So the slider is made one
- * more stop in the menu's vertical order: Up from the row below it lands on the thumb, Down
- * from the thumb goes to that row, and the same the other way round should rows ever sit above
- * it. On the thumb, Up and Down walk the MENU (the menu runs vertically, and a slider that
- * swallowed them would be a dead end the person can only leave with Escape); Left, Right, Home,
- * End, Page Up and Page Down move the effort, as the slider pattern has them. Anything else
- * (row to row, typeahead, Tab, Escape) is left to Radix.
- *
- * Runs in the capture phase, so it decides before the thumb's own handler (which would take Up
- * and Down) and before the row's roving handler.
+ * Radix menus only visit menu rows. Include the selected effort segment in that vertical order,
+ * before the radio group's own key handler can interpret Up or Down as a value change.
  */
 export function moveFocusAcrossEffort(
   event: React.KeyboardEvent<HTMLElement>,
-  thumb: HTMLElement | null
+  control: HTMLElement | null
 ): void {
-  if (!thumb || (event.key !== 'ArrowUp' && event.key !== 'ArrowDown')) return;
+  if (!control || (event.key !== 'ArrowUp' && event.key !== 'ArrowDown')) return;
   if (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey) return;
+  const selected = control.querySelector<HTMLElement>('[role="radio"][aria-checked="true"]');
+  if (!selected) return;
   const menu = event.currentTarget;
   const target = event.target as HTMLElement;
   const rows = Array.from(menu.querySelectorAll<HTMLElement>(MENU_ROW_SELECTOR));
-  // Every stop in document order: the rows and the thumb.
-  const stops = [...rows, thumb].sort((a, b) =>
+  const stops = [...rows, selected].sort((a, b) =>
     a === b ? 0 : a.compareDocumentPosition(b) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1
   );
   const step = event.key === 'ArrowDown' ? 1 : -1;
 
   let next: HTMLElement | undefined;
-  if (target === thumb) {
-    next = stops[stops.indexOf(thumb) + step];
-    // No row that way: stay on the thumb, and do not let the slider read it as a value key.
+  if (control.contains(target)) {
+    next = stops[stops.indexOf(selected) + step];
+    // At the menu edge, keep the value and focus where they are.
     if (!next) {
       event.preventDefault();
       event.stopPropagation();
       return;
     }
   } else if (target === menu) {
-    // Focus on the menu itself (the pointer left a row): with no row to go to, the thumb.
+    // Focus on the menu itself (the pointer left a row): with no row to go to, the selection.
     if (rows.length > 0) return;
-    next = thumb;
+    next = selected;
   } else {
     const at = stops.indexOf(target);
-    if (at === -1 || stops[at + step] !== thumb) return;
-    next = thumb;
+    if (at === -1 || stops[at + step] !== selected) return;
+    next = selected;
   }
   event.preventDefault();
   event.stopPropagation();