capture-viewer: render extracted JSON preview artifacts

packages/capture-viewer/src/capture-runtime.tsx

@@ -27,6 +27,7 @@ import {
   useState,
 } from 'react'
 import type { Object3D } from 'three'
+import { rewriteLoopbackAssetUrl } from './asset-url'
 import { resolveCaptureFrameMatrix } from './frame'
 import { isCaptureSessionVisible, isCaptureStreamVisible } from './layer-visibility'
 import { CaptureDeviceMotionLayer } from './layers/device-motion-layer'
@@ -39,6 +40,7 @@ import {
   isCaptureModelArtifact,
   isCapturePointCloudArtifact,
   isCaptureStreamRenderable,
+  streamHydratesJsonPayload,
 } from './stream-rendering'
 import { parseDeviceTrajectoryPackets, parseDeviceTrajectoryPayload } from './trajectory'
 
@@ -248,23 +250,42 @@ export function CaptureStreamLayer({
   const artifactUrl = useResolvedArtifact(source, stream.artifact)
   const layerKey = captureLayerKey(stream)
   const Renderer = renderers[layerKey] ?? renderers[stream.kind]
+  // Extracted previews archive the inline payload shape as a JSON artifact.
+  const payloadArtifactUrl = streamHydratesJsonPayload(stream) ? artifactUrl : null
+  const fetchedPayload = useJsonArtifactPayload(payloadArtifactUrl)
+  const payload = stream.inline ?? fetchedPayload
   const frameId = packets.at(-1)?.frameId ?? stream.frameId ?? stream.artifact?.frameId
   const frameMatrix = useMemo(
     () => resolveCaptureFrameMatrix(descriptor, frameId),
     [descriptor, frameId],
   )
   const trajectory = useMemo(() => {
     if (layerKey !== 'deviceMotion') return null
-    const inline = DeviceMotionTrajectorySchema.safeParse(stream.inline)
+    const inline = DeviceMotionTrajectorySchema.safeParse(payload)
     return inline.success
       ? parseDeviceTrajectoryPayload(inline.data)
       : parseDeviceTrajectoryPackets(packets.map((packet) => packet.payload))
-  }, [layerKey, packets, stream.inline])
+  }, [layerKey, packets, payload])
   const motionPlaybackKey = useMemo(() => {
     if (layerKey !== 'deviceMotion') return ''
-    const inlineVersion = packets.length === 0 ? JSON.stringify(stream.inline ?? null) : ''
+    // Fetched payloads can be megabytes — key them by artifact identity and
+    // load state instead of stringifying their content.
+    const inlineVersion =
+      packets.length === 0
+        ? stream.inline != null
+          ? JSON.stringify(stream.inline)
+          : `${payloadArtifactUrl ?? ''}:${fetchedPayload ? 'loaded' : 'pending'}`
+        : ''
     return [descriptor.revisionId ?? '', streamEpoch, inlineVersion].join(':')
-  }, [descriptor.revisionId, layerKey, packets.length, stream.inline, streamEpoch])
+  }, [
+    descriptor.revisionId,
+    fetchedPayload,
+    layerKey,
+    packets.length,
+    payloadArtifactUrl,
+    stream.inline,
+    streamEpoch,
+  ])
   if (frameId && !frameMatrix) {
     throw new Error(`Capture stream ${stream.id} references an invalid frame: ${frameId}.`)
   }
@@ -305,12 +326,12 @@ export function CaptureStreamLayer({
         artifactUrl={
           isCapturePointCloudArtifact(stream.artifact) ? (artifactUrl ?? undefined) : undefined
         }
-        inline={stream.inline}
+        inline={payload}
         packets={stream.availability === 'live' ? packets : []}
       />
     )
   } else if (layerKey === 'surfaceMesh') {
-    content = <CaptureSurfaceMeshLayer inline={stream.inline} />
+    content = <CaptureSurfaceMeshLayer inline={payload} />
   }
   if (!(content && frameMatrix)) return content
   return (
@@ -320,6 +341,34 @@ export function CaptureStreamLayer({
   )
 }
 
+function useJsonArtifactPayload(url: string | null): unknown {
+  const [error, setError] = useState<Error | null>(null)
+  const [payload, setPayload] = useState<unknown>(null)
+
+  useEffect(() => {
+    setError(null)
+    setPayload(null)
+    if (!url) return
+    const abort = new AbortController()
+    void fetch(rewriteLoopbackAssetUrl(url), { signal: abort.signal })
+      .then(async (response) => {
+        if (!response.ok) throw new Error(`Could not load ${url}: ${response.status}`)
+        return (await response.json()) as unknown
+      })
+      .then((data) => {
+        if (!abort.signal.aborted) setPayload(data)
+      })
+      .catch((cause: unknown) => {
+        if (abort.signal.aborted) return
+        setError(cause instanceof Error ? cause : new Error(`Could not load ${url}.`))
+      })
+    return () => abort.abort()
+  }, [url])
+
+  if (error) throw error
+  return payload
+}
+
 function useResolvedArtifact(
   source: CaptureSource,
   artifact: CaptureArtifactReference | undefined,

packages/capture-viewer/src/stream-rendering.test.ts

@@ -44,6 +44,28 @@ describe('isCaptureStreamRenderable', () => {
     ).toBe(true)
   })
 
+  test('renders extracted JSON preview artifacts', () => {
+    for (const [kind, role] of [
+      ['surface-mesh', 'surfaceMesh'],
+      ['point-cloud', 'pointCloud'],
+      ['device-motion', 'deviceMotion'],
+    ] as const) {
+      expect(
+        isCaptureStreamRenderable({
+          id: kind,
+          kind,
+          role,
+          availability: 'ready',
+          artifact: {
+            id: `preview-${kind}`,
+            mediaType: 'application/json',
+            uri: `/api/captures/c/archive/sessions/s/artifacts/preview-${kind}`,
+          },
+        }),
+      ).toBe(true)
+    }
+  })
+
   test('renders a valid inline color surface mesh', () => {
     expect(
       isCaptureStreamRenderable({

packages/capture-viewer/src/stream-rendering.ts

@@ -10,6 +10,7 @@ import {
 const GLB_MEDIA_TYPES = new Set(['model/gltf-binary', 'model/gltf+json'])
 const USDZ_MEDIA_TYPES = new Set(['model/vnd.usdz+zip'])
 const PLY_MEDIA_TYPES = new Set(['application/ply', 'application/vnd.ply', 'model/ply'])
+const JSON_MEDIA_TYPES = new Set(['application/json'])
 
 export type CaptureModelFormat = 'gltf' | 'usdz'
 
@@ -24,20 +25,41 @@ export function isCaptureStreamRenderable(
   if (layerKey === 'deviceMotion') {
     return (
       stream.availability === 'live' ||
+      streamHydratesJsonPayload(stream) ||
       DeviceMotionTrajectorySchema.safeParse(stream.inline).success
     )
   }
   if (layerKey === 'pointCloud') {
     return (
       stream.availability === 'live' ||
       isCapturePointCloudArtifact(stream.artifact) ||
+      streamHydratesJsonPayload(stream) ||
       PointCloudPayloadSchema.safeParse(stream.inline).success
     )
   }
-  if (layerKey === 'surfaceMesh') return SurfaceMeshPayloadSchema.safeParse(stream.inline).success
+  if (layerKey === 'surfaceMesh') {
+    return (
+      streamHydratesJsonPayload(stream) || SurfaceMeshPayloadSchema.safeParse(stream.inline).success
+    )
+  }
   return false
 }
 
+const JSON_PAYLOAD_LAYER_KEYS = new Set(['deviceMotion', 'pointCloud', 'surfaceMesh'])
+
+/**
+ * Extracted viewer previews (device motion, point cloud, surface mesh) are
+ * archived as JSON payload artifacts whose content matches the inline shape.
+ * One predicate decides both renderability and runtime hydration, so a
+ * stream can never be declared renderable without a hydration path.
+ */
+export function streamHydratesJsonPayload(stream: CaptureStreamDescriptor): boolean {
+  const artifact = stream.artifact
+  if (!artifact || stream.inline != null) return false
+  if (!JSON_PAYLOAD_LAYER_KEYS.has(captureLayerKey(stream))) return false
+  return JSON_MEDIA_TYPES.has(artifact.mediaType) || hasExtension(artifact.uri, ['.json'])
+}
+
 export function isCaptureModelArtifact(artifact: CaptureArtifactReference | undefined): boolean {
   return captureModelFormat(artifact) !== null
 }