diff --git a/packages/capture-viewer/src/capture-runtime.tsx b/packages/capture-viewer/src/capture-runtime.tsx index 93455ebb4..e692334ba 100644 --- a/packages/capture-viewer/src/capture-runtime.tsx +++ b/packages/capture-viewer/src/capture-runtime.tsx @@ -27,6 +27,7 @@ import { useState, } from 'react' import type { Object3D } from 'three' +import { rewriteLoopbackAssetUrl } from './asset-url' import { resolveCaptureFrameMatrix } from './frame' import { isCaptureSessionVisible, isCaptureStreamVisible } from './layer-visibility' import { CaptureDeviceMotionLayer } from './layers/device-motion-layer' @@ -39,6 +40,7 @@ import { isCaptureModelArtifact, isCapturePointCloudArtifact, isCaptureStreamRenderable, + streamHydratesJsonPayload, } from './stream-rendering' import { parseDeviceTrajectoryPackets, parseDeviceTrajectoryPayload } from './trajectory' @@ -248,6 +250,10 @@ export function CaptureStreamLayer({ const artifactUrl = useResolvedArtifact(source, stream.artifact) const layerKey = captureLayerKey(stream) const Renderer = renderers[layerKey] ?? renderers[stream.kind] + // Extracted previews archive the inline payload shape as a JSON artifact. + const payloadArtifactUrl = streamHydratesJsonPayload(stream) ? artifactUrl : null + const fetchedPayload = useJsonArtifactPayload(payloadArtifactUrl) + const payload = stream.inline ?? fetchedPayload const frameId = packets.at(-1)?.frameId ?? stream.frameId ?? stream.artifact?.frameId const frameMatrix = useMemo( () => resolveCaptureFrameMatrix(descriptor, frameId), @@ -255,16 +261,31 @@ export function CaptureStreamLayer({ ) const trajectory = useMemo(() => { if (layerKey !== 'deviceMotion') return null - const inline = DeviceMotionTrajectorySchema.safeParse(stream.inline) + const inline = DeviceMotionTrajectorySchema.safeParse(payload) return inline.success ? parseDeviceTrajectoryPayload(inline.data) : parseDeviceTrajectoryPackets(packets.map((packet) => packet.payload)) - }, [layerKey, packets, stream.inline]) + }, [layerKey, packets, payload]) const motionPlaybackKey = useMemo(() => { if (layerKey !== 'deviceMotion') return '' - const inlineVersion = packets.length === 0 ? JSON.stringify(stream.inline ?? null) : '' + // Fetched payloads can be megabytes — key them by artifact identity and + // load state instead of stringifying their content. + const inlineVersion = + packets.length === 0 + ? stream.inline != null + ? JSON.stringify(stream.inline) + : `${payloadArtifactUrl ?? ''}:${fetchedPayload ? 'loaded' : 'pending'}` + : '' return [descriptor.revisionId ?? '', streamEpoch, inlineVersion].join(':') - }, [descriptor.revisionId, layerKey, packets.length, stream.inline, streamEpoch]) + }, [ + descriptor.revisionId, + fetchedPayload, + layerKey, + packets.length, + payloadArtifactUrl, + stream.inline, + streamEpoch, + ]) if (frameId && !frameMatrix) { throw new Error(`Capture stream ${stream.id} references an invalid frame: ${frameId}.`) } @@ -305,12 +326,12 @@ export function CaptureStreamLayer({ artifactUrl={ isCapturePointCloudArtifact(stream.artifact) ? (artifactUrl ?? undefined) : undefined } - inline={stream.inline} + inline={payload} packets={stream.availability === 'live' ? packets : []} /> ) } else if (layerKey === 'surfaceMesh') { - content = + content = } if (!(content && frameMatrix)) return content return ( @@ -320,6 +341,34 @@ export function CaptureStreamLayer({ ) } +function useJsonArtifactPayload(url: string | null): unknown { + const [error, setError] = useState(null) + const [payload, setPayload] = useState(null) + + useEffect(() => { + setError(null) + setPayload(null) + if (!url) return + const abort = new AbortController() + void fetch(rewriteLoopbackAssetUrl(url), { signal: abort.signal }) + .then(async (response) => { + if (!response.ok) throw new Error(`Could not load ${url}: ${response.status}`) + return (await response.json()) as unknown + }) + .then((data) => { + if (!abort.signal.aborted) setPayload(data) + }) + .catch((cause: unknown) => { + if (abort.signal.aborted) return + setError(cause instanceof Error ? cause : new Error(`Could not load ${url}.`)) + }) + return () => abort.abort() + }, [url]) + + if (error) throw error + return payload +} + function useResolvedArtifact( source: CaptureSource, artifact: CaptureArtifactReference | undefined, diff --git a/packages/capture-viewer/src/stream-rendering.test.ts b/packages/capture-viewer/src/stream-rendering.test.ts index 48d00c02b..857777c38 100644 --- a/packages/capture-viewer/src/stream-rendering.test.ts +++ b/packages/capture-viewer/src/stream-rendering.test.ts @@ -44,6 +44,28 @@ describe('isCaptureStreamRenderable', () => { ).toBe(true) }) + test('renders extracted JSON preview artifacts', () => { + for (const [kind, role] of [ + ['surface-mesh', 'surfaceMesh'], + ['point-cloud', 'pointCloud'], + ['device-motion', 'deviceMotion'], + ] as const) { + expect( + isCaptureStreamRenderable({ + id: kind, + kind, + role, + availability: 'ready', + artifact: { + id: `preview-${kind}`, + mediaType: 'application/json', + uri: `/api/captures/c/archive/sessions/s/artifacts/preview-${kind}`, + }, + }), + ).toBe(true) + } + }) + test('renders a valid inline color surface mesh', () => { expect( isCaptureStreamRenderable({ diff --git a/packages/capture-viewer/src/stream-rendering.ts b/packages/capture-viewer/src/stream-rendering.ts index e5e773d8c..2fcfea7ae 100644 --- a/packages/capture-viewer/src/stream-rendering.ts +++ b/packages/capture-viewer/src/stream-rendering.ts @@ -10,6 +10,7 @@ import { const GLB_MEDIA_TYPES = new Set(['model/gltf-binary', 'model/gltf+json']) const USDZ_MEDIA_TYPES = new Set(['model/vnd.usdz+zip']) const PLY_MEDIA_TYPES = new Set(['application/ply', 'application/vnd.ply', 'model/ply']) +const JSON_MEDIA_TYPES = new Set(['application/json']) export type CaptureModelFormat = 'gltf' | 'usdz' @@ -24,6 +25,7 @@ export function isCaptureStreamRenderable( if (layerKey === 'deviceMotion') { return ( stream.availability === 'live' || + streamHydratesJsonPayload(stream) || DeviceMotionTrajectorySchema.safeParse(stream.inline).success ) } @@ -31,13 +33,33 @@ export function isCaptureStreamRenderable( return ( stream.availability === 'live' || isCapturePointCloudArtifact(stream.artifact) || + streamHydratesJsonPayload(stream) || PointCloudPayloadSchema.safeParse(stream.inline).success ) } - if (layerKey === 'surfaceMesh') return SurfaceMeshPayloadSchema.safeParse(stream.inline).success + if (layerKey === 'surfaceMesh') { + return ( + streamHydratesJsonPayload(stream) || SurfaceMeshPayloadSchema.safeParse(stream.inline).success + ) + } return false } +const JSON_PAYLOAD_LAYER_KEYS = new Set(['deviceMotion', 'pointCloud', 'surfaceMesh']) + +/** + * Extracted viewer previews (device motion, point cloud, surface mesh) are + * archived as JSON payload artifacts whose content matches the inline shape. + * One predicate decides both renderability and runtime hydration, so a + * stream can never be declared renderable without a hydration path. + */ +export function streamHydratesJsonPayload(stream: CaptureStreamDescriptor): boolean { + const artifact = stream.artifact + if (!artifact || stream.inline != null) return false + if (!JSON_PAYLOAD_LAYER_KEYS.has(captureLayerKey(stream))) return false + return JSON_MEDIA_TYPES.has(artifact.mediaType) || hasExtension(artifact.uri, ['.json']) +} + export function isCaptureModelArtifact(artifact: CaptureArtifactReference | undefined): boolean { return captureModelFormat(artifact) !== null }