diff --git a/packages/capture-viewer/src/capture-runtime.tsx b/packages/capture-viewer/src/capture-runtime.tsx
index 93455ebb4..e692334ba 100644
--- a/packages/capture-viewer/src/capture-runtime.tsx
+++ b/packages/capture-viewer/src/capture-runtime.tsx
@@ -27,6 +27,7 @@ import {
useState,
} from 'react'
import type { Object3D } from 'three'
+import { rewriteLoopbackAssetUrl } from './asset-url'
import { resolveCaptureFrameMatrix } from './frame'
import { isCaptureSessionVisible, isCaptureStreamVisible } from './layer-visibility'
import { CaptureDeviceMotionLayer } from './layers/device-motion-layer'
@@ -39,6 +40,7 @@ import {
isCaptureModelArtifact,
isCapturePointCloudArtifact,
isCaptureStreamRenderable,
+ streamHydratesJsonPayload,
} from './stream-rendering'
import { parseDeviceTrajectoryPackets, parseDeviceTrajectoryPayload } from './trajectory'
@@ -248,6 +250,10 @@ export function CaptureStreamLayer({
const artifactUrl = useResolvedArtifact(source, stream.artifact)
const layerKey = captureLayerKey(stream)
const Renderer = renderers[layerKey] ?? renderers[stream.kind]
+ // Extracted previews archive the inline payload shape as a JSON artifact.
+ const payloadArtifactUrl = streamHydratesJsonPayload(stream) ? artifactUrl : null
+ const fetchedPayload = useJsonArtifactPayload(payloadArtifactUrl)
+ const payload = stream.inline ?? fetchedPayload
const frameId = packets.at(-1)?.frameId ?? stream.frameId ?? stream.artifact?.frameId
const frameMatrix = useMemo(
() => resolveCaptureFrameMatrix(descriptor, frameId),
@@ -255,16 +261,31 @@ export function CaptureStreamLayer({
)
const trajectory = useMemo(() => {
if (layerKey !== 'deviceMotion') return null
- const inline = DeviceMotionTrajectorySchema.safeParse(stream.inline)
+ const inline = DeviceMotionTrajectorySchema.safeParse(payload)
return inline.success
? parseDeviceTrajectoryPayload(inline.data)
: parseDeviceTrajectoryPackets(packets.map((packet) => packet.payload))
- }, [layerKey, packets, stream.inline])
+ }, [layerKey, packets, payload])
const motionPlaybackKey = useMemo(() => {
if (layerKey !== 'deviceMotion') return ''
- const inlineVersion = packets.length === 0 ? JSON.stringify(stream.inline ?? null) : ''
+ // Fetched payloads can be megabytes — key them by artifact identity and
+ // load state instead of stringifying their content.
+ const inlineVersion =
+ packets.length === 0
+ ? stream.inline != null
+ ? JSON.stringify(stream.inline)
+ : `${payloadArtifactUrl ?? ''}:${fetchedPayload ? 'loaded' : 'pending'}`
+ : ''
return [descriptor.revisionId ?? '', streamEpoch, inlineVersion].join(':')
- }, [descriptor.revisionId, layerKey, packets.length, stream.inline, streamEpoch])
+ }, [
+ descriptor.revisionId,
+ fetchedPayload,
+ layerKey,
+ packets.length,
+ payloadArtifactUrl,
+ stream.inline,
+ streamEpoch,
+ ])
if (frameId && !frameMatrix) {
throw new Error(`Capture stream ${stream.id} references an invalid frame: ${frameId}.`)
}
@@ -305,12 +326,12 @@ export function CaptureStreamLayer({
artifactUrl={
isCapturePointCloudArtifact(stream.artifact) ? (artifactUrl ?? undefined) : undefined
}
- inline={stream.inline}
+ inline={payload}
packets={stream.availability === 'live' ? packets : []}
/>
)
} else if (layerKey === 'surfaceMesh') {
- content =
+ content =
}
if (!(content && frameMatrix)) return content
return (
@@ -320,6 +341,34 @@ export function CaptureStreamLayer({
)
}
+function useJsonArtifactPayload(url: string | null): unknown {
+ const [error, setError] = useState(null)
+ const [payload, setPayload] = useState(null)
+
+ useEffect(() => {
+ setError(null)
+ setPayload(null)
+ if (!url) return
+ const abort = new AbortController()
+ void fetch(rewriteLoopbackAssetUrl(url), { signal: abort.signal })
+ .then(async (response) => {
+ if (!response.ok) throw new Error(`Could not load ${url}: ${response.status}`)
+ return (await response.json()) as unknown
+ })
+ .then((data) => {
+ if (!abort.signal.aborted) setPayload(data)
+ })
+ .catch((cause: unknown) => {
+ if (abort.signal.aborted) return
+ setError(cause instanceof Error ? cause : new Error(`Could not load ${url}.`))
+ })
+ return () => abort.abort()
+ }, [url])
+
+ if (error) throw error
+ return payload
+}
+
function useResolvedArtifact(
source: CaptureSource,
artifact: CaptureArtifactReference | undefined,
diff --git a/packages/capture-viewer/src/stream-rendering.test.ts b/packages/capture-viewer/src/stream-rendering.test.ts
index 48d00c02b..857777c38 100644
--- a/packages/capture-viewer/src/stream-rendering.test.ts
+++ b/packages/capture-viewer/src/stream-rendering.test.ts
@@ -44,6 +44,28 @@ describe('isCaptureStreamRenderable', () => {
).toBe(true)
})
+ test('renders extracted JSON preview artifacts', () => {
+ for (const [kind, role] of [
+ ['surface-mesh', 'surfaceMesh'],
+ ['point-cloud', 'pointCloud'],
+ ['device-motion', 'deviceMotion'],
+ ] as const) {
+ expect(
+ isCaptureStreamRenderable({
+ id: kind,
+ kind,
+ role,
+ availability: 'ready',
+ artifact: {
+ id: `preview-${kind}`,
+ mediaType: 'application/json',
+ uri: `/api/captures/c/archive/sessions/s/artifacts/preview-${kind}`,
+ },
+ }),
+ ).toBe(true)
+ }
+ })
+
test('renders a valid inline color surface mesh', () => {
expect(
isCaptureStreamRenderable({
diff --git a/packages/capture-viewer/src/stream-rendering.ts b/packages/capture-viewer/src/stream-rendering.ts
index e5e773d8c..2fcfea7ae 100644
--- a/packages/capture-viewer/src/stream-rendering.ts
+++ b/packages/capture-viewer/src/stream-rendering.ts
@@ -10,6 +10,7 @@ import {
const GLB_MEDIA_TYPES = new Set(['model/gltf-binary', 'model/gltf+json'])
const USDZ_MEDIA_TYPES = new Set(['model/vnd.usdz+zip'])
const PLY_MEDIA_TYPES = new Set(['application/ply', 'application/vnd.ply', 'model/ply'])
+const JSON_MEDIA_TYPES = new Set(['application/json'])
export type CaptureModelFormat = 'gltf' | 'usdz'
@@ -24,6 +25,7 @@ export function isCaptureStreamRenderable(
if (layerKey === 'deviceMotion') {
return (
stream.availability === 'live' ||
+ streamHydratesJsonPayload(stream) ||
DeviceMotionTrajectorySchema.safeParse(stream.inline).success
)
}
@@ -31,13 +33,33 @@ export function isCaptureStreamRenderable(
return (
stream.availability === 'live' ||
isCapturePointCloudArtifact(stream.artifact) ||
+ streamHydratesJsonPayload(stream) ||
PointCloudPayloadSchema.safeParse(stream.inline).success
)
}
- if (layerKey === 'surfaceMesh') return SurfaceMeshPayloadSchema.safeParse(stream.inline).success
+ if (layerKey === 'surfaceMesh') {
+ return (
+ streamHydratesJsonPayload(stream) || SurfaceMeshPayloadSchema.safeParse(stream.inline).success
+ )
+ }
return false
}
+const JSON_PAYLOAD_LAYER_KEYS = new Set(['deviceMotion', 'pointCloud', 'surfaceMesh'])
+
+/**
+ * Extracted viewer previews (device motion, point cloud, surface mesh) are
+ * archived as JSON payload artifacts whose content matches the inline shape.
+ * One predicate decides both renderability and runtime hydration, so a
+ * stream can never be declared renderable without a hydration path.
+ */
+export function streamHydratesJsonPayload(stream: CaptureStreamDescriptor): boolean {
+ const artifact = stream.artifact
+ if (!artifact || stream.inline != null) return false
+ if (!JSON_PAYLOAD_LAYER_KEYS.has(captureLayerKey(stream))) return false
+ return JSON_MEDIA_TYPES.has(artifact.mediaType) || hasExtension(artifact.uri, ['.json'])
+}
+
export function isCaptureModelArtifact(artifact: CaptureArtifactReference | undefined): boolean {
return captureModelFormat(artifact) !== null
}