Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/viewer/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ mod native {
self.running -= 1;
match data {
Some(data) => {
renderer.deliver(id, data);
renderer.deliver_fetched(id, data);
}
None => renderer.request_failed(id),
}
Expand Down
4 changes: 2 additions & 2 deletions apps/viewer/src/web.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ impl WebViewer {

/// Takes a page of such a scene, inflated, with its cluster records.
pub fn deliver_page(&mut self, id: u32, bytes: Vec<u8>, clusters: Vec<u8>) -> bool {
self.renderer.deliver_page(id, bytes, clusters)
self.renderer.deliver_fetched_page(id, bytes, clusters)
}

/// Call when all of such a scene's tables have been handed over.
Expand Down Expand Up @@ -273,7 +273,7 @@ impl WebViewer {

/// Takes the bytes of a request, inflated if it needed that.
pub fn deliver(&mut self, id: u32, bytes: Vec<u8>) -> bool {
self.renderer.deliver(id, bytes)
self.renderer.deliver_fetched(id, bytes)
}

/// Reports that a request could not be fetched.
Expand Down
136 changes: 136 additions & 0 deletions crates/solstice-asset/examples/page_stats.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
//! Measures how a baked scene's pages use their room: `page_stats FILE [SAMPLE_EVERY]`.
//!
//! It reads the manifest for the sizes of every page, and unpacks every Nth page to see what
//! its clusters hold. `docs/streaming.md` has what this found for one large scene, under
//! "Tighter pages".

use solstice_asset::file::{
decode_header, decode_manifest, inflate, page_range, unpack_page, HEADER_BYTES,
};
use solstice_asset::gpu::{triangle_at, triangle_words, VERTEX_WORDS};
use solstice_asset::paged::PAGE_BYTES;
use solstice_asset::PackedVertex;
use std::fs::File;
use std::io::{Read, Seek, SeekFrom};

const MIB: f64 = 1_048_576.0;

fn read_at(file: &mut File, start: u64, len: usize) -> Vec<u8> {
let mut bytes = vec![0; len];
file.seek(SeekFrom::Start(start)).expect("seek");
file.read_exact(&mut bytes).expect("read");
bytes
}

fn main() {
let mut args = std::env::args().skip(1);
let path = args.next().expect("page_stats FILE [SAMPLE_EVERY]");
let every: usize = args.next().map_or(16, |n| n.parse().expect("a number"));
let mut file = File::open(&path).expect("open");
let header = decode_header(&read_at(&mut file, 0, HEADER_BYTES)).expect("header");
let manifest = read_at(
&mut file,
header.manifest_offset,
header.manifest_bytes as usize,
);
let manifest = inflate(&manifest).expect("inflate");
let scene = decode_manifest(&manifest, header.manifest_offset + header.manifest_bytes)
.expect("manifest");

let pages = scene.pages.len();
let pinned = scene.pinned_pages as usize;
let streamed = &scene.pages[pinned..];
let raw: u64 = scene.pages.iter().map(|p| p.raw_bytes() as u64).sum();
let stored: u64 = scene.pages.iter().map(|p| u64::from(p.stored_bytes)).sum();
let vertex_bytes: u64 = scene.pages.iter().map(|p| p.vertex_bytes() as u64).sum();
let streamed_raw: u64 = streamed.iter().map(|p| p.raw_bytes() as u64).sum();
let mut fill = [0u32; 10];
for page in streamed {
fill[(page.raw_bytes() * 10 / (PAGE_BYTES + 1)).min(9)] += 1;
}
let clusters = scene.cluster_count();
println!("clusters {clusters}");
println!("sets {}", scene.sets.len());
println!("pages {pages} ({pinned} pinned)");
println!("page bytes, raw {:.1} MiB", raw as f64 / MIB);
println!("page bytes, stored {:.1} MiB", stored as f64 / MIB);
println!(
"vertices {:.1} MiB ({:.0}%)",
vertex_bytes as f64 / MIB,
vertex_bytes as f64 * 100.0 / raw as f64
);
println!(
"triangles {:.1} MiB",
(raw - vertex_bytes) as f64 / MIB
);
println!(
"bytes a cluster {:.0} raw, {:.0} of pool with slots as full as they are",
raw as f64 / clusters as f64,
(pages * PAGE_BYTES) as f64 / clusters as f64
);
println!(
"slots full to {:.1}% of {} KiB",
streamed_raw as f64 * 100.0 / (streamed.len() * PAGE_BYTES) as f64,
PAGE_BYTES / 1024
);
println!("pages by tenths full {fill:?}");
println!(
"clusters a page {:.1}",
(pinned..pages)
.map(|p| u64::from(scene.page_cluster_count(p as u32)))
.sum::<u64>() as f64
/ streamed.len() as f64
);

// Sampled pages: every cluster is read back, which checks the page against its tables.
let (mut sampled, mut vertices, mut triangles) = (0u64, 0u64, 0u64);
for page in (pinned..pages).step_by(every) {
let (start, end) = page_range(&scene, page as u32);
let packed = read_at(&mut file, start, (end - start) as usize);
let data = unpack_page(&scene, page as u32, &packed).expect("page");
let info = scene.pages[page];
let (vertex_part, triangle_part) = data.split_at(info.vertex_bytes());
let records: &[PackedVertex] = bytemuck::cast_slice(vertex_part);
let words: Vec<u32> = triangle_part
.chunks_exact(4)
.map(|w| u32::from_le_bytes([w[0], w[1], w[2], w[3]]))
.collect();
for set in &scene.sets[info.first_set as usize..(info.first_set + info.set_count) as usize]
{
for c in set.first_cluster..set.first_cluster + set.cluster_count {
let cluster = scene.clusters[c as usize];
let count = cluster.vertex_count();
let first = cluster.vertex_offset / VERTEX_WORDS;
assert!(
(first + count) as usize <= records.len(),
"vertices out of the page"
);
let triangle_count = cluster.triangle_count();
let own = &words[cluster.triangle_offset as usize..]
[..triangle_words(triangle_count) as usize];
for index in 0..triangle_count {
let packed = triangle_at(own, index);
assert!(
[packed & 0xff, (packed >> 8) & 0xff, packed >> 16]
.iter()
.all(|&corner| corner < count),
"a triangle names a vertex its cluster doesn't have"
);
}
sampled += 1;
vertices += u64::from(count);
triangles += u64::from(triangle_count);
}
}
}
println!("--- every {every}th streamed page, each cluster read back ---");
println!("clusters {sampled}");
println!(
"vertices a cluster {:.1}",
vertices as f64 / sampled as f64
);
println!(
"triangles a cluster {:.1}",
triangles as f64 / sampled as f64
);
}
125 changes: 113 additions & 12 deletions crates/solstice-asset/src/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use glam::{Mat4, Quat, Vec3};
use std::fmt;

const MAGIC: [u8; 8] = *b"SOLSTICE";
const VERSION: u32 = 4;
const VERSION: u32 = 5;
/// How hard to compress when baking. Unpacking takes the same time at any level.
const DEFLATE_LEVEL: u8 = 9;
/// The optional section holding the scene's [`Sea`]: its level, then its ocean's text.
Expand Down Expand Up @@ -174,7 +174,7 @@ pub fn encode(loaded: &LoadedScene) -> Vec<u8> {
let mut scene = loaded.scene.clone();
let mut data = Vec::new();
for (info, raw) in scene.pages[pinned..].iter_mut().zip(&loaded.pages) {
let packed = miniz_oxide::deflate::compress_to_vec(raw, DEFLATE_LEVEL);
let packed = deflate_page(raw, info);
info.offset = data.len() as u64;
info.stored_bytes = packed.len() as u32;
data.extend_from_slice(&packed);
Expand Down Expand Up @@ -213,9 +213,71 @@ pub fn cluster_bytes(clusters: &[PackedCluster]) -> &[u8] {
bytemuck::cast_slice(clusters)
}

/// Deflates a page as [`encode`] does, for writers that lay out the data area themselves.
pub fn deflate_page(raw: &[u8]) -> Vec<u8> {
miniz_oxide::deflate::compress_to_vec(raw, DEFLATE_LEVEL)
/// Packs a page as [`encode`] does, for writers that lay out the data area themselves:
/// filtered, see [`filter_page`], and deflated.
pub fn deflate_page(raw: &[u8], info: &PageInfo) -> Vec<u8> {
miniz_oxide::deflate::compress_to_vec(&filter_page(raw, info), DEFLATE_LEVEL)
}

/// The `u16` fields of a vertex record.
const VERTEX_FIELDS: usize = std::mem::size_of::<crate::gpu::PackedVertex>() / 2;

/// Rewrites a page's vertices so that they deflate smaller, to the same size and with nothing
/// lost: [`unfilter_page`] undoes it. The pages of a large garden scene stored a fifth smaller
/// for it, see `docs/streaming.md`.
///
/// Each field of a vertex becomes its difference from the same field of the vertex before,
/// folded so that a small step either way is a small number, and each byte of each field goes
/// in a run of its own. Neighbouring vertices are close together, so the high bytes are mostly
/// nought and the low ones small. Triangles are left alone: they pack no better for it.
pub fn filter_page(raw: &[u8], info: &PageInfo) -> Vec<u8> {
let count = info.vertex_count as usize;
let vertex_bytes = info.vertex_bytes().min(raw.len());
let mut out = raw.to_vec();
if vertex_bytes != count * 2 * VERTEX_FIELDS {
return out;
}
let mut before = [0u16; VERTEX_FIELDS];
for (index, vertex) in raw[..vertex_bytes]
.chunks_exact(2 * VERTEX_FIELDS)
.enumerate()
{
for field in 0..VERTEX_FIELDS {
let value = u16::from_le_bytes([vertex[field * 2], vertex[field * 2 + 1]]);
let step = value.wrapping_sub(before[field]);
let folded = (step << 1) ^ (((step as i16) >> 15) as u16);
out[field * 2 * count + index] = folded as u8;
out[(field * 2 + 1) * count + index] = (folded >> 8) as u8;
before[field] = value;
}
}
out
}

/// Undoes [`filter_page`]: the page as the GPU reads it, from the page as a file stores it once
/// inflated.
pub fn unfilter_page(stored: &[u8], info: &PageInfo) -> Vec<u8> {
let count = info.vertex_count as usize;
let vertex_bytes = info.vertex_bytes().min(stored.len());
let mut out = stored.to_vec();
if vertex_bytes != count * 2 * VERTEX_FIELDS {
return out;
}
let mut before = [0u16; VERTEX_FIELDS];
for (index, vertex) in out[..vertex_bytes]
.chunks_exact_mut(2 * VERTEX_FIELDS)
.enumerate()
{
for field in 0..VERTEX_FIELDS {
let folded = u16::from(stored[field * 2 * count + index])
| u16::from(stored[(field * 2 + 1) * count + index]) << 8;
let step = (folded >> 1) ^ (folded & 1).wrapping_neg();
let value = before[field].wrapping_add(step);
vertex[field * 2..field * 2 + 2].copy_from_slice(&value.to_le_bytes());
before[field] = value;
}
}
out
}

/// The start of a file: its header and manifest, for a writer that has placed every page and
Expand Down Expand Up @@ -1145,11 +1207,12 @@ pub fn inflate_limited(packed: &[u8], limit: u64) -> Result<Vec<u8>, DecodeError

/// Unpacks a page and checks that it is the size the tables say.
pub fn unpack_page(scene: &PagedScene, page: u32, packed: &[u8]) -> Result<Vec<u8>, DecodeError> {
let raw = inflate(packed)?;
if raw.len() != scene.pages[page as usize].raw_bytes() {
let info = &scene.pages[page as usize];
let stored = inflate(packed)?;
if stored.len() != info.raw_bytes() {
return Err(DecodeError::BadPages);
}
Ok(raw)
Ok(unfilter_page(&stored, info))
}

/// Checks everything the renderer would otherwise index blindly while installing pages. A bad
Expand Down Expand Up @@ -1282,15 +1345,15 @@ mod tests {
fn sample_scene() -> LoadedScene {
let page_bytes = |vertex: u8| {
let vertices = [PackedVertex {
data: [u16::from(vertex); 8],
data: [u16::from(vertex); 6],
}; 3];
let mut bytes: Vec<u8> = bytemuck::cast_slice(&vertices).to_vec();
bytes.extend_from_slice(bytemuck::bytes_of(&pack_triangle(0, 1, 2)));
bytes
};
let info = PageInfo {
vertex_count: 3,
triangle_count: 1,
triangle_words: 1,
set_count: 1,
..Default::default()
};
Expand Down Expand Up @@ -1479,10 +1542,12 @@ mod tests {
}

#[test]
fn a_version_4_file_with_neither_section_still_loads() {
fn a_file_with_neither_section_still_loads() {
let loaded = sample_scene();
let bytes = encode(&loaded);
assert_eq!(bytes[8..12], 4u32.to_le_bytes(), "the version hasn't moved");
// Version 5 is version 4 with pages laid out more tightly: optional sections came and
// went without moving it.
assert_eq!(bytes[8..12], 5u32.to_le_bytes(), "the version hasn't moved");
let manifest = encode_manifest(&loaded.scene);
assert!(!manifest
.windows(4)
Expand Down Expand Up @@ -2061,6 +2126,42 @@ mod tests {
assert!(decode_manifest(&manifest, 0).is_err());
}

#[test]
fn filtering_a_page_loses_nothing_and_packs_smaller() {
// A strip of vertices a step apart, as neighbours in a mesh are, then some triangles.
let vertices: Vec<PackedVertex> = (0..600u16)
.map(|i| PackedVertex {
data: [
1000 + i * 3,
40_000 - i * 5,
i.wrapping_mul(977) % 64,
i.wrapping_mul(31_337),
i * 100,
65_535 - i * 7,
],
})
.collect();
let mut raw: Vec<u8> = bytemuck::cast_slice(&vertices).to_vec();
raw.extend((0..300u32).flat_map(|i| i.wrapping_mul(0x0101_0101).to_le_bytes()));
let info = PageInfo {
vertex_count: vertices.len() as u32,
triangle_words: 300,
..Default::default()
};
assert_eq!(raw.len(), info.raw_bytes());
let stored = filter_page(&raw, &info);
assert_eq!(stored.len(), raw.len());
assert_ne!(stored, raw);
assert_eq!(unfilter_page(&stored, &info), raw);
assert!(
deflate_page(&raw, &info).len() * 2
< miniz_oxide::deflate::compress_to_vec(&raw, DEFLATE_LEVEL).len()
);
// A page the tables are wrong about comes back as it went in, to be refused for its size.
let short = &raw[..100];
assert_eq!(unfilter_page(&filter_page(short, &info), &info), short);
}

#[test]
fn a_page_and_a_mip_can_be_read_without_the_rest() {
let loaded = sample_scene();
Expand Down
Loading
Loading