Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 145 additions & 18 deletions crates/solstice-builder/src/paging.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
//! set's parents always come from a later round, so page dependencies can't form a cycle, and
//! they always point at a lower id.

use rustc_hash::FxHashMap;
use solstice_asset::{
gpu::{CLUSTER_CHILDREN_MISSING, NODE_LEAF},
paged::{
Expand All @@ -12,6 +13,77 @@ use solstice_asset::{
},
PackedNode, PackedVertex, SceneAsset, TextureLayer, VirtualMesh,
};
use std::ops::Range;

/// Identical payloads may share an offset within a page. Keys compare every byte, so this
/// preserves quantized positions, attributes, skin weights and triangle order exactly.
#[derive(Default)]
struct PageRecords {
data: ClusterRecords,
vertices: FxHashMap<Vec<u8>, u32>,
triangles: FxHashMap<Vec<u32>, u32>,
clusters: usize,
}

// External cluster tables reserve the largest page's record count for every pool slot.
// Sharing a tiny payload must not pack unbounded metadata into one otherwise empty page.
const MAX_PAGE_CLUSTERS: usize = PAGE_BYTES / std::mem::size_of::<solstice_asset::PackedCluster>();

struct PageCluster {
id: u32,
vertices: Range<usize>,
triangles: Range<usize>,
}

impl PageRecords {
fn append(&mut self, source: &ClusterRecords, ranges: &[PageCluster], out: &mut PagedScene) {
self.clusters += ranges.len();
for range in ranges {
let vertices = &source.vertices[range.vertices.clone()];
let vertex_key: &[u8] = bytemuck::cast_slice(vertices);
let vertex_offset = match self.vertices.get(vertex_key) {
Some(&offset) => offset,
None => {
let offset = self.data.vertices.len() as u32;
self.data.vertices.extend_from_slice(vertices);
self.vertices.insert(vertex_key.to_vec(), offset);
offset
}
};
let triangles = &source.triangles[range.triangles.clone()];
let triangle_offset = match self.triangles.get(triangles) {
Some(&offset) => offset,
None => {
let offset = self.data.triangles.len() as u32;
self.data.triangles.extend_from_slice(triangles);
self.triangles.insert(triangles.to_vec(), offset);
offset
}
};
let cluster = &mut out.clusters[range.id as usize];
cluster.vertex_offset = vertex_offset;
cluster.triangle_offset = triangle_offset;
}
}

fn rollback(&mut self, vertices: usize, triangles: usize, clusters: usize) {
self.clusters = clusters;
self.data.vertices.truncate(vertices);
self.data.triangles.truncate(triangles);
self.vertices
.retain(|_, offset| (*offset as usize) < vertices);
self.triangles
.retain(|_, offset| (*offset as usize) < triangles);
}

fn clear(&mut self) {
self.clusters = 0;
self.data.vertices.clear();
self.data.triangles.clear();
self.vertices.clear();
self.triangles.clear();
}
}

/// One set on its way into a page.
struct Item {
Expand Down Expand Up @@ -164,7 +236,9 @@ pub fn paginate_from<E>(
// Coarsest first. The roots lead, as `ROOT_ROUND` is the largest round.
items.sort_by_key(|item| (std::cmp::Reverse(item.round), item.mesh, item.curve));

let mut records = ClusterRecords::default();
let mut records = PageRecords::default();
let mut incoming = ClusterRecords::default();
let mut ranges = Vec::new();
let mut current = PageInfo::default();
let mut current_round = None;
for item in &items {
Expand All @@ -184,21 +258,20 @@ pub fn paginate_from<E>(
"a sibling set must fit in a page"
);

let raw = PageInfo {
vertex_count: current.vertex_count + extra_vertices,
triangle_count: current.triangle_count + extra_triangles,
..Default::default()
};
if current_round.is_some_and(|round| round != item.round) || raw.raw_bytes() > PAGE_BYTES {
if current_round.is_some_and(|round| round != item.round)
|| records.clusters + range.len() > MAX_PAGE_CLUSTERS
{
finish_page(&mut out, &mut records, &mut current, current_round, sink)?;
}
current_round = Some(item.round);

incoming.vertices.clear();
incoming.triangles.clear();
ranges.clear();
for c in range {
let cluster = *local(c);
let packed = &mut out.clusters[c as usize];
packed.vertex_offset = records.vertices.len() as u32;
packed.triangle_offset = records.triangles.len() as u32;
let vertex_start = incoming.vertices.len();
let triangle_start = incoming.triangles.len();
store.read(
item.mesh,
ClusterRange {
Expand All @@ -207,11 +280,29 @@ pub fn paginate_from<E>(
first_triangle: cluster.triangle_offset as usize,
triangles: cluster.triangle_count() as usize,
},
&mut records,
&mut incoming,
);
ranges.push(PageCluster {
id: c,
vertices: vertex_start..incoming.vertices.len(),
triangles: triangle_start..incoming.triangles.len(),
});
}
current.vertex_count = records.vertices.len() as u32;
current.triangle_count = records.triangles.len() as u32;
// Try the whole sibling set before closing a page: shared payloads can make it fit
// where its unshared size would not. An overflowing set moves intact to the next page.
let previous_clusters = records.clusters;
records.append(&incoming, &ranges, &mut out);
if records.data.vertices.len() * 16 + records.data.triangles.len() * 4 > PAGE_BYTES {
records.rollback(
current.vertex_count as usize,
current.triangle_count as usize,
previous_clusters,
);
finish_page(&mut out, &mut records, &mut current, current_round, sink)?;
records.append(&incoming, &ranges, &mut out);
}
current.vertex_count = records.data.vertices.len() as u32;
current.triangle_count = records.data.triangles.len() as u32;
current.set_count += 1;
out.sets.push(ClusterSet {
page: out.pages.len() as u32,
Expand All @@ -238,14 +329,14 @@ pub fn paginate_from<E>(

fn finish_page<E>(
out: &mut PagedScene,
records: &mut ClusterRecords,
records: &mut PageRecords,
current: &mut PageInfo,
round: Option<u32>,
sink: &mut dyn FnMut(Vec<u8>) -> Result<(), E>,
) -> Result<(), E> {
let mut bytes = Vec::with_capacity(current.raw_bytes());
bytes.extend_from_slice(bytemuck::cast_slice(&records.vertices));
bytes.extend_from_slice(bytemuck::cast_slice(&records.triangles));
bytes.extend_from_slice(bytemuck::cast_slice(&records.data.vertices));
bytes.extend_from_slice(bytemuck::cast_slice(&records.data.triangles));
// The roots lead, so pinned pages are the first of them.
if round == Some(ROOT_ROUND) {
out.pinned_data.extend_from_slice(&bytes);
Expand All @@ -255,8 +346,7 @@ fn finish_page<E>(
current.first_set = out.sets.len() as u32 - current.set_count;
out.pages.push(*current);
*current = PageInfo::default();
records.vertices.clear();
records.triangles.clear();
records.clear();
Ok(())
}

Expand Down Expand Up @@ -363,6 +453,43 @@ mod tests {
}
}

#[test]
fn identical_payloads_share_storage_but_different_attributes_do_not() {
let mut source = scene(&[0, 0]);
let loaded = paginate(&source);
assert_eq!(loaded.scene.pages.len(), 1);
let [first, second] = loaded.scene.clusters.as_slice() else {
panic!("each simple mesh should make one cluster");
};
assert_eq!(first.vertex_offset, second.vertex_offset);
assert_eq!(first.triangle_offset, second.triangle_offset);
assert_eq!(loaded.scene.pages[0].vertex_count, first.vertex_count());
assert_eq!(loaded.scene.pages[0].triangle_count, first.triangle_count());
// One bit of a packed normal must prevent vertex sharing. Topology can still share.
source.meshes[1].vertices[0].data[3] ^= 1;
let loaded = paginate(&source);
let first = &loaded.scene.clusters[0];
let second = &loaded.scene.clusters[1];
assert_ne!(first.vertex_offset, second.vertex_offset);
assert_eq!(first.triangle_offset, second.triangle_offset);
assert_eq!(loaded.scene.pages[0].vertex_count, first.vertex_count() * 2);
let encoded = solstice_asset::file::encode(&loaded);
let decoded = solstice_asset::file::decode(&encoded).unwrap();
assert_eq!(decoded.scene.clusters, loaded.scene.clusters);
assert_eq!(decoded.scene.pinned_data, loaded.scene.pinned_data);
}

#[test]
fn sharing_payloads_does_not_make_a_pages_cluster_table_unbounded() {
let mut source = scene(&[0]);
source
.meshes
.resize(MAX_PAGE_CLUSTERS + 1, source.meshes[0].clone());
let loaded = paginate(&source);
assert_eq!(loaded.scene.pages.len(), 2);
assert!(loaded.scene.max_page_clusters() as usize <= MAX_PAGE_CLUSTERS);
}

#[test]
fn a_skinned_cluster_keeps_its_skins_behind_its_vertices_in_its_page() {
use solstice_asset::PackedSkin;
Expand Down
37 changes: 26 additions & 11 deletions crates/solstice-import/src/streamed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,17 +155,32 @@ fn free_bytes(path: &Path) -> Option<u64> {
let absolute = std::path::absolute(path).ok()?;
#[cfg(windows)]
{
let root = absolute.components().next()?;
let drive = root.as_os_str().to_str()?.trim_end_matches(':').to_owned();
let output = std::process::Command::new("powershell")
.args([
"-NoProfile",
"-Command",
&format!("(Get-PSDrive -Name '{drive}').Free"),
])
.output()
.ok()?;
String::from_utf8_lossy(&output.stdout).trim().parse().ok()
use std::os::windows::ffi::OsStrExt;
#[link(name = "kernel32")]
unsafe extern "system" {
fn GetDiskFreeSpaceExW(
directory: *const u16,
available: *mut u64,
total: *mut u64,
free: *mut u64,
) -> i32;
}
// The output may not exist yet. Query its nearest existing directory, which also
// handles UNC paths and volume mount points without parsing a drive letter.
let directory = absolute.ancestors().find(|p| p.is_dir())?;
let directory: Vec<u16> = directory.as_os_str().encode_wide().chain([0]).collect();
let mut available = 0;
// SAFETY: directory is a terminated UTF-16 string and available is a writable u64.
// The optional total/free outputs are null, as allowed by the Windows API.
let ok = unsafe {
GetDiskFreeSpaceExW(
directory.as_ptr(),
&mut available,
std::ptr::null_mut(),
std::ptr::null_mut(),
)
};
(ok != 0).then_some(available)
}
#[cfg(not(windows))]
{
Expand Down
116 changes: 116 additions & 0 deletions crates/solstice-render/examples/cpu_stress.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
//! Repeatable CPU scaling measurements, with no quality settings or GPU involved.
//! Run: cargo run --release -p solstice-render --example cpu_stress

use bytemuck::Zeroable;
use glam::{Mat4, Vec3};
use solstice_render::{motion::InstanceTable, scene::GpuInstance, streaming::residency::Residency};
use std::{hint::black_box, time::Instant};

fn table(count: u32) -> InstanceTable {
let mut table = InstanceTable::new(
vec![GpuInstance::zeroed(); count as usize],
(0..count).map(Some).collect(),
vec![(Vec3::ZERO, 1.0); count as usize],
);
// Initialise the transforms through the same public API used by the renderer.
for i in 0..count {
table.set_transform(i, Mat4::IDENTITY, true);
}
table.advance();
table
}

fn report(case: &str, count: u32, mut samples: Vec<f64>, checksum: u64) {
samples.sort_by(f64::total_cmp);
println!(
"{case},{count},{:.6},{:.6},{checksum}",
samples[samples.len() / 2],
samples[samples.len() - 1]
);
}

fn hash(bytes: &[u8]) -> u64 {
bytes.iter().fold(0xcbf29ce484222325, |h, &b| {
(h ^ u64::from(b)).wrapping_mul(0x100000001b3)
})
}

fn main() {
println!("case,count,median_ms,max_ms,checksum");
for count in [1_024, 8_192, 32_768] {
let mut table = table(count);
let mut reads = Vec::new();
let mut updates = Vec::new();
for round in 0..7 {
let start = Instant::now();
for i in 0..count {
// A common game loop: read an object's current transform, then move it.
let mut transform = table.transform(black_box(i)).unwrap();
transform.w_axis.x += 1.0;
table.set_transform(i, transform, false);
}
if round > 0 {
reads.push(start.elapsed().as_secs_f64() * 1_000.0);
}
let start = Instant::now();
black_box(table.advance());
if round > 0 {
updates.push(start.elapsed().as_secs_f64() * 1_000.0);
}
}
for i in 0..count {
assert_eq!(table.transform(i).unwrap().w_axis.x, 7.0);
}
let checksum = hash(bytemuck::cast_slice(table.gpu_data(0..count as usize)));
report("read_move", count, reads, checksum);
report("advance", count, updates, checksum);
}
for count in [1_024, 8_192, 32_768] {
let mut table = table(count);
let mut samples = Vec::new();
for round in 0..7 {
let start = Instant::now();
// Multiple simulation ticks before drawing, submitted in reverse instance order.
for tick in 0..8 {
for i in (0..count).rev() {
table.set_transform(
i,
Mat4::from_translation(Vec3::new(i as f32, tick as f32, 0.0)),
tick == 7,
);
}
}
black_box(table.advance());
if round > 0 {
samples.push(start.elapsed().as_secs_f64() * 1_000.0);
}
}
let checksum = hash(bytemuck::cast_slice(table.gpu_data(0..count as usize)));
report("eight_updates", count, samples, checksum);
}
for count in [4_096u32, 65_536, 1_048_576] {
let mut dependencies = vec![vec![0]; count as usize];
dependencies[0].clear();
let mut residency = Residency::new(dependencies, 1, 512);
let requests: Vec<u32> = (0..count)
.map(|i| {
if i % 8 == 0 {
2 + i.wrapping_mul(2654435761) % 1024
} else {
0
}
})
.collect();
let mut samples = Vec::new();
let mut checksum = 0;
for frame in 0..21 {
let start = Instant::now();
let fetched = residency.plan(frame, black_box(&requests), 32);
if frame > 0 {
samples.push(start.elapsed().as_secs_f64() * 1_000.0);
}
checksum ^= hash(bytemuck::cast_slice(&fetched));
}
report("stream_plan", count, samples, checksum);
}
}
Loading
Loading