From c9afb53f7aa621415f2cb7439b89658024a96297 Mon Sep 17 00:00:00 2001 From: SteveYu000 <111431635+SteveYu000@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:08:08 +0800 Subject: [PATCH 1/2] Guard AV1 sample descriptions in MP4 Check the profile, first operating-point level/tier and colour-sampling fields represented by av1C against every emitted Sequence Header. Reuse FFmpeg's extract_extradata BSF; let the first actual output sequence supply the MP4 description, including fully re-encoded selections. Keep compatible smart-rendered middles. If MP4's single stsd entry cannot describe a changed configuration, close the first attempt and retry the selected video fully re-encoded, once only. Explain in the log that this supersedes the initial copy plan. Audio and subtitle modes are unchanged. Only the mp4 output muxer enables the guard. MKV/WebM keep their existing logic; their sequence-header handling policy is left to the maintainer to decide separately. Do not include SAR, source-clock/VFR or M2TS changes. Add six guard unit tests and an 18-export suite registered in CI. On identical inputs, the new suite improves from 15/18 to 18/18, and the historical MP4 matrix from 38/55 to 45/55 without a new failure. Workspace tests: 576 passed, four known Windows baseline failures filtered; clippy workspace/all-targets exits 0 with the existing warnings. --- docs/technical/av1-mp4.md | 65 ++++++ docs/technical/building.md | 1 + rust/crates/core/src/av1_mp4.rs | 358 ++++++++++++++++++++++++++++++ rust/crates/core/src/bitstream.rs | 5 + rust/crates/core/src/cut.rs | 58 ++++- rust/crates/core/src/lib.rs | 1 + tests/av1_mp4_tests.py | 255 +++++++++++++++++++++ tests/ci.sh | 2 +- tests/run_av1_mp4_tests.sh | 4 + 9 files changed, 746 insertions(+), 3 deletions(-) create mode 100644 docs/technical/av1-mp4.md create mode 100644 rust/crates/core/src/av1_mp4.rs create mode 100644 tests/av1_mp4_tests.py create mode 100755 tests/run_av1_mp4_tests.sh diff --git a/docs/technical/av1-mp4.md b/docs/technical/av1-mp4.md new file mode 100644 index 0000000..8781c88 --- /dev/null +++ b/docs/technical/av1-mp4.md @@ -0,0 +1,65 @@ +# AV1 configuration in MP4 + +An AV1 boundary encoder can write different Sequence Header OBUs from the +recording's. Decoding that seam successfully does not prove that MP4's +`av1C` describes every sample which references it. + +The writer applies this check **only when the selected output muxer is +`mp4`**. FFmpeg's `extract_extradata` filter supplies the Sequence Headers; +the guard reads only the fields copied into `av1C`: profile, the first +operating point's level and tier, bit depth, monochrome, chroma subsampling +and chroma sample position. All headers in an extracted packet are checked. +Different Sequence Header bytes with the same fields remain permitted; the +guard does not require byte-identical sequences and does not rewrite frames. + +The first sequence actually written supplies the track configuration. Where +that sequence is present only in the input extradata, it is attached to the +first output packet. This also fixes a selection wholly re-encoded with a +configuration different from the source's, which needs no second attempt. + +FFmpeg 7.1's MOV/MP4 writer emits one `stsd` entry. A `NEW_EXTRADATA` side +datum updates its configuration, but cannot assign a second sample entry to +the changed samples. If the fields change during an MP4 export, the failed +context closes and the selection is retried with all video ranges +re-encoded. Audio and subtitle modes retain their existing settings. The +log identifies the initial copy figures as a discarded plan and states that +the retry copies no video. There is at most one retry; a further change is +an error. + +This fallback adds an attempt, encoding time and generation loss. It does +not implement multiple MP4 sample descriptions. The analysis/GUI plan is +still the initial estimate; progress can restart on the retry. Source-clock, +VFR and SAR changes belong to separate contributions. + +**Matroska and WebM keep their existing logic.** They do not enter this +guard or its full-video retry. Whether a changed AV1 sequence in Matroska +should cause a refusal, a warning, an option or a full re-encode is a policy +decision left to the maintainer, outside this change. + +## Reproduce + +```bash +cd rust && cargo build --release --locked -p smartcut-cli && cd .. +bash tests/run_av1_mp4_tests.sh +# Or: bash tests/ci.sh av1_mp4 +``` + +Python 3.11+, FFmpeg/ffprobe with libaom-av1, librav1e and libsvtav1 are +required. The suite generates three 240-picture CFR inputs (8/10-bit +libaom and 8-bit rav1e), then checks 15 MP4 exports and three Matroska +controls. MP4 checks include strict decoding, frame counts, display PTS, +DTS, video duration, `av1C` against the independent CBS `trace_headers` +reader, and packet/pixel equality inside the declared copied intervals. +The Matroska controls assert that no configuration retry was added; they +do not certify Matroska sequence-header conformance. + +Use `--cut`, `--ffmpeg`, `--ffprobe`, `--work-dir` and `--fixtures` for +explicit paths and paired runs on identical inputs. Commands, hashes, +results and output logs remain under the printed work directory. Media +and built binaries are not committed. + +References: + +- [AV1 in ISO BMFF v1.3.0, sections 2.2-2.4](https://aomediacodec.github.io/av1-isobmff/v1.3.0.html) +- [FFmpeg 7.1.1 single-entry stsd writer and packet extradata handling](https://github.com/FFmpeg/FFmpeg/blob/n7.1.1/libavformat/movenc.c) +- [FFmpeg 7.1.1 extract_extradata filter](https://github.com/FFmpeg/FFmpeg/blob/n7.1.1/libavcodec/bsf/extract_extradata.c) diff --git a/docs/technical/building.md b/docs/technical/building.md index 2543cf6..830f1b3 100644 --- a/docs/technical/building.md +++ b/docs/technical/building.md @@ -85,6 +85,7 @@ bash tests/run_dvd_tests.sh # a DVD-Video disc, as a folder and as an bash tests/run_bd_audio_tests.sh # the sound a disc carries, written out 39 bash tests/run_vc1_tests.sh # the VC-1 encoder, put through a decoder 4 bash tests/run_vp9_av1_tests.sh # VP9 and AV1: cut, and still the source's bytes 5 +bash tests/run_av1_mp4_tests.sh # AV1 MP4 sample descriptions and MKV policy 18 bash tests/run_vfr_tests.sh # variable rate, both ways round 8 bash tests/run_demux_tests.sh # what a reader loses by reading what it does not 9 bash tests/run_transrate_tests.sh # fitting a disc: the identity, the size, the cost 15 diff --git a/rust/crates/core/src/av1_mp4.rs b/rust/crates/core/src/av1_mp4.rs new file mode 100644 index 0000000..c1ad682 --- /dev/null +++ b/rust/crates/core/src/av1_mp4.rs @@ -0,0 +1,358 @@ +//! Check the AV1 fields shared by every sample in one MP4 description. +//! FFmpeg extracts the header; only its public container fields are read here. +use anyhow::{bail, Context, Result}; +use ff::packet::{Mut, Ref}; +use ffmpeg_next as ff; +use std::ffi::{c_char, c_int, c_void}; + +// ffmpeg-sys-next 7.1 omits bsf.h. These are the public FFmpeg 7.1 +// declarations, not private codec/muxer internals. +#[repr(C)] +struct Bsf { + class: *const c_void, + filter: *const c_void, + private: *mut c_void, + par_in: *mut ff::ffi::AVCodecParameters, + par_out: *mut ff::ffi::AVCodecParameters, + time_base_in: ff::ffi::AVRational, + time_base_out: ff::ffi::AVRational, +} +extern "C" { + fn av_bsf_get_by_name(name: *const c_char) -> *const c_void; + fn av_bsf_alloc(filter: *const c_void, context: *mut *mut Bsf) -> c_int; + fn av_bsf_init(context: *mut Bsf) -> c_int; + fn av_bsf_free(context: *mut *mut Bsf); + fn av_bsf_send_packet(context: *mut Bsf, packet: *mut ff::ffi::AVPacket) -> c_int; + fn av_bsf_receive_packet(context: *mut Bsf, packet: *mut ff::ffi::AVPacket) -> c_int; +} + +#[derive(Debug)] +pub(crate) struct Changed; +impl std::fmt::Display for Changed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("AV1 configurations cannot share one MP4 sample description") + } +} +impl std::error::Error for Changed {} + +pub(crate) struct Guard { + filter: *mut Bsf, + fields: Option<[u8; 3]>, + initial: Vec, +} +impl Drop for Guard { + fn drop(&mut self) { + unsafe { av_bsf_free(&mut self.filter) } + } +} +fn check(code: i32) -> Result<()> { + if code < 0 { + return Err(ff::Error::from(code).into()); + } + Ok(()) +} +impl Guard { + pub(crate) fn new(params: &ff::codec::Parameters) -> Result { + let mut result = Self { + filter: std::ptr::null_mut(), + fields: None, + initial: Vec::new(), + }; + unsafe { + let p = params.as_ptr(); + if !(*p).extradata.is_null() && (*p).extradata_size > 0 { + let data = std::slice::from_raw_parts((*p).extradata, (*p).extradata_size as usize); + result.initial = if data.first() == Some(&0x81) { + data.get(4..).unwrap_or_default().to_vec() + } else { + data.to_vec() + }; + } + let filter = av_bsf_get_by_name(c"extract_extradata".as_ptr()); + if filter.is_null() { + bail!("FFmpeg has no extract_extradata filter"); + } + check(av_bsf_alloc(filter, &mut result.filter))?; + check(ff::ffi::avcodec_parameters_copy( + (*result.filter).par_in, + params.as_ptr(), + ))?; + // Writer assigns final timestamps after this data-only filter. + (*result.filter).time_base_in = ff::ffi::AVRational { num: 1, den: 1 }; + check(av_bsf_init(result.filter))?; + } + Ok(result) + } + + pub(crate) fn packet(&mut self, packet: &mut ff::Packet) -> Result<()> { + unsafe { + check(av_bsf_send_packet(self.filter, packet.as_mut_ptr()))?; + check(av_bsf_receive_packet(self.filter, packet.as_mut_ptr()))?; + let mut size = 0; + let mut extra = ff::ffi::av_packet_get_side_data( + packet.as_ptr(), + ff::ffi::AVPacketSideDataType::AV_PKT_DATA_NEW_EXTRADATA, + &mut size, + ); + if extra.is_null() { + if self.fields.is_some() { + return Ok(()); + } + if self.initial.is_empty() { + bail!("first AV1 picture has no sequence configuration"); + } + size = self.initial.len(); + extra = ff::ffi::av_packet_new_side_data( + packet.as_mut_ptr(), + ff::ffi::AVPacketSideDataType::AV_PKT_DATA_NEW_EXTRADATA, + size, + ); + if extra.is_null() { + bail!("cannot allocate initial AV1 configuration"); + } + std::ptr::copy_nonoverlapping(self.initial.as_ptr(), extra, size); + } + let data = std::slice::from_raw_parts(extra, size); + for sequence in sequences(data).context("invalid extracted AV1 sequence header")? { + let fields = fields(sequence).context("truncated AV1 sequence header")?; + if self.fields.is_some_and(|old| old != fields) { + return Err(Changed.into()); + } + self.fields = Some(fields); + } + } + Ok(()) + } +} + +fn sequences(mut data: &[u8]) -> Option> { + let mut headers = Vec::new(); + while let Some((&head, rest)) = data.split_first() { + if head & 0x81 != 0 { + return None; + } + let mut at = usize::from(head & 4 != 0); + let mut size = 0usize; + if head & 2 != 0 { + for n in 0..8 { + let byte = *rest.get(at)?; + at += 1; + size = size.checked_add(usize::from(byte & 127).checked_shl(n * 7)?)?; + if byte & 128 == 0 { + break; + } + if n == 7 { + return None; + } + } + } else { + size = rest.len().checked_sub(at)?; + } + let end = at.checked_add(size)?; + let body = rest.get(at..end)?; + if (head >> 3) & 15 == 1 { + headers.push(body); + } + data = rest.get(end..)?; + } + (!headers.is_empty()).then_some(headers) +} + +/// AV1 5.5.1/5.5.2, only as far as the av1C colour/subsampling fields. +/// The shared bit reader must not unescape AV1 as though it were a NAL. +fn fields(data: &[u8]) -> Option<[u8; 3]> { + let mut b = crate::bitstream::Bits::raw(data); + let profile = b.u(3)?; + b.u(1)?; + let reduced = b.u(1)? != 0; + let (mut level, mut tier) = (0, 0); + if reduced { + level = b.u(5)?; + } else { + let mut buffer_bits = 0; + let mut model = false; + if b.u(1)? != 0 { + b.skip(64)?; + if b.u(1)? != 0 { + let mut zeros = 0; + while b.u(1)? == 0 { + zeros += 1; + if zeros == 32 { + break; + } + } + if zeros < 32 { + b.skip(zeros)?; + } + } + model = b.u(1)? != 0; + if model { + buffer_bits = b.u(5)? as usize + 1; + b.skip(42)?; + } + } + let delay = b.u(1)? != 0; + let count = b.u(5)? + 1; + for i in 0..count { + b.skip(12)?; + let l = b.u(5)?; + let t = if l > 7 { b.u(1)? } else { 0 }; + if i == 0 { + level = l; + tier = t; + } + if model && b.u(1)? != 0 { + b.skip(buffer_bits * 2 + 1)?; + } + if delay && b.u(1)? != 0 { + b.skip(4)?; + } + } + } + let width = b.u(4)? as usize + 1; + let height = b.u(4)? as usize + 1; + b.skip(width + height)?; + if !reduced && b.u(1)? != 0 { + b.skip(7)?; + } + b.skip(3)?; + if !reduced { + b.skip(4)?; + let order_hint = b.u(1)? != 0; + if order_hint { + b.skip(2)?; + } + let screen = if b.u(1)? != 0 { 2 } else { b.u(1)? }; + if screen != 0 && b.u(1)? == 0 { + b.u(1)?; + } + if order_hint { + b.skip(3)?; + } + } + b.skip(3)?; + let high = b.u(1)?; + let twelve = if profile == 2 && high != 0 { + b.u(1)? + } else { + 0 + }; + let mono = if profile == 1 { 0 } else { b.u(1)? }; + let colour = if b.u(1)? != 0 { + (b.u(8)?, b.u(8)?, b.u(8)?) + } else { + (2, 2, 2) + }; + let (sx, sy, position) = if mono != 0 { + b.u(1)?; + (1, 1, 0) + } else if colour == (1, 13, 0) { + (0, 0, 0) + } else { + b.u(1)?; + let (x, y) = match profile { + 0 => (1, 1), + 1 => (0, 0), + _ if twelve != 0 => { + let x = b.u(1)?; + (x, if x != 0 { b.u(1)? } else { 0 }) + } + _ => (1, 0), + }; + let p = if x != 0 && y != 0 { b.u(2)? } else { 0 }; + (x, y, p) + }; + Some([ + 0x81, + ((profile << 5) | level) as u8, + ((tier << 7) | (high << 6) | (twelve << 5) | (mono << 4) | (sx << 3) | (sy << 2) | position) + as u8, + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn container_fields_follow_real_eight_and_ten_bit_headers() { + // Headers from tiny libaom testsrc2 fixtures. 00 00 03 is literal + // AV1 data; treating it as H.26x emulation prevention corrupts it. + let ten = [0x0a, 0x0a, 0, 0, 0, 3, 0xb5, 0xfe, 0xf3, 0x6b, 0xe5, 1]; + let eight = [ + 0x0a, 0x0d, 0, 0, 0, 3, 0xb5, 0xfe, 0xf3, 0x6b, 0xe4, 0x40, 0x80, 0x80, 0x61, + ]; + assert_eq!(fields(sequences(&ten).unwrap()[0]), Some([0x81, 0, 0x4c])); + assert_eq!(fields(sequences(&eight).unwrap()[0]), Some([0x81, 0, 0x0c])); + } + + #[test] + fn all_headers_in_one_packet_are_checked() { + let one = [0x0a, 0x0a, 0, 0, 0, 3, 0xb5, 0xfe, 0xf3, 0x6b, 0xe5, 1]; + let two = [one.as_slice(), one.as_slice()].concat(); + assert_eq!(sequences(&two).unwrap().len(), 2); + assert!(sequences(&[0x12, 0]).is_none()); // no Sequence Header + } + + #[test] + fn malformed_obu_lengths_are_rejected() { + assert!(sequences(&[0x0a, 0x80]).is_none()); + assert!(sequences(&[0x0a, 255, 255, 255, 255, 255, 255, 255, 255]).is_none()); + assert!(fields(&[]).is_none()); + assert!(fields(&[0]).is_none()); + // AV1 bytes are not subject to H.264 emulation-prevention removal. + let mut b = crate::bitstream::Bits::raw(&[0, 0, 3]); + assert_eq!(b.u(24), Some(3)); + } + + const EIGHT: &[u8] = &[ + 0x0a, 0x0d, 0, 0, 0, 3, 0xb5, 0xfe, 0xf3, 0x6b, 0xe4, 0x40, 0x80, 0x80, 0x61, + ]; + const TEN: &[u8] = &[0x0a, 0x0a, 0, 0, 0, 3, 0xb5, 0xfe, 0xf3, 0x6b, 0xe5, 1]; + + fn parameters(initial: &[u8]) -> ff::codec::Parameters { + let mut params = ff::codec::Parameters::new(); + unsafe { + let p = params.as_mut_ptr(); + (*p).codec_type = ff::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; + (*p).codec_id = ff::codec::Id::AV1.into(); + if !initial.is_empty() { + (*p).extradata = ff::ffi::av_mallocz( + initial.len() + ff::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize, + ).cast(); + assert!(!(*p).extradata.is_null()); + std::ptr::copy_nonoverlapping(initial.as_ptr(), (*p).extradata, initial.len()); + (*p).extradata_size = initial.len() as i32; + } + } + params + } + + #[test] + fn first_output_sequence_overrides_source_extradata() { + let mut guard = Guard::new(¶meters(EIGHT)).unwrap(); + let mut packet = ff::Packet::copy(TEN); + guard.packet(&mut packet).unwrap(); + assert_eq!(guard.fields, Some([0x81, 0, 0x4c])); + assert_eq!(packet.data().unwrap(), TEN); + } + + #[test] + fn an_incompatible_later_sequence_requests_one_mp4_retry() { + let mut guard = Guard::new(¶meters(&[])).unwrap(); + let mut first = ff::Packet::copy(EIGHT); + guard.packet(&mut first).unwrap(); + let mut repeat = ff::Packet::copy(EIGHT); + guard.packet(&mut repeat).unwrap(); + assert_eq!(repeat.data().unwrap(), EIGHT); + let mut changed = ff::Packet::copy(TEN); + assert!(guard.packet(&mut changed).unwrap_err().is::()); + } + + #[test] + fn an_incompatible_second_header_in_one_packet_is_not_missed() { + let mut guard = Guard::new(¶meters(&[])).unwrap(); + let mut packet = ff::Packet::copy(&[EIGHT, TEN].concat()); + assert!(guard.packet(&mut packet).unwrap_err().is::()); + } + +} diff --git a/rust/crates/core/src/bitstream.rs b/rust/crates/core/src/bitstream.rs index df65ac6..bda7002 100644 --- a/rust/crates/core/src/bitstream.rs +++ b/rust/crates/core/src/bitstream.rs @@ -865,6 +865,11 @@ pub(crate) struct Bits { } impl Bits { + /// AV1 has no H.26x emulation-prevention bytes to remove. + pub(crate) fn raw(data: &[u8]) -> Self { + Self { data: data.to_vec(), pos: 0 } + } + pub(crate) fn new(nal: &[u8]) -> Self { let mut data = Vec::with_capacity(nal.len()); let mut i = 0; diff --git a/rust/crates/core/src/cut.rs b/rust/crates/core/src/cut.rs index cf8f6e7..f83bb73 100644 --- a/rust/crates/core/src/cut.rs +++ b/rust/crates/core/src/cut.rs @@ -753,6 +753,7 @@ struct SegmentCtx<'a> { /// Everything the writer needs that is shared across segments. struct Writer { octx: ff::format::context::Output, + av1_mp4: Option, /// Whether a track's `pid` is really a PID, for the messages that name /// one. See [`crate::Source::on_a_ts`]. on_a_ts: bool, @@ -1324,7 +1325,10 @@ impl Writer { /// each picture's own length: under pulldown the lengths differ, and a /// decode-order sum overtakes the presentation time it is supposed to /// precede. The muxer rejects that outright -- `pts < dts`. - fn push(&mut self, e: Emitted) -> Result<()> { + fn push(&mut self, mut e: Emitted) -> Result<()> { + if let Some(config) = self.av1_mp4.as_mut() { + config.packet(&mut e.packet)?; + } self.opening = false; if let Some(held) = self.held.as_mut() { if !held.sealed { @@ -9148,7 +9152,44 @@ pub fn join_with_progress( // that file is gone. By then the output context has been dropped on the // way out, which is what lets Windows remove it. let done = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cut_into(reels, master.min(reels.len().saturating_sub(1)), output, opts, progress) + let master = master.min(reels.len().saturating_sub(1)); + let progress: Option = + progress.map(std::sync::Arc::from); + let run = |reels: &[Reel]| { + let report = progress.clone().map(|f| { + Box::new(move |p, job, pass| f(p, job, pass)) as Report + }); + cut_into(reels, master, output, opts, report) + }; + let first = run(reels); + if !first.as_ref().err().is_some_and(|e| e.is::()) { + return first; + } + // Only the MP4 writer can raise Changed. The failed context has + // closed before this retry, including on Windows. Matroska/WebM + // never enter the guard and retain their existing rendering policy. + crate::say!( + "note: an AV1 configuration change cannot share MP4's single sample description. \ + The entire selected video is re-encoded with one configuration." + ); + crate::say!( + "note: the copy figures above describe the discarded plan; \ + this retry copies no video (0%). Audio and subtitle modes are unchanged." + ); + let full: Vec> = reels.iter().map(|r| { + r.plans.iter().map(|p| { + let mut full = crate::plan::reencode_range(&r.src.points, p.t_in, p.t_out); + if let Some(s) = full.segments.first_mut() { + s.frames = p.segments.iter().map(|s| s.frames).sum(); + } + full + }).collect() + }).collect(); + let fallback: Vec = reels.iter().zip(&full).map(|(r, plans)| Reel { + src: r.src, plans, after: r.after.clone(), + }).collect(); + // One retry only: another configuration change is an error, not a loop. + run(&fallback).context("writing AV1 MP4 with one configuration after full-video fallback") })); if !matches!(done, Ok(Ok(()))) && ours { let _ = std::fs::remove_file(output); @@ -9199,6 +9240,7 @@ const WRITING_SHARE: f64 = 0.7; /// How a cut says where it has got to: which pass it is in, how far through /// the whole job that is, and how far through the pass itself. pub type Report = Box; +type SharedReport = std::sync::Arc; /// As [`cut`], reporting how far along it is. /// @@ -10051,6 +10093,11 @@ fn cut_into( let stated_rate = matches!(octx.format().name(), "matroska" | "webm") .then(|| rate_to_state(reels, grid)) .flatten(); + // MP4 only. Matroska/WebM keep their current sequence-header policy; + // deciding that policy is separate work for the maintainer. + let av1_mp4 = if src.video.codec == "av1" && octx.format().name() == "mp4" { + Some(crate::av1_mp4::Guard::new(¶ms)?) + } else { None }; { let mut ost = octx.add_stream(ff::encoder::find(ff::codec::Id::None))?; ost.set_parameters(params.clone()); @@ -10074,6 +10121,12 @@ fn cut_into( // writes an empty box: a file that says "wrote" and that nothing // can open. The version said as 1, the layout being the same. let par = ost.parameters().as_mut_ptr(); + if av1_mp4.is_some() { + // Let the first actual output sequence describe MP4's track, + // even when the whole selection is written by the encoder. + ff::ffi::av_freep(std::ptr::addr_of_mut!((*par).extradata).cast()); + (*par).extradata_size = 0; + } if src.video.codec == "hevc" && !(*par).extradata.is_null() && (*par).extradata_size >= 23 @@ -10851,6 +10904,7 @@ fn cut_into( }); let mut writer = Writer { + av1_mp4, on_a_ts: src.on_a_ts, into_ts: octx.format().name().contains("mpegts"), octx, diff --git a/rust/crates/core/src/lib.rs b/rust/crates/core/src/lib.rs index 79a6dad..12496e8 100644 --- a/rust/crates/core/src/lib.rs +++ b/rust/crates/core/src/lib.rs @@ -14,6 +14,7 @@ pub mod aac; pub mod adts; pub mod arib; pub mod audio; +pub(crate) mod av1_mp4; pub mod bdav; pub mod bitstream; pub mod blank; diff --git a/tests/av1_mp4_tests.py b/tests/av1_mp4_tests.py new file mode 100644 index 0000000..eba652c --- /dev/null +++ b/tests/av1_mp4_tests.py @@ -0,0 +1,255 @@ +"""AV1 MP4 sample-description regressions; Matroska keeps its current policy. + +Needs Python 3.11+, FFmpeg with libaom-av1/librav1e/libsvtav1 and ffprobe. +Media, commands, hashes and failures are retained in a fresh work directory. +""" +import argparse +from fractions import Fraction +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def sha(path): + with Path(path).open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def unhex(text): + return b''.join(bytes.fromhex(line.split(':', 1)[1].split(' ')[0].replace(' ', '')) + for line in text.splitlines() if ':' in line) + + +def boxes(data): + pos = 0 + while pos + 8 <= len(data): + size = int.from_bytes(data[pos:pos+4], 'big') + kind = data[pos+4:pos+8] + head = 8 + if size == 1: + size = int.from_bytes(data[pos+8:pos+16], 'big') + head = 16 + if size == 0: + size = len(data) - pos + assert size >= head and pos+size <= len(data), 'malformed MP4 box' + yield kind, data[pos+head:pos+size] + pos += size + assert pos == len(data), 'trailing partial MP4 box' + + +def find(data, wanted): + for kind, body in boxes(data): + if kind == wanted: + yield body + elif kind in (b'moov', b'trak', b'mdia', b'minf', b'stbl'): + yield from find(body, wanted) + + +def config(path): + videos = [body for body in find(path.read_bytes(), b'stsd') if body[12:16] == b'av01'] + assert len(videos) == 1, 'expected one AV1 track' + assert int.from_bytes(videos[0][4:8], 'big') == 1, 'test targets the single-description writer' + entries = list(boxes(videos[0][8:])) + assert len(entries) == 1 + records = [b for k, b in boxes(entries[0][1][78:]) if k == b'av1C'] + assert len(records) == 1 and len(records[0]) >= 4 + extra = records[0] + assert extra[0] == 0x81 and extra[3] == 0 + return extra + + +def sequence_payloads(d): + result = set() + for packet in d['packets']: + data = unhex(packet.get('data', '')) + while data: + head, data = data[0], data[1:] + if head & 4: + data = data[1:] + size = len(data) + if head & 2: + size = 0 + for n in range(8): + byte, data = data[0], data[1:] + size |= (byte & 127) << (7*n) + if not byte & 128: + break + body, data = data[:size], data[size:] + if (head >> 3) & 15 == 1: + result.add(body) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cut', type=Path, default=Path('rust/target/release/smartcut' + ('.exe' if os.name == 'nt' else ''))) + parser.add_argument('--ffmpeg', default=os.environ.get('FFMPEG', 'ffmpeg')) + parser.add_argument('--ffprobe', default=os.environ.get('FFPROBE', 'ffprobe')) + parser.add_argument('--fixtures', type=Path, help='reuse these exact input files in a paired run') + parser.add_argument('--work-dir', type=Path) + args = parser.parse_args() + if args.work_dir: + args.work_dir.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix='smartcut-av1-mp4-', dir=args.work_dir)) + fixtures = args.fixtures.resolve() if args.fixtures else work / 'fixtures' + fixtures.mkdir(parents=True, exist_ok=True) + cut = args.cut.resolve() + report = {'cut': str(cut), 'cut_sha256': sha(cut), 'test_sha256': sha(__file__), + 'commands': [], 'fixtures': [], 'cases': []} + + def save(): + (work/'results.json').write_text(json.dumps(report, indent=2)+'\n', encoding='utf-8') + + def run(command): + command = list(map(str, command)) + p = subprocess.run(command, capture_output=True, timeout=240) + report['commands'].append({'command': command, 'exit_code': p.returncode}) + assert p.returncode == 0, f'exit {p.returncode}: {command}\n{p.stderr.decode(errors="replace")[-2500:]}' + return p + + def probe(path): + return json.loads(run([args.ffprobe, '-v', 'error', '-select_streams', 'v:0', '-show_streams', + '-show_packets', '-show_data', '-show_data_hash', 'sha256', '-of', 'json', path]).stdout) + + def pixels(path): + text = run([args.ffmpeg, '-v', 'error', '-i', path, '-map', '0:v:0', + '-fps_mode', 'passthrough', '-f', 'framemd5', '-']).stdout.decode() + return [line.rsplit(',', 1)[1].strip() for line in text.splitlines() if line and not line.startswith('#')] + + def check_config(path, row): + extra = config(path) + trace = run([args.ffmpeg, '-hide_banner', '-loglevel', 'info', '-i', path, '-map', '0:v:0', + '-c', 'copy', '-bsf:v', 'trace_headers', '-f', 'null', '-']).stderr.decode(errors='replace') + headers = [] + for line in trace.splitlines(): + match = re.search(r'\b(seq_profile|seq_level_idx\[0\]|seq_tier\[0\]|high_bitdepth|twelve_bit|mono_chrome|chroma_sample_position)\s+\S+\s+=\s*(\d+)', line) + if not match: + continue + key, value = match[1], int(match[2]) + if key == 'seq_profile': + headers.append({}) + headers[-1][key] = value + assert headers, 'independent CBS parser found no Sequence Header' + # All generated sources and boundary encoders use profile 0, 4:2:0. + # AV1 infers subsampling_x/y=1 for this profile; they are not coded bits. + fields = [] + for header in headers: + assert header['seq_profile'] == 0 + fields.append(bytes([0x81, header['seq_level_idx[0]'], + header.get('seq_tier[0]', 0) << 7 + | header['high_bitdepth'] << 6 + | header.get('twelve_bit', 0) << 5 + | header.get('mono_chrome', 0) << 4 + | 12 | header.get('chroma_sample_position', 0)])) + row['av1C_prefix'] = extra[:4].hex() + row['sequence_field_prefixes'] = sorted(set(x.hex() for x in fields)) + assert all(value == extra[:3] for value in fields), 'av1C disagrees with a Sequence Header' + + try: + report['ffmpeg_version'] = run([args.ffmpeg, '-version']).stdout.decode().splitlines()[0] + report['ffprobe_version'] = run([args.ffprobe, '-version']).stdout.decode().splitlines()[0] + sources = [] + for name, encoder, pix_fmt in [('aom8', 'libaom-av1', 'yuv420p'), + ('aom10', 'libaom-av1', 'yuv420p10le'), + ('rav1e', 'librav1e', 'yuv420p')]: + path = fixtures/(name+'.mp4') + if not path.exists(): + options = (['-cpu-used', '8', '-crf', '22', '-b:v', '0'] + if encoder == 'libaom-av1' else ['-speed', '10', '-qp', '70']) + run([args.ffmpeg, '-v', 'error', '-y', '-f', 'lavfi', '-i', 'testsrc2=size=192x112:rate=25', + '-frames:v', '240', '-an', '-c:v', encoder, *options, '-g', '24', '-pix_fmt', pix_fmt, path]) + sources.append((name, path)) + mkv = fixtures/'rav1e.mkv' + if not mkv.exists(): + run([args.ffmpeg, '-v', 'error', '-y', '-i', fixtures/'rav1e.mp4', '-map', '0:v:0', '-c', 'copy', mkv]) + sources.append(('rav1e-mkv', mkv)) + cuts = [('partial', [(13, 227)]), ('multi', [(13, 81), (100, 227)]), + ('aligned', [(24, 216)]), ('one', [(23, 24)]), ('full', [(0, 240)])] + for name, source in sources: + src = probe(source) + source_pixels = pixels(source) + packets = sorted(src['packets'], key=lambda p: int(p['pts'])) + source_tb = Fraction(src['streams'][0]['time_base']) + times = [int(p['pts'])*source_tb for p in packets] + times.append(times[-1]+int(packets[-1]['duration'])*source_tb) + assert len(source_pixels) == len(packets) == 240 + report['fixtures'].append({'name': name, 'path': str(source), 'sha256': sha(source)}) + for label, ranges in cuts: + if source.suffix == '.mkv' and label not in ('partial', 'multi', 'full'): + continue + out = work/(name+'-'+label+source.suffix) + row = {'name': name+'-'+label, 'input': str(source), 'input_sha256': sha(source), + 'ranges': ranges, 'output': str(out), 'passed': False} + report['cases'].append(row) + try: + command = [cut, source, '--no-chapters', '-o', out] + for a, b in ranges: + command += ['--keep', f'{float(times[a]-times[0]):.12f}-{float(times[b]-times[0]):.12f}'] + proc = run(command) + log = proc.stdout+proc.stderr + out.with_suffix(out.suffix+'.log').write_bytes(log) + row['full_video_fallback'] = b'The entire selected video is re-encoded' in log + if row['full_video_fallback']: + assert b'discarded plan' in log and b'copies no video (0%)' in log + if source.suffix == '.mkv': + assert not row['full_video_fallback'], 'MKV must retain the existing policy' + d = probe(out) + row['output_sha256'] = sha(out) + row['sequence_payload_count'] = len(sequence_payloads(d)) + tb = Fraction(d['streams'][0]['time_base']) + observed = sorted(d['packets'], key=lambda p: int(p['pts'])) + origin = int(observed[0]['pts'])*tb + expected = [] + duration = Fraction(0) + indices = [] + for a, b in ranges: + expected.extend(duration+t-times[a] for t in times[a:b]) + duration += times[b]-times[a] + indices.extend(range(a, b)) + assert len(observed) == len(expected), 'wrong frame count' + assert all(abs(int(p['pts'])*tb-origin-t) <= tb for p, t in zip(observed, expected)), 'display PTS changed' + dts = [int(p['dts']) for p in d['packets']] + assert all(a < b for a, b in zip(dts, dts[1:])), 'non-monotonic DTS' + if out.suffix == '.mp4': + assert abs(Fraction(d['streams'][0]['duration'])-duration) <= tb, 'video duration changed' + run([args.ffmpeg, '-v', 'error', '-xerror', '-err_detect', 'explode', '-i', out, + '-map', '0:v:0', '-f', 'null', '-']) + made_pixels = pixels(out) + assert len(made_pixels) == len(expected), 'decoded frame count changed' + intervals = [(float(a), float(b)) for a, b in + re.findall(rb'^\s*copy\s+([\d.]+)\s+->\s*([\d.]+)', log, re.M)] + copied = [] if row['full_video_fallback'] else [ + (oi, si) for oi, si in enumerate(indices) + if any(a+0.001 < float(times[si]-times[0]) < b-0.001 for a, b in intervals)] + for oi, si in copied: + assert observed[oi]['data_hash'] == packets[si]['data_hash'], 'copy payload changed' + assert made_pixels[oi] == source_pixels[si], 'copy pixels changed' + row['copied_interior_frames_checked'] = len(copied) + if label == 'full': + assert [p['data_hash'] for p in observed] == [p['data_hash'] for p in packets], 'pure copy changed packets' + assert made_pixels == source_pixels + elif label != 'one' and not row['full_video_fallback']: + assert copied, 'test did not exercise a copied middle' + if out.suffix == '.mp4': + check_config(out, row) + row['frames'] = len(observed) + row['passed'] = True + print('ok', row['name'], flush=True) + except (AssertionError, KeyError, ValueError, subprocess.TimeoutExpired) as error: + row['error'] = str(error) + print('FAIL', row['name'], str(error)[:200], flush=True) + save() + finally: + report['passed'] = sum(row['passed'] for row in report['cases']) + report['total'] = len(report['cases']) + save() + print(f'{report["passed"]}/{report["total"]} passed; evidence: {work}', flush=True) + return 0 if report['total'] == 18 and report['passed'] == 18 else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/ci.sh b/tests/ci.sh index 55166ad..eed8bfc 100755 --- a/tests/ci.sh +++ b/tests/ci.sh @@ -28,7 +28,7 @@ SUITES=( audio_codec audio_smart audio_format audio_head bilingual index proxy disc bdav udf dvd bd_audio - vp9_av1 vfr demux transrate join + vp9_av1 av1_mp4 vfr demux transrate join ) [ $# -gt 0 ] && SUITES=("$@") diff --git a/tests/run_av1_mp4_tests.sh b/tests/run_av1_mp4_tests.sh new file mode 100755 index 0000000..6d9e81e --- /dev/null +++ b/tests/run_av1_mp4_tests.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -eu +cd "$(dirname "$0")/.." +python3 tests/av1_mp4_tests.py "$@" From f10ae7b81136104d72db0218412445f44793279b Mon Sep 17 00:00:00 2001 From: SteveYu000 <111431635+SteveYu000@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:08:27 +0800 Subject: [PATCH 2/2] Align AV1 boundary levels with copied MP4 sequences An MP4 cut mixing a rav1e source with SVT-AV1 boundaries can keep its original pictures when only the boundary's declared level differs. Use the source or active sequence fields for mixed exports and promote a compatible boundary level, including the source's unconstrained index 31. Update both the extracted configuration and the sample OBU, including the conditional tier bit, trailing bits and OBU size. Preserve packet properties and copied access units. Keep the existing single retry for conflicting configurations, a lower target level, or finite promotions whose constraints cannot be established. Fully encoded selections use the encoder's actual configuration. Exercise both seam orders, a finite promotion, a real chroma mismatch, and the original rav1e partial and multi-range reproductions. Inspect each sequence with FFmpeg and compare the copied key packets and pixels. --- docs/technical/av1-mp4.md | 32 +- docs/technical/building.md | 2 +- rust/crates/core/src/av1_mp4.rs | 487 +++++++++++++++++++++++----- rust/crates/core/src/bitstream.rs | 4 + rust/crates/core/src/cut.rs | 13 +- rust/crates/core/src/cut_conform.rs | 2 +- tests/av1_mp4_tests.py | 35 +- 7 files changed, 485 insertions(+), 90 deletions(-) diff --git a/docs/technical/av1-mp4.md b/docs/technical/av1-mp4.md index 8781c88..3bbe32b 100644 --- a/docs/technical/av1-mp4.md +++ b/docs/technical/av1-mp4.md @@ -9,17 +9,36 @@ The writer applies this check **only when the selected output muxer is the guard reads only the fields copied into `av1C`: profile, the first operating point's level and tier, bit depth, monochrome, chroma subsampling and chroma sample position. All headers in an extracted packet are checked. -Different Sequence Header bytes with the same fields remain permitted; the -guard does not require byte-identical sequences and does not rewrite frames. +Different Sequence Header bytes can share a description when their required +fields agree. The first sequence actually written supplies the track configuration. Where that sequence is present only in the input extradata, it is attached to the first output packet. This also fixes a selection wholly re-encoded with a configuration different from the source's, which needs no second attempt. +When the plan includes copied pictures, a boundary encoder may declare a +lower level than the source. The writer uses the active output fields, or +the source sequence / fixed av1C fields when the first picture is encoded. +If only the level differs and the target permits the encoder's level, it +rewrites `seq_level_idx[0]` in the boundary Sequence Header and in its +extracted packet configuration. A regular header includes a tier bit above +index 7. A reduced still-picture header always infers tier zero. The writer +keeps the rest of the syntax, regenerates the trailing one bit and zero +padding, and updates the OBU size if needed. Copied packets are read only. +For entirely encoded selections the encoder's actual configuration applies. + +Index 31 has no level-based constraints. For a finite target, the coded and +target indices must be defined, the target must be higher, and the other +av1C fields (including tier) must agree. The guard declines finite +promotions with explicit decoder model information, and 3.1 to 4.0, where +the minimum compression ratio can tighten. A lower finite target, a change +of bit depth/chroma/profile, or inconsistent copied sequences takes the +existing retry path. See [AV1 Annex A.3](https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=651). + FFmpeg 7.1's MOV/MP4 writer emits one `stsd` entry. A `NEW_EXTRADATA` side datum updates its configuration, but cannot assign a second sample entry to -the changed samples. If the fields change during an MP4 export, the failed +the changed samples. If the fields cannot be aligned during an MP4 export, the failed context closes and the selection is retried with all video ranges re-encoded. Audio and subtitle modes retain their existing settings. The log identifies the initial copy figures as a discarded plan and states that @@ -45,9 +64,10 @@ bash tests/run_av1_mp4_tests.sh ``` Python 3.11+, FFmpeg/ffprobe with libaom-av1, librav1e and libsvtav1 are -required. The suite generates three 240-picture CFR inputs (8/10-bit -libaom and 8-bit rav1e), then checks 15 MP4 exports and three Matroska -controls. MP4 checks include strict decoding, frame counts, display PTS, +required. The suite checks 18 MP4 exports and three Matroska controls. +The rav1e level-31 and finite Level 4.0 cases require mixed output with +the copied key packets and pixels intact. An input with colocated chroma +requires exactly one full-video retry. MP4 checks include strict decoding, frame counts, display PTS, DTS, video duration, `av1C` against the independent CBS `trace_headers` reader, and packet/pixel equality inside the declared copied intervals. The Matroska controls assert that no configuration retry was added; they diff --git a/docs/technical/building.md b/docs/technical/building.md index 830f1b3..dc557ff 100644 --- a/docs/technical/building.md +++ b/docs/technical/building.md @@ -85,7 +85,7 @@ bash tests/run_dvd_tests.sh # a DVD-Video disc, as a folder and as an bash tests/run_bd_audio_tests.sh # the sound a disc carries, written out 39 bash tests/run_vc1_tests.sh # the VC-1 encoder, put through a decoder 4 bash tests/run_vp9_av1_tests.sh # VP9 and AV1: cut, and still the source's bytes 5 -bash tests/run_av1_mp4_tests.sh # AV1 MP4 sample descriptions and MKV policy 18 +bash tests/run_av1_mp4_tests.sh # AV1 MP4 seam levels and Matroska controls 21 bash tests/run_vfr_tests.sh # variable rate, both ways round 8 bash tests/run_demux_tests.sh # what a reader loses by reading what it does not 9 bash tests/run_transrate_tests.sh # fitting a disc: the identity, the size, the cost 15 diff --git a/rust/crates/core/src/av1_mp4.rs b/rust/crates/core/src/av1_mp4.rs index c1ad682..7061a31 100644 --- a/rust/crates/core/src/av1_mp4.rs +++ b/rust/crates/core/src/av1_mp4.rs @@ -1,8 +1,10 @@ //! Check the AV1 fields shared by every sample in one MP4 description. //! FFmpeg extracts the header; only its public container fields are read here. +use crate::SegmentKind; use anyhow::{bail, Context, Result}; use ff::packet::{Mut, Ref}; use ffmpeg_next as ff; +use smartcut_mpeg2::bits::{Sink, Writer}; use std::ffi::{c_char, c_int, c_void}; // ffmpeg-sys-next 7.1 omits bsf.h. These are the public FFmpeg 7.1 @@ -39,6 +41,8 @@ pub(crate) struct Guard { filter: *mut Bsf, fields: Option<[u8; 3]>, initial: Vec, + source_fields: Option<[u8; 3]>, + has_copy: bool, } impl Drop for Guard { fn drop(&mut self) { @@ -52,11 +56,13 @@ fn check(code: i32) -> Result<()> { Ok(()) } impl Guard { - pub(crate) fn new(params: &ff::codec::Parameters) -> Result { + pub(crate) fn new(params: &ff::codec::Parameters, has_copy: bool) -> Result { let mut result = Self { filter: std::ptr::null_mut(), fields: None, initial: Vec::new(), + source_fields: None, + has_copy, }; unsafe { let p = params.as_ptr(); @@ -67,6 +73,16 @@ impl Guard { } else { data.to_vec() }; + // Prefer the actual source sequence. An av1C record may also + // consist solely of its four fixed bytes, without configOBUs. + result.source_fields = sequences(&result.initial) + .and_then(|s| fields(s[0])) + .or(match data { + [0x81, profile_level, colour, _, ..] => { + Some([0x81, *profile_level, *colour]) + } + _ => None, + }); } let filter = av_bsf_get_by_name(c"extract_extradata".as_ptr()); if filter.is_null() { @@ -84,93 +100,160 @@ impl Guard { Ok(result) } - pub(crate) fn packet(&mut self, packet: &mut ff::Packet) -> Result<()> { + pub(crate) fn packet(&mut self, packet: &mut ff::Packet, kind: SegmentKind) -> Result<()> { unsafe { check(av_bsf_send_packet(self.filter, packet.as_mut_ptr()))?; check(av_bsf_receive_packet(self.filter, packet.as_mut_ptr()))?; - let mut size = 0; - let mut extra = ff::ffi::av_packet_get_side_data( - packet.as_ptr(), - ff::ffi::AVPacketSideDataType::AV_PKT_DATA_NEW_EXTRADATA, - &mut size, - ); - if extra.is_null() { - if self.fields.is_some() { - return Ok(()); - } - if self.initial.is_empty() { - bail!("first AV1 picture has no sequence configuration"); - } - size = self.initial.len(); - extra = ff::ffi::av_packet_new_side_data( - packet.as_mut_ptr(), - ff::ffi::AVPacketSideDataType::AV_PKT_DATA_NEW_EXTRADATA, - size, - ); - if extra.is_null() { - bail!("cannot allocate initial AV1 configuration"); - } - std::ptr::copy_nonoverlapping(self.initial.as_ptr(), extra, size); + } + if packet_config(packet).is_none() { + if self.fields.is_some() { + return Ok(()); + } + if self.initial.is_empty() { + bail!("first AV1 picture has no sequence configuration"); } - let data = std::slice::from_raw_parts(extra, size); - for sequence in sequences(data).context("invalid extracted AV1 sequence header")? { - let fields = fields(sequence).context("truncated AV1 sequence header")?; - if self.fields.is_some_and(|old| old != fields) { - return Err(Changed.into()); + set_packet_config(packet, &self.initial)?; + } + let target = match kind { + SegmentKind::Reencode if self.has_copy => self.fields.or(self.source_fields), + _ => None, + }; + let config = packet_config(packet).context("missing AV1 configuration")?; + if let Some(target) = target { + if let Some(aligned) = align_obus(config, target)? { + // The muxer reads side data; a decoder reads OBUs in the + // sample. Make the same edit to both, preserving packet props. + if let Some(bytes) = align_obus(packet.data().unwrap_or_default(), target)? { + let size = c_int::try_from(bytes.len())?; + unsafe { + check(ff::ffi::av_packet_make_writable(packet.as_mut_ptr()))?; + let old_size = (*packet.as_ptr()).size; + if size > old_size { + check(ff::ffi::av_grow_packet( + packet.as_mut_ptr(), + size - old_size, + ))?; + } else { + ff::ffi::av_shrink_packet(packet.as_mut_ptr(), size); + } + } + packet + .data_mut() + .context("missing AV1 packet buffer")? + .copy_from_slice(&bytes); } - self.fields = Some(fields); + set_packet_config(packet, &aligned)?; + } + } + let config = packet_config(packet).context("missing AV1 configuration")?; + for sequence in sequences(config).context("invalid extracted AV1 sequence header")? { + let fields = fields(sequence).context("truncated AV1 sequence header")?; + if self.fields.is_some_and(|old| old != fields) { + return Err(Changed.into()); } + self.fields = Some(fields); } Ok(()) } } -fn sequences(mut data: &[u8]) -> Option> { - let mut headers = Vec::new(); - while let Some((&head, rest)) = data.split_first() { - if head & 0x81 != 0 { - return None; +const NEW_CONFIG: ff::ffi::AVPacketSideDataType = + ff::ffi::AVPacketSideDataType::AV_PKT_DATA_NEW_EXTRADATA; + +fn packet_config(packet: &ff::Packet) -> Option<&[u8]> { + let mut size = 0; + unsafe { + let p = ff::ffi::av_packet_get_side_data(packet.as_ptr(), NEW_CONFIG, &mut size); + (!p.is_null()).then(|| std::slice::from_raw_parts(p, size)) + } +} + +fn set_packet_config(packet: &mut ff::Packet, data: &[u8]) -> Result<()> { + unsafe { + let p = ff::ffi::av_packet_new_side_data(packet.as_mut_ptr(), NEW_CONFIG, data.len()); + if p.is_null() { + bail!("cannot allocate AV1 configuration"); } - let mut at = usize::from(head & 4 != 0); - let mut size = 0usize; - if head & 2 != 0 { - for n in 0..8 { - let byte = *rest.get(at)?; - at += 1; - size = size.checked_add(usize::from(byte & 127).checked_shl(n * 7)?)?; - if byte & 128 == 0 { - break; - } - if n == 7 { - return None; - } + std::ptr::copy_nonoverlapping(data.as_ptr(), p, data.len()); + } + Ok(()) +} + +struct Obu<'a> { + prefix: &'a [u8], // header, optional extension and optional LEB128 size + body: &'a [u8], + rest: &'a [u8], +} + +fn split_obu(data: &[u8]) -> Option> { + let head = *data.first()?; + if head & 0x81 != 0 { + return None; + } + let mut at = 1 + usize::from(head & 4 != 0); + let size = if head & 2 == 0 { + data.len().checked_sub(at)? + } else { + let mut value = 0u64; + for n in 0..8 { + let byte = *data.get(at)?; + at += 1; + value |= u64::from(byte & 127) << (7 * n); + if byte & 128 == 0 { + break; + } + if n == 7 { + return None; } - } else { - size = rest.len().checked_sub(at)?; } - let end = at.checked_add(size)?; - let body = rest.get(at..end)?; - if (head >> 3) & 15 == 1 { - headers.push(body); + usize::try_from(u32::try_from(value).ok()?).ok()? + }; + let end = at.checked_add(size)?; + Some(Obu { + prefix: data.get(..at)?, + body: data.get(at..end)?, + rest: data.get(end..)?, + }) +} + +fn sequences(mut data: &[u8]) -> Option> { + let mut headers = Vec::new(); + while !data.is_empty() { + let obu = split_obu(data)?; + if (obu.prefix[0] >> 3) & 15 == 1 { + headers.push(obu.body); } - data = rest.get(end..)?; + data = obu.rest; } (!headers.is_empty()).then_some(headers) } -/// AV1 5.5.1/5.5.2, only as far as the av1C colour/subsampling fields. -/// The shared bit reader must not unescape AV1 as though it were a NAL. +struct Sequence { + fields: [u8; 3], + level_bit: usize, + reduced: bool, + decoder_model: bool, + syntax_end: usize, +} + fn fields(data: &[u8]) -> Option<[u8; 3]> { + Some(parse_sequence(data)?.fields) +} + +/// AV1 5.5.1/5.5.2. Remember the two bit positions needed to splice in +/// operating point zero's level and regenerate trailing_bits. +fn parse_sequence(data: &[u8]) -> Option { let mut b = crate::bitstream::Bits::raw(data); let profile = b.u(3)?; b.u(1)?; let reduced = b.u(1)? != 0; let (mut level, mut tier) = (0, 0); + let mut level_bit = b.position(); + let mut model = false; if reduced { level = b.u(5)?; } else { let mut buffer_bits = 0; - let mut model = false; if b.u(1)? != 0 { b.skip(64)?; if b.u(1)? != 0 { @@ -195,6 +278,9 @@ fn fields(data: &[u8]) -> Option<[u8; 3]> { let count = b.u(5)? + 1; for i in 0..count { b.skip(12)?; + if i == 0 { + level_bit = b.position(); + } let l = b.u(5)?; let t = if l > 7 { b.u(1)? } else { 0 }; if i == 0 { @@ -262,12 +348,106 @@ fn fields(data: &[u8]) -> Option<[u8; 3]> { let p = if x != 0 && y != 0 { b.u(2)? } else { 0 }; (x, y, p) }; - Some([ - 0x81, - ((profile << 5) | level) as u8, - ((tier << 7) | (high << 6) | (twelve << 5) | (mono << 4) | (sx << 3) | (sy << 2) | position) - as u8, - ]) + b.skip(usize::from(mono == 0) + 1)?; // separate_uv_delta_q, film_grain_params_present + Some(Sequence { + level_bit, + reduced, + decoder_model: model, + syntax_end: b.position(), + fields: [ + 0x81, + ((profile << 5) | level) as u8, + ((tier << 7) + | (high << 6) + | (twelve << 5) + | (mono << 4) + | (sx << 3) + | (sy << 2) + | position) as u8, + ], + }) +} + +// Annex A.3 defines these finite indices; 31 removes level constraints. +// Among the listed same-tier promotions, 3.1 -> 4.0 can tighten the +// minimum compression ratio (MainCR / MaxDisplayRate). Explicit decoder +// models also need separate buffer verification for a finite promotion. +fn may_raise(from: u8, to: u8, decoder_model: bool) -> bool { + let defined = |n| matches!(n, 0 | 1 | 4 | 5 | 8 | 9 | 12..=19); + defined(from) + && from < to + && (to == 31 || (defined(to) && !decoder_model && (from, to) != (5, 8))) +} + +impl Sequence { + fn with_level(&self, data: &[u8], target: [u8; 3]) -> Result> { + let from = self.fields[1] & 31; + let to = target[1] & 31; + if self.fields[1] >> 5 != target[1] >> 5 + || self.fields[2] != target[2] + || !may_raise(from, to, self.decoder_model) + { + return Err(Changed.into()); + } + let end = self.syntax_end / 8; + let marker = 0x80u8 >> (self.syntax_end % 8); + if data + .get(end) + .is_none_or(|v| v & (marker | (marker - 1)) != marker) + || data[end + 1..].iter().any(|&v| v != 0) + { + bail!("invalid AV1 sequence trailing bits"); + } + let tail = self.level_bit + 5 + usize::from(!self.reduced && from > 7); + let mut bits = Writer::new(); + bits.copy(data, 0, self.level_bit); + bits.u(u32::from(to), 5); + if !self.reduced && to > 7 { + bits.u(u32::from(target[2] >> 7), 1); + } + bits.copy(data, tail, self.syntax_end - tail); + bits.u(1, 1); // trailing_one_bit; finish fills to a byte with zeroes + Ok(bits.finish()) + } +} + +/// Return bytes only when a boundary sequence needs a safe promotion. +/// Keep every other OBU, including its original size spelling, byte-exact. +fn align_obus(data: &[u8], target: [u8; 3]) -> Result>> { + let mut rest = data; + let mut preserved = data; + let mut out = Vec::new(); + while !rest.is_empty() { + let obu = split_obu(rest).context("invalid boundary AV1 OBU")?; + let head = obu.prefix[0]; + if (head >> 3) & 15 == 1 { + let seq = parse_sequence(obu.body).context("invalid boundary AV1 sequence")?; + if seq.fields != target { + let body = seq.with_level(obu.body, target)?; + out.extend_from_slice(&preserved[..preserved.len() - rest.len()]); + if head & 2 == 0 || body.len() == obu.body.len() { + out.extend_from_slice(obu.prefix); + } else { + out.extend_from_slice(&obu.prefix[..1 + usize::from(head & 4 != 0)]); + let mut size = body.len(); + while size >= 128 { + out.push((size as u8 & 127) | 128); + size >>= 7; + } + out.push(size as u8); + } + out.extend_from_slice(&body); + preserved = obu.rest; + } + } + rest = obu.rest; + } + if out.is_empty() { + Ok(None) + } else { + out.extend_from_slice(preserved); + Ok(Some(out)) + } } #[cfg(test)] @@ -318,7 +498,8 @@ mod tests { if !initial.is_empty() { (*p).extradata = ff::ffi::av_mallocz( initial.len() + ff::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize, - ).cast(); + ) + .cast(); assert!(!(*p).extradata.is_null()); std::ptr::copy_nonoverlapping(initial.as_ptr(), (*p).extradata, initial.len()); (*p).extradata_size = initial.len() as i32; @@ -329,30 +510,188 @@ mod tests { #[test] fn first_output_sequence_overrides_source_extradata() { - let mut guard = Guard::new(¶meters(EIGHT)).unwrap(); + let mut guard = Guard::new(¶meters(EIGHT), false).unwrap(); let mut packet = ff::Packet::copy(TEN); - guard.packet(&mut packet).unwrap(); + guard.packet(&mut packet, SegmentKind::Reencode).unwrap(); assert_eq!(guard.fields, Some([0x81, 0, 0x4c])); assert_eq!(packet.data().unwrap(), TEN); } #[test] fn an_incompatible_later_sequence_requests_one_mp4_retry() { - let mut guard = Guard::new(¶meters(&[])).unwrap(); + let mut guard = Guard::new(¶meters(&[]), false).unwrap(); let mut first = ff::Packet::copy(EIGHT); - guard.packet(&mut first).unwrap(); + guard.packet(&mut first, SegmentKind::Reencode).unwrap(); let mut repeat = ff::Packet::copy(EIGHT); - guard.packet(&mut repeat).unwrap(); + guard.packet(&mut repeat, SegmentKind::Reencode).unwrap(); assert_eq!(repeat.data().unwrap(), EIGHT); let mut changed = ff::Packet::copy(TEN); - assert!(guard.packet(&mut changed).unwrap_err().is::()); + assert!(guard + .packet(&mut changed, SegmentKind::Reencode) + .unwrap_err() + .is::()); } #[test] fn an_incompatible_second_header_in_one_packet_is_not_missed() { - let mut guard = Guard::new(¶meters(&[])).unwrap(); + let mut guard = Guard::new(¶meters(&[]), false).unwrap(); let mut packet = ff::Packet::copy(&[EIGHT, TEN].concat()); - assert!(guard.packet(&mut packet).unwrap_err().is::()); + assert!(guard + .packet(&mut packet, SegmentKind::Reencode) + .unwrap_err() + .is::()); } + #[test] + fn a_mixed_export_promotes_only_encoded_headers_and_preserves_packet_properties() { + // The rav1e MP4 has the fixed av1C fields and no configOBUs. + let target = [0x81, 31, 0x0c, 0]; + let mut guard = Guard::new(¶meters(&target), true).unwrap(); + let data = [[0x12, 0].as_slice(), EIGHT, EIGHT].concat(); + let mut encoded = ff::Packet::copy(&data); + encoded.set_flags(ff::packet::Flags::KEY); + encoded.set_pts(Some(23)); + encoded.set_dts(Some(21)); + encoded.set_duration(2); + guard.packet(&mut encoded, SegmentKind::Reencode).unwrap(); + assert!(encoded.is_key()); + assert_eq!( + (encoded.pts(), encoded.dts(), encoded.duration()), + (Some(23), Some(21), 2) + ); + let bytes = encoded.data().unwrap(); + assert_eq!(&bytes[..2], &[0x12, 0]); + let headers = sequences(bytes).unwrap(); + assert_eq!(headers.len(), 2); + assert_eq!( + headers, + sequences(packet_config(&encoded).unwrap()).unwrap() + ); + for header in headers { + assert_eq!(fields(header), Some([0x81, 31, 0x0c])); + assert_eq!(header.len(), EIGHT.len() - 2 + 1); // one tier bit crosses a byte boundary + } + let mut copied = ff::Packet::copy(bytes); + guard.packet(&mut copied, SegmentKind::Copy).unwrap(); + assert_eq!(copied.data().unwrap(), bytes); + let mut differing_copy = ff::Packet::copy(EIGHT); + assert!(guard + .packet(&mut differing_copy, SegmentKind::Copy) + .unwrap_err() + .is::()); + assert_eq!(differing_copy.data().unwrap(), EIGHT); + } + + #[test] + fn an_encoder_level_above_the_source_triggers_retry_and_the_retry_uses_its_own_level() { + let level4 = align_obus(EIGHT, [0x81, 8, 0x0c]).unwrap().unwrap(); + let src = parameters(EIGHT); + let mut mixed = Guard::new(&src, true).unwrap(); + let mut boundary = ff::Packet::copy(&level4); + assert!(mixed + .packet(&mut boundary, SegmentKind::Reencode) + .unwrap_err() + .is::()); + let mut full = Guard::new(&src, false).unwrap(); + let mut boundary = ff::Packet::copy(&level4); + full.packet(&mut boundary, SegmentKind::Reencode).unwrap(); + assert_eq!(full.fields, Some([0x81, 8, 0x0c])); + assert_eq!(boundary.data().unwrap(), level4); + } + + #[test] + fn tier_bit_presence_depends_on_regular_or_reduced_sequence_syntax() { + // Actual libaom 64x64 still picture. Tier is inferred even at level 31. + let still = [0x0a, 6, 0x18, 0x15, 0x7f, 0xfd, 0xa0, 0x08]; + let promoted = align_obus(&still, [0x81, 31, 0x0c]).unwrap().unwrap(); + assert_eq!(promoted.len(), still.len()); + let was = parse_sequence(&still[2..]).unwrap(); + let now = parse_sequence(&promoted[2..]).unwrap(); + assert!(now.reduced); + assert_eq!(was.syntax_end, now.syntax_end); + assert_eq!(now.fields, [0x81, 31, 0x0c]); + let level4 = align_obus(EIGHT, [0x81, 8, 0x0c]).unwrap().unwrap(); + let level5 = align_obus(&level4, [0x81, 12, 0x0c]).unwrap().unwrap(); + assert_eq!(level4.len(), level5.len()); // both levels have an explicit tier + assert_eq!( + fields(sequences(&level5).unwrap()[0]), + Some([0x81, 12, 0x0c]) + ); + } + + #[test] + fn extension_other_obus_and_the_implicit_final_size_are_preserved() { + // The padded body crosses the 127-byte LEB128 boundary on re-padding. + let mut input = vec![0x12, 0, 0x0e, 0, 0x80, 1]; + input.extend_from_slice(&EIGHT[2..]); + input.resize(6 + 128, 0); + input.extend_from_slice(&[0x7a, 0x81, 0, 0]); // preserve this original size spelling + input.push(0x08); // the final Sequence Header has no size field + input.extend_from_slice(&EIGHT[2..]); + let out = align_obus(&input, [0x81, 31, 0x0c]).unwrap().unwrap(); + assert_eq!(&out[..5], &[0x12, 0, 0x0e, 0, 14]); + let first = split_obu(&out[2..]).unwrap(); + assert_eq!(&first.rest[..5], &[0x7a, 0x81, 0, 0, 0x08]); + assert_eq!(sequences(&out).unwrap(), vec![first.body, first.body]); + let mut damaged = EIGHT.to_vec(); + *damaged.last_mut().unwrap() &= 0xfe; + assert!(align_obus(&damaged, [0x81, 31, 0x0c]).is_err()); + } + + #[test] + fn only_operating_point_zero_is_changed_when_timing_and_other_points_are_present() { + let source = &EIGHT[2..]; + let base = parse_sequence(source).unwrap(); + let mut w = Writer::new(); + w.copy(source, 0, 5); + w.u(1, 1); // timing_info_present + w.u(1, 32); + w.u(25, 32); + w.u(0, 1); // equal_picture_interval + w.u(0, 1); // decoder_model_info_present + w.u(0, 1); // initial_display_delay_present + w.u(1, 5); // two operating points + w.u(0x103, 12); + w.u(0, 5); + w.u(0x101, 12); + w.u(8, 5); + w.u(1, 1); // second point's own high tier + let tail = base.level_bit + 5; + w.copy(source, tail, base.syntax_end - tail); + w.u(1, 1); + let body = w.finish(); + let framed = [[0x08].as_slice(), &body].concat(); + let out = align_obus(&framed, [0x81, 31, 0x0c]).unwrap().unwrap(); + let before = parse_sequence(&body).unwrap(); + let after = parse_sequence(&out[1..]).unwrap(); + assert_eq!(before.level_bit, after.level_bit); + assert_eq!(after.syntax_end, before.syntax_end + 1); + let bit = |data: &[u8], i: usize| (data[i / 8] >> (7 - i % 8)) & 1; + for i in 0..before.syntax_end { + if i < before.level_bit { + assert_eq!(bit(&body, i), bit(&out[1..], i)); + } else if i >= before.level_bit + 5 { + assert_eq!(bit(&body, i), bit(&out[1..], i + 1)); + } + } + } + + #[test] + fn incompatible_fields_and_unproven_finite_levels_are_not_promoted() { + for target in [ + [0x81, 8, 0x8c], + [0x81, 8, 0x4c], + [0x81, 8, 0x0e], + [0x81, 0x28, 0x0c], + [0x81, 2, 0x0c], + [0x81, 25, 0x0c], + ] { + assert!(align_obus(EIGHT, target).unwrap_err().is::()); + } + assert!(!may_raise(31, 8, false)); + assert!(!may_raise(5, 8, false)); + assert!(!may_raise(0, 8, true)); + assert!(may_raise(0, 8, false)); + assert!(may_raise(0, 31, false)); + } } diff --git a/rust/crates/core/src/bitstream.rs b/rust/crates/core/src/bitstream.rs index bda7002..12f95b0 100644 --- a/rust/crates/core/src/bitstream.rs +++ b/rust/crates/core/src/bitstream.rs @@ -870,6 +870,10 @@ impl Bits { Self { data: data.to_vec(), pos: 0 } } + pub(crate) fn position(&self) -> usize { + self.pos + } + pub(crate) fn new(nal: &[u8]) -> Self { let mut data = Vec::with_capacity(nal.len()); let mut i = 0; diff --git a/rust/crates/core/src/cut.rs b/rust/crates/core/src/cut.rs index f83bb73..b10a966 100644 --- a/rust/crates/core/src/cut.rs +++ b/rust/crates/core/src/cut.rs @@ -1325,9 +1325,9 @@ impl Writer { /// each picture's own length: under pulldown the lengths differ, and a /// decode-order sum overtakes the presentation time it is supposed to /// precede. The muxer rejects that outright -- `pts < dts`. - fn push(&mut self, mut e: Emitted) -> Result<()> { + fn push(&mut self, kind: SegmentKind, mut e: Emitted) -> Result<()> { if let Some(config) = self.av1_mp4.as_mut() { - config.packet(&mut e.packet)?; + config.packet(&mut e.packet, kind)?; } self.opening = false; if let Some(held) = self.held.as_mut() { @@ -4633,7 +4633,7 @@ fn copy_segment( }, None => packet, }; - writer.push(Emitted { + writer.push(SegmentKind::Copy, Emitted { packet, display, fields, @@ -5881,7 +5881,7 @@ fn drain_encoder_shown( } None => packet, }; - writer.push(Emitted { + writer.push(SegmentKind::Reencode, Emitted { packet, display, fields, @@ -6069,7 +6069,7 @@ fn reencode_segment( // goes straight out with the timing worked out above. Pictures::Vc1(enc) => { let packet = encode_vc1(enc, &$f, shown_for)?; - writer.push(Emitted { + writer.push(SegmentKind::Reencode, Emitted { packet, display, fields, @@ -10096,7 +10096,8 @@ fn cut_into( // MP4 only. Matroska/WebM keep their current sequence-header policy; // deciding that policy is separate work for the maintainer. let av1_mp4 = if src.video.codec == "av1" && octx.format().name() == "mp4" { - Some(crate::av1_mp4::Guard::new(¶ms)?) + let has_copy = all_plans().flat_map(|p| &p.segments).any(|s| s.kind == SegmentKind::Copy); + Some(crate::av1_mp4::Guard::new(¶ms, has_copy)?) } else { None }; { let mut ost = octx.add_stream(ff::encoder::find(ff::codec::Id::None))?; diff --git a/rust/crates/core/src/cut_conform.rs b/rust/crates/core/src/cut_conform.rs index 63f5124..fb81c4b 100644 --- a/rust/crates/core/src/cut_conform.rs +++ b/rust/crates/core/src/cut_conform.rs @@ -381,7 +381,7 @@ fn pictures_afresh( // fields it said each picture was repeated three // times over. let packet = encode_vc1(enc, picture, 2)?; - writer.push(Emitted { + writer.push(SegmentKind::Reencode, Emitted { packet, display, fields: length, diff --git a/tests/av1_mp4_tests.py b/tests/av1_mp4_tests.py index eba652c..33bfb42 100644 --- a/tests/av1_mp4_tests.py +++ b/tests/av1_mp4_tests.py @@ -167,8 +167,21 @@ def check_config(path, row): if not mkv.exists(): run([args.ffmpeg, '-v', 'error', '-y', '-i', fixtures/'rav1e.mp4', '-map', '0:v:0', '-c', 'copy', mkv]) sources.append(('rav1e-mkv', mkv)) + finite = fixtures/'svt-level40.mp4' + if not finite.exists(): + run([args.ffmpeg, '-v', 'error', '-y', '-f', 'lavfi', '-i', 'testsrc2=size=192x112:rate=25', + '-frames:v', '240', '-an', '-c:v', 'libsvtav1', '-preset', '12', '-crf', '28', + '-g', '24', '-pix_fmt', 'yuv420p', '-svtav1-params', 'level=40:lp=2', finite]) + chroma = fixtures/'chroma.mp4' + if not chroma.exists(): + run([args.ffmpeg, '-v', 'error', '-y', '-i', fixtures/'aom8.mp4', '-map', '0:v:0', + '-c', 'copy', '-bsf:v', 'av1_metadata=chroma_sample_position=colocated', chroma]) + assert config(finite)[:3] == bytes.fromhex('81080c'), 'expected Level 4.0 fixture' + assert config(chroma)[:3] == bytes.fromhex('81000e'), 'expected colocated chroma fixture' + sources.extend([('finite4', finite), ('chroma', chroma)]) cuts = [('partial', [(13, 227)]), ('multi', [(13, 81), (100, 227)]), - ('aligned', [(24, 216)]), ('one', [(23, 24)]), ('full', [(0, 240)])] + ('aligned', [(24, 216)]), ('one', [(23, 24)]), ('full', [(0, 240)]), + ('copy-first', [(0, 227)])] for name, source in sources: src = probe(source) source_pixels = pixels(source) @@ -179,6 +192,10 @@ def check_config(path, row): assert len(source_pixels) == len(packets) == 240 report['fixtures'].append({'name': name, 'path': str(source), 'sha256': sha(source)}) for label, ranges in cuts: + if name in ('finite4', 'chroma') and label != 'partial': + continue + if label == 'copy-first' and name != 'rav1e': + continue if source.suffix == '.mkv' and label not in ('partial', 'multi', 'full'): continue out = work/(name+'-'+label+source.suffix) @@ -229,6 +246,8 @@ def check_config(path, row): assert observed[oi]['data_hash'] == packets[si]['data_hash'], 'copy payload changed' assert made_pixels[oi] == source_pixels[si], 'copy pixels changed' row['copied_interior_frames_checked'] = len(copied) + row['identical_source_packets'] = sum(p['data_hash'] == packets[si]['data_hash'] + for p, si in zip(observed, indices)) if label == 'full': assert [p['data_hash'] for p in observed] == [p['data_hash'] for p in packets], 'pure copy changed packets' assert made_pixels == source_pixels @@ -236,6 +255,18 @@ def check_config(path, row): assert copied, 'test did not exercise a copied middle' if out.suffix == '.mp4': check_config(out, row) + if name in ('finite4', 'rav1e') and label in ('partial', 'multi', 'copy-first'): + assert not row['full_video_fallback'], 'level-only seam discarded the copied middle' + assert config(source)[:3] == config(out)[:3], 'level must match the copied source' + # These CFR fixtures have exact millisecond cut points. Include the + # very first copied key packet, where the source Sequence Header lives. + for oi, si in enumerate(indices): + if any(a <= float(times[si]-times[0]) < b for a, b in intervals): + assert observed[oi]['data_hash'] == packets[si]['data_hash'], 'copied header changed' + assert made_pixels[oi] == source_pixels[si], 'copied picture changed' + if name == 'chroma': + assert row['full_video_fallback'], 'incompatible chroma must retry' + assert log.count(b'The entire selected video is re-encoded') == 1 row['frames'] = len(observed) row['passed'] = True print('ok', row['name'], flush=True) @@ -248,7 +279,7 @@ def check_config(path, row): report['total'] = len(report['cases']) save() print(f'{report["passed"]}/{report["total"]} passed; evidence: {work}', flush=True) - return 0 if report['total'] == 18 and report['passed'] == 18 else 1 + return 0 if report['total'] == 21 and report['passed'] == 21 else 1 if __name__ == '__main__':