From edbf35b0935a98246b003adc9c393099c12b4bda Mon Sep 17 00:00:00 2001 From: kaffeindecaf Date: Sun, 25 Oct 2026 16:05:09 +0100 Subject: [PATCH] feat: add pixel-mosh collapse texture --- README.md | 28 ++++++++----- ROADMAP.md | 9 ++-- benchmarks/collapse.json | 80 ++++++++++++++++++++---------------- effex.py | 9 +++- stateful_effects.py | 54 +++++++++++++++++++++--- tests/collapse_regression.py | 49 +++++++++++++++++++++- tests/menu_smoke.py | 3 +- 7 files changed, 172 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 45cd5a7..7ceb0d2 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ uses FFmpeg's `-enc_time_base demux` option (tested with FFmpeg 7.1). ./effex.py lobotomy+neon input/clip.mp4 --set lobotomy.kill=g --set neon.period=2 ./effex.py sediment input/clip.mp4 --set intensity=70 --set layers=3 --set excavation=10 --set seed=123 ./effex.py sediment+deepfry input/clip.mp4 --set sediment.decay=30 --set sediment.color=warm --set deepfry.strength=60 -./effex.py collapse input/clip.mp4 --set intensity=80 --set seed=17 -./effex.py collapse input/clip.mp4 --set mode=seam --set seam_strength=80 +./effex.py collapse input/clip.mp4 --set block_size=5 --set mosh=70 +./effex.py collapse input/clip.mp4 --set mode=elastic --set seed=17 ./effex.py inversion input/clip.mp4 --set lookahead=0.5 --set offset=0.3 --set region=bands ./effex.py datamosh input/a.mp4 --set second=input/b.mp4 --set cut=dice --set survival=50 --set seed=123 ./effex.py stretch input/clip.mp4 --dry-run @@ -119,6 +119,10 @@ the original canvas dimensions. High-gradient regions resist shrinking. Analysis is capped at 192 pixels on the longest side and updates at most five times per second; the displacement field is smoothed between updates. `mode=elastic` keeps the earlier seeded, spatially nonuniform elastic warp. +Both modes finish with a reduced-resolution pixel grid enlarged using nearest +neighbors. Moving regions can borrow a limited blend of the previous frame's +blocks for a codec-mosh-like trail; the effect does not modify codec packets or +perform full datamoshing. | option | range / values | default | | --- | --- | --- | @@ -133,6 +137,8 @@ times per second; the displacement field is smoothed between updates. | `speed` | 0-4 elastic cycles per second | 0.35 | | `coherence` | 0-100%, elapsed-time field smoothing | 80% | | `seed` | 0-2147483647; elastic mode only, 0 derives phases from the source | 0 | +| `block_size` | 1-16 reduced-grid cells; larger means chunkier pixels | 4 | +| `mosh` | 0-100%, motion-triggered previous-block carry (up to 50% mix) | 60% | In seam mode, shrink is capped at 50% per axis. The target reduction is `50% × intensity × seam_strength × horizontal/vertical control`; `preserve` @@ -142,8 +148,10 @@ at source dimensions. `smoothness` and `coherence` steady the reduced-resolution seam map; motion, speed, and random phases apply only to elastic mode. Zero intensity is an exact passthrough; two zero axis scales disable both modes. -Zero seam strength disables seam mode. Seam mode leaves the first frame -unchanged and reaches its configured scale at the end of the clip. +Zero seam strength disables seam mode. Seam geometry starts at identity and +reaches its configured scale at clip end; the pixel-mosh texture applies from +the first active frame. +`block_size=1` with `mosh=0` disables the pixel-mosh texture. State resets on a new input/shape, explicit reset, rewind, frame discontinuity, explicit scene cut, or a large RGB scene change. Persistent analysis state is bounded independently of clip length; full-frame remap buffers are transient. @@ -166,14 +174,14 @@ case, Python 3.13.5, NumPy 2.2.4 and OpenCV 4.12.0 (raw data in | mode | frame size | mean ms/frame | p95 ms/frame | peak process RSS | | --- | --- | --- | --- | --- | -| elastic | 640x360 | 4.23 | 4.57 | 63.3 MiB | -| elastic | 1280x720 | 9.79 | 10.60 | 110.1 MiB | -| seam | 640x360 | 16.59 | 89.64 | 62.6 MiB | -| seam | 1280x720 | 25.31 | 123.05 | 109.0 MiB | +| elastic | 640x360 | 5.44 | 6.25 | 64.4 MiB | +| elastic | 1280x720 | 13.62 | 15.43 | 110.5 MiB | +| seam | 640x360 | 22.62 | 117.82 | 63.5 MiB | +| seam | 1280x720 | 32.13 | 127.87 | 108.1 MiB | Seam averages include frames reusing a cached map; carving-update frames alone -averaged 89.37 ms at 360p and 109.63 ms at 720p. Persistent state was about -0.38 MiB for elastic and 0.53 MiB for seam mode in these cases. These are +averaged 116.51 ms at 360p and 127.58 ms at 720p. Persistent state was about +0.38 MiB for elastic and 0.54 MiB for seam mode in these cases. These are measurements of this fixture on this host, not a promise for every input. ## causality inversion diff --git a/ROADMAP.md b/ROADMAP.md index 066720a..c3fb3e0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -376,15 +376,18 @@ Shared rules for the stateful effects: - [x] E2.46 - content-aware scaling and elastic warp (prompt effect 05): change image composition while preserving output dimensions. - Default to progressive content-aware scale: linearly ramp seam-carved shrink from identity at clip start to the configured target at clip end, then remap each frame to its original canvas. - Analyze at reduced resolution, preserve high-energy regions, cap shrink to half per axis, and smooth cached decisions. Keep the earlier seeded elastic warp as an alternate mode. - - Options: intensity, horizontal/vertical shrink, content preservation, seam strength/mode, elastic motion influence, field smoothness/speed, temporal coherence, seed. - - Test linear progress against clip duration, fixed output size, content preservation, repeatability, bounded state and real CLI/menu renders. + - Options: intensity, horizontal/vertical shrink, content preservation, seam strength/mode, pixel block size, motion-block carry, elastic motion influence, field smoothness/speed, temporal coherence, seed. + - Test linear progress against clip duration, fixed output size, content preservation, pixelation, motion-block carry, repeatability, bounded state and real CLI/menu renders. _Done: `collapse` defaults to a content-aware seam scale that begins at the source composition and grows linearly to its target over probed clip duration. The low-resolution energy map removes seams from low-detail regions, and the renderer inverts that displacement so salient content shrinks instead of growing. Each output is mapped to the original frame dimensions. Analysis is capped at 192px; shrink is limited to 50% per axis and refreshed at <=5Hz with - temporal smoothing. The seeded elastic warp remains selectable. + temporal smoothing. A nearest-neighbor reduced pixel grid adds chunky output; + motion-triggered blocks blend at most half of their prior-frame value for a + restrained codec-mosh texture without bitstream corruption. The seeded + elastic warp remains selectable. Duration flows through the shared stateful adapter. Tests cover exact start, linear midpoint field, end-frame deformation, output shape/audio, resets, preservation, optional dependencies and TUI/CLI integration. Optional headless diff --git a/benchmarks/collapse.json b/benchmarks/collapse.json index 7025d61..f893295 100644 --- a/benchmarks/collapse.json +++ b/benchmarks/collapse.json @@ -1,5 +1,5 @@ { - "recorded_utc": "2026-10-25T14:16:27.765814+00:00", + "recorded_utc": "2026-10-25T14:54:16.689872+00:00", "platform": "Linux-7.1.13+parrot7-amd64-x86_64-with-glibc2.41", "logical_cpus": 22, "scope": "synthetic moving RGB; process() only; fresh process per mode/size; not end-to-end render FPS", @@ -20,17 +20,19 @@ "smoothness": 70, "speed": 0.35, "coherence": 80, - "seed": 17 + "seed": 17, + "block_size": 4, + "mosh": 60 }, - "mean_ms": 4.230598408328963, - "p50_ms": 4.365750500255672, - "p95_ms": 4.567256800373798, - "max_ms": 5.016768999666965, - "processor_fps": 236.37318021754476, + "mean_ms": 5.438171333351723, + "p50_ms": 5.658735499764589, + "p95_ms": 6.254641550503948, + "max_ms": 9.980480000194802, + "processor_fps": 183.8853428296948, "seam_updates": 0, "seam_update_mean_ms": null, - "peak_rss_mib": 63.31640625, - "persistent_state_mib": 0.375762939453125, + "peak_rss_mib": 64.3984375, + "persistent_state_mib": 0.3794708251953125, "work_shape": [ 108, 192 @@ -56,17 +58,19 @@ "smoothness": 70, "speed": 0.35, "coherence": 80, - "seed": 17 + "seed": 17, + "block_size": 4, + "mosh": 60 }, - "mean_ms": 9.785860641697278, - "p50_ms": 9.651802000007592, - "p95_ms": 10.604852450251201, - "max_ms": 14.228853000531672, - "processor_fps": 102.18825268561746, + "mean_ms": 13.62421634172127, + "p50_ms": 12.938760000452021, + "p95_ms": 15.432022549703106, + "max_ms": 17.359763000058592, + "processor_fps": 73.39871702842183, "seam_updates": 0, "seam_update_mean_ms": null, - "peak_rss_mib": 110.08984375, - "persistent_state_mib": 0.375762939453125, + "peak_rss_mib": 110.52734375, + "persistent_state_mib": 0.3794708251953125, "work_shape": [ 108, 192 @@ -92,17 +96,19 @@ "smoothness": 70, "speed": 0.35, "coherence": 80, - "seed": 17 + "seed": 17, + "block_size": 4, + "mosh": 60 }, - "mean_ms": 16.585925233357557, - "p50_ms": 2.023097500568838, - "p95_ms": 89.63626609997846, - "max_ms": 92.01955000025919, - "processor_fps": 60.29208415752432, + "mean_ms": 22.617298016621135, + "p50_ms": 3.8585624997722334, + "p95_ms": 117.81829465030569, + "max_ms": 120.20515199947113, + "processor_fps": 44.213946301857725, "seam_updates": 20, - "seam_update_mean_ms": 89.37203475011302, - "peak_rss_mib": 62.61328125, - "persistent_state_mib": 0.533966064453125, + "seam_update_mean_ms": 116.50683889988613, + "peak_rss_mib": 63.53515625, + "persistent_state_mib": 0.5376739501953125, "work_shape": [ 108, 192 @@ -128,17 +134,19 @@ "smoothness": 70, "speed": 0.35, "coherence": 80, - "seed": 17 + "seed": 17, + "block_size": 4, + "mosh": 60 }, - "mean_ms": 25.305225183334795, - "p50_ms": 9.182545500152628, - "p95_ms": 123.04726984962144, - "max_ms": 130.75589300024149, - "processor_fps": 39.51753018418377, + "mean_ms": 32.12750047495471, + "p50_ms": 13.086824499623617, + "p95_ms": 127.86849640019682, + "max_ms": 131.97810400015442, + "processor_fps": 31.1259819536711, "seam_updates": 20, - "seam_update_mean_ms": 109.627773550028, - "peak_rss_mib": 109.03125, - "persistent_state_mib": 0.533966064453125, + "seam_update_mean_ms": 127.5789061499836, + "peak_rss_mib": 108.13671875, + "persistent_state_mib": 0.5376739501953125, "work_shape": [ 108, 192 @@ -149,4 +157,4 @@ "opencv_threads": 1 } ] -} \ No newline at end of file +} diff --git a/effex.py b/effex.py index 3b1b4dd..3362ee0 100755 --- a/effex.py +++ b/effex.py @@ -1943,6 +1943,10 @@ def build_temporal_sediment(ctx: EffectContext) -> Plan: "percent", 80, minimum=0, maximum=100, step=5), Param("seed", "random seed", "0 derives repeatable elastic phases from the clip", "int", 0, minimum=0, maximum=2147483647, step=1), + Param("block_size", "pixel blocks", "larger blocks make the output chunkier", + "int", 4, minimum=1, maximum=16, step=1), + Param("mosh", "block carry", "blend prior-frame blocks in moving areas; not codec datamosh", + "percent", 60, minimum=0, maximum=100, step=5), ) @@ -1955,6 +1959,7 @@ def build_collapse(ctx: EffectContext) -> Plan: notes=[ "content-aware mode linearly shrinks over the clip and remaps to original dimensions", "seam analysis is capped at 192px; target scales remove at most half of either axis", + "a reduced pixel grid and partial moving-block carry add a restrained codec-mosh texture", "elastic mode remains available for seeded smooth fields and CPU OpenCV remapping", "needs requirements-collapse.txt; no stateful preview or original VFR PTS", ], @@ -2112,8 +2117,8 @@ def build_inversion(ctx: EffectContext) -> Plan: "pass 2: ffmpeg -f concat -i list.txt -c copy .mp4", )), Effect(key="collapse", name="elastic collapse", - blurb="progressive content-aware scaling or elastic warp", - tag="stateful seam carving and CPU remap", + blurb="progressive content-aware scale with pixelated block mosh", + tag="seam carve, pixel grid, motion block carry", build=build_collapse, params=COLLAPSE_PARAMS, frame_processor=ElasticCollapse, stages=( diff --git a/stateful_effects.py b/stateful_effects.py index c0eac02..d8bfbe3 100644 --- a/stateful_effects.py +++ b/stateful_effects.py @@ -283,11 +283,12 @@ def process(self, frame, frame_index: int, timestamp: Optional[float], class ElasticCollapse: - """Progressive content-aware scaling or elastic warp with bounded state. + """Progressive content-aware scaling with pixel-mosh texture. Seam mode linearly scales a reduced-resolution carved field over clip time; - elastic mode evolves a seeded nonuniform field. OpenCV is loaded only on - initialization; full-size maps are transient and analysis stays bounded. + elastic mode evolves a seeded nonuniform field. A bounded pixel grid adds + blocky output, with limited previous-frame carry in moving areas. OpenCV is + loaded only on initialization; full-size maps are transient. """ MAX_WORK_SIDE = 192 @@ -296,6 +297,7 @@ class ElasticCollapse: "intensity": 100, "horizontal": 100, "vertical": 100, "preserve": 75, "mode": "seam", "seam_strength": 100, "motion": 50, "smoothness": 70, "speed": 0.35, "coherence": 80, "seed": 0, + "block_size": 4, "mosh": 60, } def __init__(self): @@ -313,6 +315,7 @@ def __init__(self): self._phase: Any = None self._field: Any = None self._previous_rgb: Any = None + self._previous_mosh_grid: Any = None self._previous_timestamp: Optional[float] = None self._next_index: Optional[int] = None self._clock = 0.0 @@ -352,6 +355,8 @@ def initialize(self, config: dict, frame_shape: tuple, fps: float, self._config[key] = mode else: high = 4.0 if key == "speed" else (2147483647 if key == "seed" else 100) + if key == "block_size": + high = 16 number = self._number(config.get(key, default), default, 0, high) self._config[key] = number if key == "speed" else int(number) self._fps = self._number(fps, 30.0, 0.001, 1000) @@ -375,7 +380,7 @@ def work_shape(self) -> tuple: def state_nbytes(self) -> int: return sum(int(value.nbytes) for value in (self._x, self._y, self._phase, self._field, self._previous_rgb, - self._seam_target) + self._seam_target, self._previous_mosh_grid) if value is not None) @property @@ -396,6 +401,7 @@ def reset(self) -> None: self._phase = self._np.random.default_rng(self._seed).uniform(0, 2 * math.pi, 4) self._field = None self._previous_rgb = None + self._previous_mosh_grid = None self._previous_timestamp = None self._next_index = None self._clock = 0.0 @@ -451,6 +457,38 @@ def _elastic_field(self, current): field *= (1 + self._config["motion"] / 100.0 * moving)[:, :, None] return field.astype(np.float32) + def _pixel_mosh(self, frame, motion_delta): + """Pixelate on the bounded work grid and carry old blocks in motion areas.""" + np, cv = self._np, self._cv + block_size = self._config["block_size"] + mosh = self._config["mosh"] / 100.0 + if block_size == 1 and mosh == 0: + self._previous_mosh_grid = None + return frame + + work_height, work_width = self._work_shape + height, width = frame.shape[:2] + reduced = cv.resize(frame, (work_width, work_height), + interpolation=cv.INTER_AREA) + step = max(1, block_size) + grid_width = max(1, (work_width + step - 1) // step) + grid_height = max(1, (work_height + step - 1) // step) + current_grid = cv.resize(reduced, (grid_width, grid_height), + interpolation=cv.INTER_AREA) + styled = current_grid.astype(np.float32) + if (mosh and motion_delta is not None + and self._previous_mosh_grid is not None): + motion_grid = cv.resize(motion_delta.astype(np.float32), + (grid_width, grid_height), + interpolation=cv.INTER_AREA) + moving = np.clip((motion_grid - 4.0) / 48.0, 0.0, 1.0) + weight = (moving * (0.5 * mosh))[:, :, None] + previous = self._previous_mosh_grid.astype(np.float32) + styled = styled * (1.0 - weight) + previous * weight + self._previous_mosh_grid = current_grid.copy() + styled = np.clip(np.rint(styled), 0, 255).astype(np.uint8) + return cv.resize(styled, (width, height), interpolation=cv.INTER_NEAREST) + def process(self, frame, frame_index: int, timestamp: Optional[float], context: Optional[dict] = None): if self._np is None: @@ -471,6 +509,9 @@ def process(self, frame, frame_index: int, timestamp: Optional[float], if ((self._next_index is not None and index != self._next_index) or rewind or cut or (context or {}).get("scene_cut")): self.reset() + motion_delta = (np.abs(current.astype(np.float32) + - self._previous_rgb.astype(np.float32)).mean(axis=2) + if self._previous_rgb is not None else None) if self._config["mode"] == "seam": self._progress = ( min(1.0, max(0.0, stamp / self._clip_duration)) @@ -508,8 +549,9 @@ def process(self, frame, frame_index: int, timestamp: Optional[float], map_y = full[:, :, 1] * (amount * max(0, height - 1)) map_x += np.arange(width, dtype=np.float32)[None, :] map_y += np.arange(height, dtype=np.float32)[:, None] - return cv.remap(frame, map_x, map_y, interpolation=cv.INTER_LINEAR, - borderMode=cv.BORDER_REFLECT_101) + warped = cv.remap(frame, map_x, map_y, interpolation=cv.INTER_LINEAR, + borderMode=cv.BORDER_REFLECT_101) + return self._pixel_mosh(warped, motion_delta) class CausalityInversion: diff --git a/tests/collapse_regression.py b/tests/collapse_regression.py index cddf266..5dbae31 100644 --- a/tests/collapse_regression.py +++ b/tests/collapse_regression.py @@ -176,6 +176,7 @@ def test_collapse_is_registered_with_all_controls(self): self.assertEqual([defaults[key] for key in ("intensity", "horizontal", "vertical", "seam_strength")], [100, 100, 100, 100]) + self.assertEqual((defaults["block_size"], defaults["mosh"]), (4, 60)) def test_seam_mode_is_distinct_and_zero_strength_disables_it(self): frame = self.frame() @@ -193,7 +194,8 @@ def test_content_aware_scale_grows_linearly_over_clip_duration(self): frame = self.frame() processor = self.processor( mode="seam", intensity=100, horizontal=100, vertical=0, - preserve=0, seam_strength=100, coherence=0, _clip_duration=2, + preserve=0, seam_strength=100, coherence=0, block_size=1, mosh=0, + _clip_duration=2, ) first = processor.process(frame, 0, 0.0, {}) first_field = processor.displacement.copy() @@ -214,13 +216,54 @@ def test_content_aware_scale_shrinks_the_salient_subject(self): frame[20:44, 30:66] = 255 processor = self.processor( mode="seam", intensity=100, horizontal=100, vertical=0, - preserve=100, seam_strength=100, coherence=0, _clip_duration=1, + preserve=100, seam_strength=100, coherence=0, block_size=1, mosh=0, + _clip_duration=1, ) out = processor.process(frame, 30, 1.0, {}) ys, xs = np.where(out.mean(axis=2) > 128) self.assertEqual(out.shape, frame.shape) self.assertLess(int(np.ptp(xs)) + 1, 36) + def test_block_size_reduces_high_frequency_detail(self): + y, x = np.indices((64, 96)) + checker = (((x + y) % 2) * 255).astype(np.uint8) + frame = np.repeat(checker[:, :, None], 3, axis=2) + fine = self.processor(mode="seam", block_size=1, mosh=0, preserve=0) + chunky = self.processor(mode="seam", block_size=4, mosh=0, preserve=0) + fine_out = fine.process(frame, 0, 0.0, {}) + chunky_out = chunky.process(frame, 0, 0.0, {}) + fine_edges = np.count_nonzero(np.diff(fine_out[32, :, 0])) + chunky_edges = np.count_nonzero(np.diff(chunky_out[32, :, 0])) + + self.assertEqual(chunky_out.shape, frame.shape) + self.assertLess(chunky_edges, fine_edges / 2) + + def test_mosh_carries_old_blocks_only_when_content_moves(self): + before = np.zeros((64, 96, 3), dtype=np.uint8) + before[20:44, 8:32] = 255 + after = np.zeros_like(before) + after[20:44, 56:80] = 255 + + def run(mosh): + processor = self.processor( + mode="seam", block_size=4, mosh=mosh, preserve=0, + coherence=0, _clip_duration=1, + ) + processor.process(before, 0, 0.0, {}) + return processor.process(after, 1, 0.0, {}) + + clean = run(0) + carried = run(80) + self.assertEqual(carried.shape, after.shape) + self.assertGreater(float(np.abs(carried.astype(float) - clean).mean()), 2) + + def run_static(mosh): + processor = self.processor(mode="seam", block_size=4, mosh=mosh) + processor.process(before, 0, 0.0, {}) + return processor.process(before, 1, 1 / 30, {}) + + np.testing.assert_array_equal(run_static(0), run_static(80)) + def test_seam_updates_are_rate_limited_and_deterministic(self): frame = self.frame() a, b = self.processor(mode="seam"), self.processor(mode="seam") @@ -261,6 +304,7 @@ def test_cli_seam_scale_starts_at_source_and_grows_toward_clip_end(self): "--set", "mode=seam", "--set", "intensity=100", "--set", "horizontal=100", "--set", "vertical=100", "--set", "preserve=0", "--set", "seam_strength=100", + "--set", "block_size=1", "--set", "mosh=0", "--json", "--preset", "ultrafast", ], env_extra=env) self.assertEqual(proc.returncode, 0, proc.stderr.decode()) @@ -295,6 +339,7 @@ def test_cli_bounds_and_real_renders(self): for setting in ("intensity=101", "horizontal=-1", "vertical=101", "preserve=101", "mode=unknown", "seam_strength=101", "motion=-1", "smoothness=101", "speed=-1", "speed=5", + "block_size=0", "block_size=17", "mosh=-1", "mosh=101", "coherence=101", "seed=-1"): with self.subTest(invalid=setting): proc = run([EFFEX, "collapse", clip, "--set", setting, diff --git a/tests/menu_smoke.py b/tests/menu_smoke.py index 35bf89b..b4fee76 100644 --- a/tests/menu_smoke.py +++ b/tests/menu_smoke.py @@ -852,7 +852,8 @@ def scenario_collapse(c: Checker): label in last_frame(d) for label in ("horizontal deformation amount", "vertical deformation amount", "protect strong edges", "shrink amount", - "smooth field changes", "random seed"))) + "smooth field changes", "random seed", "pixel blocks", + "block carry"))) if mode == "elastic": for _ in range(4): press_and_wait(d, DOWN)