diff --git a/sam-tool-tauri/src-tauri/src/decoder.rs b/sam-tool-tauri/src-tauri/src/decoder.rs index de57042..93c098b 100644 --- a/sam-tool-tauri/src-tauri/src/decoder.rs +++ b/sam-tool-tauri/src-tauri/src/decoder.rs @@ -2,15 +2,37 @@ use crate::video::VideoInfo; use std::collections::{HashMap, VecDeque}; use std::io::{BufRead, BufReader}; use std::path::PathBuf; -use std::process::{Child, ChildStdout, Command, Stdio}; +use std::process::{Child, ChildStderr, ChildStdout, Command, Stdio}; use std::sync::{Arc, Condvar, Mutex}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; +/// Cap on stderr lines retained per ffmpeg process. The hwaccel-init message +/// is usually 1-3 lines; we keep more to cover later-frame errors too. +const STDERR_TAIL_LINES: usize = 32; + pub const DEFAULT_CACHE_CAPACITY: usize = 160; pub const MIN_CACHE_CAPACITY: usize = 50; pub const MAX_CACHE_CAPACITY: usize = 800; +/// Cache cap for the RGBA preview decoder. Raw frames are 30-80× bigger +/// than MJPEG so we cap depth for memory headroom. Sized for 1080p (the +/// most common case): 16 frames × 8.3 MB ≈ 133 MB. At 720p this drops to +/// ~45 MB. At 4K it's ~528 MB, so users with 4K sources should set +/// preview_max_height in settings. +pub const RGBA_PREVIEW_CACHE_CAPACITY: usize = 16; +pub const RGBA_PREVIEW_LOOKAHEAD: u64 = 8; const READ_BUF_BYTES: usize = 256 * 1024; + +/// Wire format produced by an ffmpeg child. Preview decoder uses Rgba so +/// the JS side avoids a CPU MJPEG decode per frame (which was the real +/// per-frame bottleneck — see review findings A + E). Extract decoder +/// stays on Mjpeg because `extract_one` writes the decoder output to disk +/// and users expect .jpg files. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutputFormat { + Mjpeg, + Rgba, +} /// If the requested frame is this far ahead of the worker, kill + reseek. /// Higher = fewer ffmpeg restarts during forward scan; the cap is bounded /// by lookahead × cache capacity anyway. @@ -86,6 +108,10 @@ struct SharedState { eof: bool, error: Option, shutdown: bool, + /// Last N stderr lines from the most recent ffmpeg child. Populated by + /// a non-blocking drain thread; read only when something has gone wrong + /// (e.g. hwaccel fallback) to surface the ffmpeg diagnostic. + stderr_tail: VecDeque, } struct Shared { @@ -103,9 +129,19 @@ pub struct DecoderSession { /// When Some(H), ffmpeg downscales to height H (aspect preserved) via /// `-vf scale=-2:H`. Used by the playback decoder for cheap previews; /// the extract decoder always passes None so saved frames are full-res. - preview_max_height: Option, + format: OutputFormat, + /// Pre-computed (width, height) ffmpeg will emit. For Mjpeg this is the + /// source dims (mjpeg encoder doesn't care). For Rgba these MUST be + /// even and exact, since the reader frames input by `out_w * out_h * 4` + /// byte counts — there's no marker to find frame boundaries in raw. + output_dims: (u32, u32), shared: Arc, worker: Option>, + /// One-shot notice surfaced via `take_notice` after a hwaccel fallback. + /// Lets the frontend show "Hardware decode 'cuda' failed; running + /// software" instead of silently testing the same code path under three + /// different settings. + last_notice: Mutex>, } impl DecoderSession { @@ -114,26 +150,61 @@ impl DecoderSession { info: VideoInfo, hwaccel: Option, preview_max_height: Option, + format: OutputFormat, ) -> Self { + let preview_max_height = preview_max_height.filter(|h| *h > 0); + let output_dims = compute_output_dims(&info, preview_max_height); + // RGBA frames are huge (1080p = 8.3 MB), so a 160-entry cache would + // be ~1.3 GB. Cap depth and lookahead for the Rgba path; Mjpeg + // keeps its existing generous defaults. + let (init_cap, init_lookahead) = match format { + OutputFormat::Mjpeg => ( + DEFAULT_CACHE_CAPACITY, + (DEFAULT_CACHE_CAPACITY as u64) / 2, + ), + OutputFormat::Rgba => (RGBA_PREVIEW_CACHE_CAPACITY, RGBA_PREVIEW_LOOKAHEAD), + }; Self { path, info, hwaccel: hwaccel.filter(|s| !s.is_empty()), - preview_max_height: preview_max_height.filter(|h| *h > 0), + format, + output_dims, shared: Arc::new(Shared { state: Mutex::new(SharedState { child: None, - cache: FrameCache::new(DEFAULT_CACHE_CAPACITY), + cache: FrameCache::new(init_cap), next_idx: 0, reader_idx: 0, - lookahead: (DEFAULT_CACHE_CAPACITY as u64) / 2, + lookahead: init_lookahead, eof: false, error: None, shutdown: false, + stderr_tail: VecDeque::new(), }), cond: Condvar::new(), }), worker: None, + last_notice: Mutex::new(None), + } + } + + /// Drain and return the most recent fallback / diagnostic notice (if any). + /// One-shot: a subsequent call returns None until the next event sets it. + pub fn take_notice(&self) -> Option { + self.last_notice.lock().ok().and_then(|mut n| n.take()) + } + + fn snapshot_stderr_tail(&self) -> String { + let st = self.shared.state.lock().unwrap(); + if st.stderr_tail.is_empty() { + "(no ffmpeg stderr captured)".into() + } else { + st.stderr_tail + .iter() + .cloned() + .collect::>() + .join("\n") } } @@ -142,6 +213,11 @@ impl DecoderSession { } pub fn set_cache_capacity(&self, capacity: usize) { + // RGBA preview cap is fixed by memory budget — a user-set 800-frame + // cache at 1080p RGBA would be ~6.6 GB. + if self.format == OutputFormat::Rgba { + return; + } let clamped = capacity.clamp(MIN_CACHE_CAPACITY, MAX_CACHE_CAPACITY); let mut st = self.shared.state.lock().unwrap(); st.cache.set_capacity(clamped); @@ -158,9 +234,14 @@ impl DecoderSession { // path that worked on h264 but chokes on an HEVC segment). // Once cleared, subsequent failures propagate. if let Some(prev) = self.hwaccel.take() { - eprintln!( - "[decoder] hwaccel '{prev}' failed ({e}); falling back to software" + let tail = self.snapshot_stderr_tail(); + let notice = format!( + "Hardware decode '{prev}' failed; falling back to software.\nffmpeg said:\n{tail}" ); + eprintln!("[decoder] {notice}"); + if let Ok(mut slot) = self.last_notice.lock() { + *slot = Some(notice); + } self.stop_worker(); self.decode_at_inner(idx) } else { @@ -233,16 +314,18 @@ impl DecoderSession { fn seek_and_start_worker(&mut self, idx: u64) -> Result<(), String> { self.stop_worker(); - let (child, stdout) = spawn_ffmpeg( + let (child, stdout, stderr) = spawn_ffmpeg( &self.path, &self.info, idx, self.hwaccel.as_deref(), - self.preview_max_height, + self.format, + self.output_dims, )?; { let mut st = self.shared.state.lock().unwrap(); + st.stderr_tail.clear(); st.child = Some(child); st.next_idx = idx; st.reader_idx = idx; @@ -252,9 +335,14 @@ impl DecoderSession { } self.shared.cond.notify_all(); + let drain_shared = self.shared.clone(); + thread::spawn(move || drain_stderr(stderr, drain_shared)); + + let format = self.format; + let dims = self.output_dims; let shared = self.shared.clone(); let handle = thread::spawn(move || { - worker_loop(stdout, shared); + worker_loop(stdout, shared, format, dims); }); self.worker = Some(handle); Ok(()) @@ -324,13 +412,37 @@ fn vaapi_device() -> String { "/dev/dri/renderD128".to_string() } +/// Compute the (width, height) ffmpeg will output. Scales down to +/// preview_max_height only when source is taller; preserves aspect via a +/// rounded-to-even width. Returns source dims when no scaling applies. +/// The reader uses these to frame raw video output, so they MUST match +/// what ffmpeg actually emits — we set them explicitly via `-s WxH` in +/// the Rgba path to take the guesswork out. +fn compute_output_dims(info: &VideoInfo, preview_max_height: Option) -> (u32, u32) { + if let Some(h) = preview_max_height { + if info.height > h && info.height > 0 { + // Maintain aspect; round width to nearest even (ffmpeg requires + // even dims for many filters). + let w = ((info.width as u64 * h as u64) + (info.height as u64 / 2)) + / (info.height as u64); + let w_even = ((w as u32) + 1) & !1; + return (w_even.max(2), h); + } + } + // Even source dims for safety (almost always already even). + let w = (info.width + 1) & !1; + let h = (info.height + 1) & !1; + (w.max(2), h.max(2)) +} + fn spawn_ffmpeg( path: &PathBuf, info: &VideoInfo, idx: u64, hwaccel: Option<&str>, - preview_max_height: Option, -) -> Result<(Child, BufReader), String> { + format: OutputFormat, + out_dims: (u32, u32), +) -> Result<(Child, BufReader, ChildStderr), String> { let path_str = path.to_str().ok_or_else(|| "non-utf8 path".to_string())?; let t = idx as f64 / info.fps; let mut cmd = Command::new("ffmpeg"); @@ -362,39 +474,96 @@ fn spawn_ffmpeg( cmd.args(["-ss", &format!("{:.6}", t)]); cmd.args(["-i", path_str]); cmd.args(["-an"]); - // Preview scaling: cap output height to `h`, width auto (`-2` keeps - // aspect & ensures even). Skip the filter entirely when the source is - // already shorter than the cap — no upscale, no wasted filter step. - if let Some(h) = preview_max_height { - if info.height > h { - let vf = format!("scale=-2:{h}"); - cmd.args(["-vf", &vf]); + // Pin output dimensions exactly. For Mjpeg this is cosmetic (matches + // source); for Rgba it MUST match what the reader expects to byte-frame + // raw video output. + let (ow, oh) = out_dims; + if ow != info.width || oh != info.height { + cmd.args(["-s", &format!("{ow}x{oh}")]); + } + match format { + OutputFormat::Mjpeg => { + cmd.args([ + "-q:v", + "3", + "-f", + "image2pipe", + "-vcodec", + "mjpeg", + "pipe:1", + ]); + } + OutputFormat::Rgba => { + // Raw RGBA, no entropy coding. Worker grabs the bytes verbatim + // into ImageData, eliminating the CPU MJPEG encode (ffmpeg) + // and the CPU JPEG decode (createImageBitmap on WebKit2GTK) + // that bracketed every preview frame in the old pipeline. + cmd.args([ + "-pix_fmt", + "rgba", + "-f", + "rawvideo", + "pipe:1", + ]); } } - cmd.args([ - "-q:v", - "3", - "-f", - "image2pipe", - "-vcodec", - "mjpeg", - "pipe:1", - ]); - // stderr null: capturing it cost a spawn+join drain thread per seek - // plus up to 150 ms of polling on EOF. hwaccel-failure messages still - // surface via the exit code path through `decode_at`'s fallback. - cmd.stdout(Stdio::piped()).stderr(Stdio::null()); + // stderr is captured (was /dev/null) into a non-blocking drain thread + // that pushes lines into shared.state.stderr_tail. Bounded buffer, no + // join on EOF — the earlier 150ms wait-for-stderr-tail problem was the + // synchronous WAIT, not the capture itself. + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = cmd.spawn().map_err(|e| format!("ffmpeg spawn: {e}"))?; let stdout = child .stdout .take() .ok_or_else(|| "ffmpeg stdout unavailable".to_string())?; - Ok((child, BufReader::with_capacity(READ_BUF_BYTES, stdout))) + let stderr = child + .stderr + .take() + .ok_or_else(|| "ffmpeg stderr unavailable".to_string())?; + Ok(( + child, + BufReader::with_capacity(READ_BUF_BYTES, stdout), + stderr, + )) } -fn worker_loop(mut stdout: BufReader, shared: Arc) { +/// Read ffmpeg stderr line-by-line into a bounded ring buffer on shared +/// state. Exits when ffmpeg closes the pipe (EOF) or the read errors. +/// Cheap: no blocking on the main thread, used only to surface diagnostic +/// info when something has gone wrong. +fn drain_stderr(stderr: ChildStderr, shared: Arc) { + let reader = std::io::BufReader::new(stderr); + for line_res in reader.lines() { + let line = match line_res { + Ok(l) => l, + Err(_) => return, + }; + let trimmed = line.trim().to_string(); + if trimmed.is_empty() { + continue; + } + let mut st = match shared.state.lock() { + Ok(g) => g, + Err(_) => return, + }; + st.stderr_tail.push_back(trimmed); + while st.stderr_tail.len() > STDERR_TAIL_LINES { + st.stderr_tail.pop_front(); + } + } +} + +fn worker_loop( + mut stdout: BufReader, + shared: Arc, + format: OutputFormat, + dims: (u32, u32), +) { + // Pre-compute the exact frame size for the Rgba path so we don't pay + // for marker scanning that doesn't exist in raw video. + let raw_frame_bytes = (dims.0 as usize) * (dims.1 as usize) * 4; loop { - // Backpressure: wait if we're already far enough ahead. { let st = shared.state.lock().unwrap(); if st.shutdown { @@ -413,7 +582,12 @@ fn worker_loop(mut stdout: BufReader, shared: Arc) { } } - match read_jpeg_bytes(&mut stdout) { + let read_result = match format { + OutputFormat::Mjpeg => read_jpeg_bytes(&mut stdout), + OutputFormat::Rgba => read_raw_frame(&mut stdout, raw_frame_bytes, dims), + }; + + match read_result { Ok(bytes) => { let mut st = shared.state.lock().unwrap(); if st.shutdown { @@ -438,6 +612,37 @@ fn worker_loop(mut stdout: BufReader, shared: Arc) { } } +/// Read exactly one raw RGBA frame from the pipe and prefix it with +/// `[width:u32 LE, height:u32 LE]`. The JS worker then knows the +/// dimensions without having to track them out-of-band. Returns +/// `Err("pipe closed")` (or "EOF") so the worker loop's EOF detection +/// branch fires cleanly. +fn read_raw_frame( + reader: &mut BufReader, + frame_bytes: usize, + dims: (u32, u32), +) -> Result, String> { + use std::io::Read; + let mut out = Vec::with_capacity(8 + frame_bytes); + out.extend_from_slice(&dims.0.to_le_bytes()); + out.extend_from_slice(&dims.1.to_le_bytes()); + out.resize(8 + frame_bytes, 0); + let buf = &mut out[8..]; + let mut filled = 0usize; + while filled < frame_bytes { + match reader.read(&mut buf[filled..]) { + Ok(0) => { + return Err(format!( + "ffmpeg pipe closed mid-frame ({filled}/{frame_bytes} bytes)" + )); + } + Ok(n) => filled += n, + Err(e) => return Err(format!("pipe read: {e}")), + } + } + Ok(out) +} + /// Read one JPEG (SOI 0xFF 0xD8 .. EOI 0xFF 0xD9) from an image2pipe stream. /// /// Chunked scan: for each `fill_buf` chunk we vector-scan for the marker diff --git a/sam-tool-tauri/src-tauri/src/lib.rs b/sam-tool-tauri/src-tauri/src/lib.rs index 8b8f85a..65806dc 100644 --- a/sam-tool-tauri/src-tauri/src/lib.rs +++ b/sam-tool-tauri/src-tauri/src/lib.rs @@ -10,7 +10,7 @@ mod stream; mod user; mod video; -use decoder::DecoderSession; +use decoder::{DecoderSession, OutputFormat}; use serde::Serialize; use state::AppState; use std::collections::HashMap; @@ -41,12 +41,28 @@ fn open_video( .filter(|s| !s.is_empty()); let preview_max_height = settings.as_ref().map(|s| s.preview_max_height); - // Two sessions: - // - preview: may downscale via -vf scale=-2:H for cheap playback. - // - extract: always full resolution; what extract_one writes to disk. - // Both spawn ffmpeg lazily — opening a video is still cheap. - let preview = DecoderSession::open(p.clone(), info.clone(), hwaccel.clone(), preview_max_height); - let extract = DecoderSession::open(p, info.clone(), hwaccel, None); + // Two sessions, two formats: + // - preview: raw RGBA frames so JS skips the JPEG-decode round-trip + // that bracketed every preview frame in the old pipeline. May also + // downscale via `-s WxH` for cheap playback when preview_max_height + // is set. + // - extract: MJPEG output (full resolution) so `extract_one` writes + // .jpg files unchanged. + // Both spawn ffmpeg lazily. + let preview = DecoderSession::open( + p.clone(), + info.clone(), + hwaccel.clone(), + preview_max_height, + OutputFormat::Rgba, + ); + let extract = DecoderSession::open( + p, + info.clone(), + hwaccel, + None, + OutputFormat::Mjpeg, + ); if let Some(s) = settings.as_ref() { preview.set_cache_capacity(s.cache_capacity); extract.set_cache_capacity(s.cache_capacity); @@ -109,6 +125,19 @@ fn import_coco(path: String) -> Result { coco::import(p) } +/// Drain the most recent diagnostic notice from the preview decoder (e.g. +/// "Hardware decode 'cuda' failed; falling back to software"). One-shot: +/// a subsequent call returns None until the next event sets it. Frontend +/// is expected to poll this after first decode and surface as a toast. +#[tauri::command] +fn take_decoder_notice(state: State<'_, AppState>) -> Result, String> { + let guard = state + .preview_decoder + .lock() + .map_err(|e| format!("lock: {e}"))?; + Ok(guard.as_ref().and_then(|d| d.take_notice())) +} + #[tauri::command] fn set_cache_capacity(state: State<'_, AppState>, capacity: usize) -> Result { let clamped = capacity.clamp(decoder::MIN_CACHE_CAPACITY, decoder::MAX_CACHE_CAPACITY); @@ -336,6 +365,7 @@ pub fn run() { close_video, current_video, decode_frame, + take_decoder_notice, set_cache_capacity, import_coco, extract_frame, diff --git a/sam-tool-tauri/src/ipc.ts b/sam-tool-tauri/src/ipc.ts index b21c760..1beea8b 100644 --- a/sam-tool-tauri/src/ipc.ts +++ b/sam-tool-tauri/src/ipc.ts @@ -23,6 +23,9 @@ export const api = { currentVideo: () => invoke("current_video"), decodeFrame: (frameIdx: number) => invoke("decode_frame", { frameIdx }), + /** One-shot diagnostic notice (e.g. "Hardware decode 'cuda' failed; + * falling back to software"). Returns null if nothing pending. */ + takeDecoderNotice: () => invoke("take_decoder_notice"), setCacheCapacity: (capacity: number) => invoke("set_cache_capacity", { capacity }), diff --git a/sam-tool-tauri/src/modes/ExtractMode.tsx b/sam-tool-tauri/src/modes/ExtractMode.tsx index 52993cd..26d86d8 100644 --- a/sam-tool-tauri/src/modes/ExtractMode.tsx +++ b/sam-tool-tauri/src/modes/ExtractMode.tsx @@ -62,6 +62,25 @@ export function ExtractMode({ username, settings }: Props) { useEffect(() => { speedRef.current = speed; }, [speed]); useEffect(() => { reviewModeRef.current = reviewMode; }, [reviewMode]); + // Wall-clock anchor for VLC-style playback. target = anchorIdx + elapsed * + // fps * speed. Must be re-pegged whenever the user changes speed mid-play + // or seeks during play; otherwise a 1× → 2× bump after 10s would compute + // target relative to the ORIGINAL anchor, leaping forward by elapsed × Δspeed + // frames in a single tick (and the symmetric symptom on speed-down + seek). + const anchorTimeRef = useRef(0); + const anchorIdxRef = useRef(0); + const rebaseAnchor = useCallback(() => { + anchorTimeRef.current = performance.now(); + anchorIdxRef.current = frameIdxRef.current; + }, []); + // Re-peg on speed change WHILE PLAYING. If paused, the next play will peg + // anew. We don't watch reviewMode/frame_skip because skip rounding is + // resilient to mid-play changes (worst case: one tick of misalignment). + useEffect(() => { + if (playing) rebaseAnchor(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [speed]); + const [coco, setCoco] = useState(null); // Extraction state @@ -106,6 +125,12 @@ export function ExtractMode({ username, settings }: Props) { // stuck in "playing" while the canvas freezes. const decodeErrorRef = useRef(null); const [playbackError, setPlaybackError] = useState(null); + // Set after the first decode following a video open. We poll the backend + // once for a hwaccel-fallback notice (e.g. "Hardware decode 'cuda' failed; + // running software") so the user can SEE that their selection didn't + // actually engage, instead of testing three identically-behaving options. + const noticeCheckedRef = useRef(false); + const [decoderNotice, setDecoderNotice] = useState(null); const clearBitmapCache = useCallback(() => { for (const bitmap of bitmapCacheRef.current.values()) { @@ -248,6 +273,15 @@ export function ExtractMode({ username, settings }: Props) { if (drew) { renderedIdx.current = want; decodeErrorRef.current = null; + // One-shot hwaccel-fallback notice poll. Runs after the first + // successful decode, which is when ffmpeg has finished its + // hwaccel init attempt. Fire-and-forget; the IPC is cheap. + if (!noticeCheckedRef.current) { + noticeCheckedRef.current = true; + api.takeDecoderNotice().then((notice) => { + if (notice) setDecoderNotice(notice); + }).catch(() => {}); + } } // !drew means renderFrame was preempted by a newer request; loop // continues and the new target gets a fresh decode. @@ -345,40 +379,33 @@ export function ExtractMode({ username, settings }: Props) { return () => ro.disconnect(); }, [video, drawBitmap, getCachedBitmap, pump, settings?.preview_max_height]); - // --- playback (invoke-pull, every-frame) --------------------------------- - // SAM-Tool is an EXTRACTION tool: the user is reviewing frames to pick - // which to extract, so we MUST display every frame in order — dropping - // any frame defeats the use case. That rules out Channel-push (which - // emits at wall-clock rate and drops frames when the consumer lags). - // - // Architecture instead: classic pull loop. The loop sets requestedIdx to - // the next frame and awaits `pump()`. pump returns the instant the frame - // is on the canvas (single-flight, no rAF polling). We then schedule the - // next iteration with setTimeout to hit the target wall-clock interval. - // If decode/render exceeds the interval, the loop simply runs slower — - // every frame still painted. Effective playback rate is - // min(fps × speed, decode_capacity). The user can lower the - // preview-quality preset to raise decode_capacity on slow machines. - // - // The React-reconciliation fix is preserved: setFrameIdx is NOT called - // per frame. frameIdxRef.current is the authoritative cursor, and the - // 10 Hz interval below syncs it to state for the slider/readout. speed, - // reviewMode, and frame_skip are read via refs so toggling them mid-play - // doesn't tear down the loop. + // --- playback (realtime preview) ----------------------------------------- + // VLC-like playback: wall-clock time is authoritative and the decode pump + // coalesces to the latest requested frame. If decode/render cannot keep up, + // intermediate frames are skipped so video remains playable. Step/extract + // still use exact frame requests. useEffect(() => { if (!playing || !video) return; let cancelled = false; - let nextDueAt = performance.now(); + let raf = 0; + rebaseAnchor(); - const loop = async () => { - while (!cancelled && playing) { - const skip = Math.max(1, frameSkipRef.current | 0); - const current = frameIdxRef.current; - let target = Math.min(video.total_frames - 1, current + skip); - if (target <= current) { - setPlaying(false); - return; - } + const tick = () => { + if (cancelled) return; + if (decodeErrorRef.current) { + setPlaybackError(decodeErrorRef.current); + setPlaying(false); + return; + } + + const skip = Math.max(1, frameSkipRef.current | 0); + const elapsed = (performance.now() - anchorTimeRef.current) / 1000; + const rawAdvance = Math.floor(elapsed * video.fps * speedRef.current); + const advance = Math.floor(rawAdvance / skip) * skip; + let target = Math.min(video.total_frames - 1, anchorIdxRef.current + advance); + const current = frameIdxRef.current; + + if (target > current) { // Review-mode scan: with skip > 1 a naive +skip jump can leap past // annotated frames between current+1 and current+skip. Walk that // window and clamp `target` down to the first annotated frame. @@ -391,16 +418,10 @@ export function ExtractMode({ username, settings }: Props) { } } requestedIdx.current = target; - await pump(); - if (cancelled) return; - if (decodeErrorRef.current) { - setPlaybackError(decodeErrorRef.current); - setPlaying(false); - return; - } - // Cursor is ref-only here — the 10 Hz syncer below batches updates - // to React state, so we don't pay the reconciliation cost per frame. + // Cursor is ref-only here; the 10 Hz syncer below batches updates to + // React state, so playback does not pay reconciliation cost per frame. frameIdxRef.current = target; + pump(); if (reviewModeRef.current && annotatedFrames.current.has(target)) { // Annotated stop — flush state synchronously so the slider lands @@ -409,26 +430,20 @@ export function ExtractMode({ username, settings }: Props) { setPlaying(false); return; } - - const advanced = Math.max(1, target - current); - const interval = (advanced * 1000) / (video.fps * speedRef.current); - const now = performance.now(); - const due = nextDueAt + interval; - if (due > now) { - await new Promise((r) => setTimeout(r, due - now)); - nextDueAt = due; - } else { - // Decode took longer than the target interval; reset baseline - // instead of accumulating debt. Playback runs slower but no - // frames are dropped. - nextDueAt = now; - } } + + if (target >= video.total_frames - 1) { + setPlaying(false); + return; + } + raf = requestAnimationFrame(tick); }; - loop(); + + raf = requestAnimationFrame(tick); return () => { cancelled = true; + cancelAnimationFrame(raf); // Sync the ref into state so the slider/readout aren't stale after // pause. setFrameIdx(frameIdxRef.current); @@ -457,6 +472,11 @@ export function ExtractMode({ username, settings }: Props) { const clamped = Math.max(0, Math.min(video.total_frames - 1, idx)); frameIdxRef.current = clamped; requestedIdx.current = clamped; + // If a play loop is active, re-peg the wall-clock anchor to here so the + // next tick doesn't compute target relative to the pre-seek position + // (which would either yank the user forward or stall until wall-clock + // catches up). + if (playing) rebaseAnchor(); const cached = getCachedBitmap(clamped); if (cached) { drawBitmap(cached, clamped); @@ -465,7 +485,7 @@ export function ExtractMode({ username, settings }: Props) { pump(); } setFrameIdx((prev) => (prev === clamped ? prev : clamped)); - }, [drawBitmap, getCachedBitmap, pump, video, playing]); + }, [drawBitmap, getCachedBitmap, pump, video, playing, rebaseAnchor]); const stepFrames = useCallback((delta: number) => { if (!video) return; @@ -716,6 +736,9 @@ export function ExtractMode({ username, settings }: Props) { renderedIdx.current = -1; decodeErrorRef.current = null; setPlaybackError(null); + // Fresh decoder session — re-arm the one-shot notice probe. + noticeCheckedRef.current = false; + setDecoderNotice(null); // Reset per-video extraction context, then auto-adopt the per-video // default folder. If the user already extracted frames into it in a // prior session, listExtractedFrames rehydrates them immediately. @@ -952,6 +975,16 @@ export function ExtractMode({ username, settings }: Props) {

)} + + {decoderNotice && ( +

+ ⚠ {decoderNotice} + {" "} + +

+ )} ); } diff --git a/sam-tool-tauri/src/services/frameDecoderPool.ts b/sam-tool-tauri/src/services/frameDecoderPool.ts index e9f2014..bcd19cb 100644 --- a/sam-tool-tauri/src/services/frameDecoderPool.ts +++ b/sam-tool-tauri/src/services/frameDecoderPool.ts @@ -1,25 +1,47 @@ -// Worker that takes a packed-frame ArrayBuffer (8-byte LE i64 frame number + -// JPEG bytes), decodes the JPEG to an ImageBitmap off the main thread, and -// returns both via transferable postMessage so the main thread only does the -// final drawImage. Inlined as a Blob URL so we don't need a separate worker -// build entry. +// Worker that decodes a packed-frame ArrayBuffer to an ImageBitmap off the +// main thread. Wire format (LE): +// [0..8) i64 frame number +// [8..12) u32 width (RGBA wire only) +// [12..16) u32 height (RGBA wire only) +// [16..) width*height*4 bytes of RGBA pixels // -// Two perf details worth keeping: -// * We do NOT copy the JPEG payload before constructing the Blob. A -// Uint8Array view onto the transferred buffer is enough — the Blob -// takes ownership of the bytes without an intermediate allocation, -// saving ~10 MB/s of GC pressure at 30 fps. -// * The worker is lazy-initialised on first decode. Mounting ExtractMode -// no longer pays a worker spin-up cost. +// The previous pipeline was [frame#, JPEG bytes] — that JPEG had to be +// software-decoded by libjpeg-turbo (~3-8 ms per 720p frame on WebKit2GTK) +// before it could become an ImageBitmap. With raw RGBA we feed pixels +// straight into ImageData and skip the JPEG round-trip entirely; the cost +// drops to a single memcpy from the transferred buffer into the bitmap. +// Inlined as a Blob URL so we don't need a separate worker build entry. + +// Skip optional per-frame processing the browser would otherwise do: +// - colorSpaceConversion 'none': no ICC profile in ffmpeg raw output. +// - imageOrientation 'none': no EXIF rotation in our output. +// - premultiplyAlpha 'none': source is already RGBA from ffmpeg. +const BITMAP_OPTS = `{ + colorSpaceConversion: "none", + imageOrientation: "none", + premultiplyAlpha: "none", +}`; const WORKER_SOURCE = ` self.onmessage = async (e) => { const { id, buffer } = e.data; let frameNumber = -1; try { - frameNumber = Number(new BigInt64Array(buffer, 0, 1)[0]); - const blob = new Blob([new Uint8Array(buffer, 8)], { type: "image/jpeg" }); - const bitmap = await createImageBitmap(blob); + const view = new DataView(buffer); + frameNumber = Number(view.getBigInt64(0, true)); + const width = view.getUint32(8, true); + const height = view.getUint32(12, true); + if (width <= 0 || height <= 0) { + throw new Error("bad frame header w=" + width + " h=" + height); + } + const need = width * height * 4; + const have = buffer.byteLength - 16; + if (have !== need) { + throw new Error("rgba size mismatch want=" + need + " got=" + have); + } + const pixels = new Uint8ClampedArray(buffer, 16, need); + const imageData = new ImageData(pixels, width, height); + const bitmap = await createImageBitmap(imageData, ${BITMAP_OPTS}); self.postMessage({ id, frameNumber, bitmap }, [bitmap]); } catch (err) { const msg = err && err.message ? err.message : String(err);