playback

sam-tool-tauri/src-tauri/src/decoder.rs

@@ -2,15 +2,37 @@ use crate::video::VideoInfo;
 use std::collections::{HashMap, VecDeque};
 use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
-use std::process::{Child, ChildStdout, Command, Stdio};
+use std::process::{Child, ChildStderr, ChildStdout, Command, Stdio};
 use std::sync::{Arc, Condvar, Mutex};
 use std::thread::{self, JoinHandle};
 use std::time::{Duration, Instant};
 
+/// Cap on stderr lines retained per ffmpeg process. The hwaccel-init message
+/// is usually 1-3 lines; we keep more to cover later-frame errors too.
+const STDERR_TAIL_LINES: usize = 32;
+
 pub const DEFAULT_CACHE_CAPACITY: usize = 160;
 pub const MIN_CACHE_CAPACITY: usize = 50;
 pub const MAX_CACHE_CAPACITY: usize = 800;
+/// Cache cap for the RGBA preview decoder. Raw frames are 30-80× bigger
+/// than MJPEG so we cap depth for memory headroom. Sized for 1080p (the
+/// most common case): 16 frames × 8.3 MB ≈ 133 MB. At 720p this drops to
+/// ~45 MB. At 4K it's ~528 MB, so users with 4K sources should set
+/// preview_max_height in settings.
+pub const RGBA_PREVIEW_CACHE_CAPACITY: usize = 16;
+pub const RGBA_PREVIEW_LOOKAHEAD: u64 = 8;
 const READ_BUF_BYTES: usize = 256 * 1024;
+
+/// Wire format produced by an ffmpeg child. Preview decoder uses Rgba so
+/// the JS side avoids a CPU MJPEG decode per frame (which was the real
+/// per-frame bottleneck — see review findings A + E). Extract decoder
+/// stays on Mjpeg because `extract_one` writes the decoder output to disk
+/// and users expect .jpg files.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum OutputFormat {
+    Mjpeg,
+    Rgba,
+}
 /// If the requested frame is this far ahead of the worker, kill + reseek.
 /// Higher = fewer ffmpeg restarts during forward scan; the cap is bounded
 /// by lookahead × cache capacity anyway.
@@ -86,6 +108,10 @@ struct SharedState {
     eof: bool,
     error: Option<String>,
     shutdown: bool,
+    /// Last N stderr lines from the most recent ffmpeg child. Populated by
+    /// a non-blocking drain thread; read only when something has gone wrong
+    /// (e.g. hwaccel fallback) to surface the ffmpeg diagnostic.
+    stderr_tail: VecDeque<String>,
 }
 
 struct Shared {
@@ -103,9 +129,19 @@ pub struct DecoderSession {
     /// When Some(H), ffmpeg downscales to height H (aspect preserved) via
     /// `-vf scale=-2:H`. Used by the playback decoder for cheap previews;
     /// the extract decoder always passes None so saved frames are full-res.
-    preview_max_height: Option<u32>,
+    format: OutputFormat,
+    /// Pre-computed (width, height) ffmpeg will emit. For Mjpeg this is the
+    /// source dims (mjpeg encoder doesn't care). For Rgba these MUST be
+    /// even and exact, since the reader frames input by `out_w * out_h * 4`
+    /// byte counts — there's no marker to find frame boundaries in raw.
+    output_dims: (u32, u32),
     shared: Arc<Shared>,
     worker: Option<JoinHandle<()>>,
+    /// One-shot notice surfaced via `take_notice` after a hwaccel fallback.
+    /// Lets the frontend show "Hardware decode 'cuda' failed; running
+    /// software" instead of silently testing the same code path under three
+    /// different settings.
+    last_notice: Mutex<Option<String>>,
 }
 
 impl DecoderSession {
@@ -114,26 +150,61 @@ impl DecoderSession {
         info: VideoInfo,
         hwaccel: Option<String>,
         preview_max_height: Option<u32>,
+        format: OutputFormat,
     ) -> Self {
+        let preview_max_height = preview_max_height.filter(|h| *h > 0);
+        let output_dims = compute_output_dims(&info, preview_max_height);
+        // RGBA frames are huge (1080p = 8.3 MB), so a 160-entry cache would
+        // be ~1.3 GB. Cap depth and lookahead for the Rgba path; Mjpeg
+        // keeps its existing generous defaults.
+        let (init_cap, init_lookahead) = match format {
+            OutputFormat::Mjpeg => (
+                DEFAULT_CACHE_CAPACITY,
+                (DEFAULT_CACHE_CAPACITY as u64) / 2,
+            ),
+            OutputFormat::Rgba => (RGBA_PREVIEW_CACHE_CAPACITY, RGBA_PREVIEW_LOOKAHEAD),
+        };
         Self {
             path,
             info,
             hwaccel: hwaccel.filter(|s| !s.is_empty()),
-            preview_max_height: preview_max_height.filter(|h| *h > 0),
+            format,
+            output_dims,
             shared: Arc::new(Shared {
                 state: Mutex::new(SharedState {
                     child: None,
-                    cache: FrameCache::new(DEFAULT_CACHE_CAPACITY),
+                    cache: FrameCache::new(init_cap),
                     next_idx: 0,
                     reader_idx: 0,
-                    lookahead: (DEFAULT_CACHE_CAPACITY as u64) / 2,
+                    lookahead: init_lookahead,
                     eof: false,
                     error: None,
                     shutdown: false,
+                    stderr_tail: VecDeque::new(),
                 }),
                 cond: Condvar::new(),
             }),
             worker: None,
+            last_notice: Mutex::new(None),
+        }
+    }
+
+    /// Drain and return the most recent fallback / diagnostic notice (if any).
+    /// One-shot: a subsequent call returns None until the next event sets it.
+    pub fn take_notice(&self) -> Option<String> {
+        self.last_notice.lock().ok().and_then(|mut n| n.take())
+    }
+
+    fn snapshot_stderr_tail(&self) -> String {
+        let st = self.shared.state.lock().unwrap();
+        if st.stderr_tail.is_empty() {
+            "(no ffmpeg stderr captured)".into()
+        } else {
+            st.stderr_tail
+                .iter()
+                .cloned()
+                .collect::<Vec<_>>()
+                .join("\n")
         }
     }
 
@@ -142,6 +213,11 @@ impl DecoderSession {
     }
 
     pub fn set_cache_capacity(&self, capacity: usize) {
+        // RGBA preview cap is fixed by memory budget — a user-set 800-frame
+        // cache at 1080p RGBA would be ~6.6 GB.
+        if self.format == OutputFormat::Rgba {
+            return;
+        }
         let clamped = capacity.clamp(MIN_CACHE_CAPACITY, MAX_CACHE_CAPACITY);
         let mut st = self.shared.state.lock().unwrap();
         st.cache.set_capacity(clamped);
@@ -158,9 +234,14 @@ impl DecoderSession {
                 // path that worked on h264 but chokes on an HEVC segment).
                 // Once cleared, subsequent failures propagate.
                 if let Some(prev) = self.hwaccel.take() {
-                    eprintln!(
-                        "[decoder] hwaccel '{prev}' failed ({e}); falling back to software"
+                    let tail = self.snapshot_stderr_tail();
+                    let notice = format!(
+                        "Hardware decode '{prev}' failed; falling back to software.\nffmpeg said:\n{tail}"
                     );
+                    eprintln!("[decoder] {notice}");
+                    if let Ok(mut slot) = self.last_notice.lock() {
+                        *slot = Some(notice);
+                    }
                     self.stop_worker();
                     self.decode_at_inner(idx)
                 } else {
@@ -233,16 +314,18 @@ impl DecoderSession {
     fn seek_and_start_worker(&mut self, idx: u64) -> Result<(), String> {
         self.stop_worker();
 
-        let (child, stdout) = spawn_ffmpeg(
+        let (child, stdout, stderr) = spawn_ffmpeg(
             &self.path,
             &self.info,
             idx,
             self.hwaccel.as_deref(),
-            self.preview_max_height,
+            self.format,
+            self.output_dims,
         )?;
 
         {
             let mut st = self.shared.state.lock().unwrap();
+            st.stderr_tail.clear();
             st.child = Some(child);
             st.next_idx = idx;
             st.reader_idx = idx;
@@ -252,9 +335,14 @@ impl DecoderSession {
         }
         self.shared.cond.notify_all();
 
+        let drain_shared = self.shared.clone();
+        thread::spawn(move || drain_stderr(stderr, drain_shared));
+
+        let format = self.format;
+        let dims = self.output_dims;
         let shared = self.shared.clone();
         let handle = thread::spawn(move || {
-            worker_loop(stdout, shared);
+            worker_loop(stdout, shared, format, dims);
         });
         self.worker = Some(handle);
         Ok(())
@@ -324,13 +412,37 @@ fn vaapi_device() -> String {
     "/dev/dri/renderD128".to_string()
 }
 
+/// Compute the (width, height) ffmpeg will output. Scales down to
+/// preview_max_height only when source is taller; preserves aspect via a
+/// rounded-to-even width. Returns source dims when no scaling applies.
+/// The reader uses these to frame raw video output, so they MUST match
+/// what ffmpeg actually emits — we set them explicitly via `-s WxH` in
+/// the Rgba path to take the guesswork out.
+fn compute_output_dims(info: &VideoInfo, preview_max_height: Option<u32>) -> (u32, u32) {
+    if let Some(h) = preview_max_height {
+        if info.height > h && info.height > 0 {
+            // Maintain aspect; round width to nearest even (ffmpeg requires
+            // even dims for many filters).
+            let w = ((info.width as u64 * h as u64) + (info.height as u64 / 2))
+                / (info.height as u64);
+            let w_even = ((w as u32) + 1) & !1;
+            return (w_even.max(2), h);
+        }
+    }
+    // Even source dims for safety (almost always already even).
+    let w = (info.width + 1) & !1;
+    let h = (info.height + 1) & !1;
+    (w.max(2), h.max(2))
+}
+
 fn spawn_ffmpeg(
     path: &PathBuf,
     info: &VideoInfo,
     idx: u64,
     hwaccel: Option<&str>,
-    preview_max_height: Option<u32>,
-) -> Result<(Child, BufReader<ChildStdout>), String> {
+    format: OutputFormat,
+    out_dims: (u32, u32),
+) -> Result<(Child, BufReader<ChildStdout>, ChildStderr), String> {
     let path_str = path.to_str().ok_or_else(|| "non-utf8 path".to_string())?;
     let t = idx as f64 / info.fps;
     let mut cmd = Command::new("ffmpeg");
@@ -362,39 +474,96 @@ fn spawn_ffmpeg(
     cmd.args(["-ss", &format!("{:.6}", t)]);
     cmd.args(["-i", path_str]);
     cmd.args(["-an"]);
-    // Preview scaling: cap output height to `h`, width auto (`-2` keeps
-    // aspect & ensures even). Skip the filter entirely when the source is
-    // already shorter than the cap — no upscale, no wasted filter step.
-    if let Some(h) = preview_max_height {
-        if info.height > h {
-            let vf = format!("scale=-2:{h}");
-            cmd.args(["-vf", &vf]);
+    // Pin output dimensions exactly. For Mjpeg this is cosmetic (matches
+    // source); for Rgba it MUST match what the reader expects to byte-frame
+    // raw video output.
+    let (ow, oh) = out_dims;
+    if ow != info.width || oh != info.height {
+        cmd.args(["-s", &format!("{ow}x{oh}")]);
+    }
+    match format {
+        OutputFormat::Mjpeg => {
+            cmd.args([
+                "-q:v",
+                "3",
+                "-f",
+                "image2pipe",
+                "-vcodec",
+                "mjpeg",
+                "pipe:1",
+            ]);
+        }
+        OutputFormat::Rgba => {
+            // Raw RGBA, no entropy coding. Worker grabs the bytes verbatim
+            // into ImageData, eliminating the CPU MJPEG encode (ffmpeg)
+            // and the CPU JPEG decode (createImageBitmap on WebKit2GTK)
+            // that bracketed every preview frame in the old pipeline.
+            cmd.args([
+                "-pix_fmt",
+                "rgba",
+                "-f",
+                "rawvideo",
+                "pipe:1",
+            ]);
         }
     }
-    cmd.args([
-        "-q:v",
-        "3",
-        "-f",
-        "image2pipe",
-        "-vcodec",
-        "mjpeg",
-        "pipe:1",
-    ]);
-    // stderr null: capturing it cost a spawn+join drain thread per seek
-    // plus up to 150 ms of polling on EOF. hwaccel-failure messages still
-    // surface via the exit code path through `decode_at`'s fallback.
-    cmd.stdout(Stdio::piped()).stderr(Stdio::null());
+    // stderr is captured (was /dev/null) into a non-blocking drain thread
+    // that pushes lines into shared.state.stderr_tail. Bounded buffer, no
+    // join on EOF — the earlier 150ms wait-for-stderr-tail problem was the
+    // synchronous WAIT, not the capture itself.
+    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
     let mut child = cmd.spawn().map_err(|e| format!("ffmpeg spawn: {e}"))?;
     let stdout = child
         .stdout
         .take()
         .ok_or_else(|| "ffmpeg stdout unavailable".to_string())?;
-    Ok((child, BufReader::with_capacity(READ_BUF_BYTES, stdout)))
+    let stderr = child
+        .stderr
+        .take()
+        .ok_or_else(|| "ffmpeg stderr unavailable".to_string())?;
+    Ok((
+        child,
+        BufReader::with_capacity(READ_BUF_BYTES, stdout),
+        stderr,
+    ))
 }
 
-fn worker_loop(mut stdout: BufReader<ChildStdout>, shared: Arc<Shared>) {
+/// Read ffmpeg stderr line-by-line into a bounded ring buffer on shared
+/// state. Exits when ffmpeg closes the pipe (EOF) or the read errors.
+/// Cheap: no blocking on the main thread, used only to surface diagnostic
+/// info when something has gone wrong.
+fn drain_stderr(stderr: ChildStderr, shared: Arc<Shared>) {
+    let reader = std::io::BufReader::new(stderr);
+    for line_res in reader.lines() {
+        let line = match line_res {
+            Ok(l) => l,
+            Err(_) => return,
+        };
+        let trimmed = line.trim().to_string();
+        if trimmed.is_empty() {
+            continue;
+        }
+        let mut st = match shared.state.lock() {
+            Ok(g) => g,
+            Err(_) => return,
+        };
+        st.stderr_tail.push_back(trimmed);
+        while st.stderr_tail.len() > STDERR_TAIL_LINES {
+            st.stderr_tail.pop_front();
+        }
+    }
+}
+
+fn worker_loop(
+    mut stdout: BufReader<ChildStdout>,
+    shared: Arc<Shared>,
+    format: OutputFormat,
+    dims: (u32, u32),
+) {
+    // Pre-compute the exact frame size for the Rgba path so we don't pay
+    // for marker scanning that doesn't exist in raw video.
+    let raw_frame_bytes = (dims.0 as usize) * (dims.1 as usize) * 4;
     loop {
-        // Backpressure: wait if we're already far enough ahead.
         {
             let st = shared.state.lock().unwrap();
             if st.shutdown {
@@ -413,7 +582,12 @@ fn worker_loop(mut stdout: BufReader<ChildStdout>, shared: Arc<Shared>) {
             }
         }
 
-        match read_jpeg_bytes(&mut stdout) {
+        let read_result = match format {
+            OutputFormat::Mjpeg => read_jpeg_bytes(&mut stdout),
+            OutputFormat::Rgba => read_raw_frame(&mut stdout, raw_frame_bytes, dims),
+        };
+
+        match read_result {
             Ok(bytes) => {
                 let mut st = shared.state.lock().unwrap();
                 if st.shutdown {
@@ -438,6 +612,37 @@ fn worker_loop(mut stdout: BufReader<ChildStdout>, shared: Arc<Shared>) {
     }
 }
 
+/// Read exactly one raw RGBA frame from the pipe and prefix it with
+/// `[width:u32 LE, height:u32 LE]`. The JS worker then knows the
+/// dimensions without having to track them out-of-band. Returns
+/// `Err("pipe closed")` (or "EOF") so the worker loop's EOF detection
+/// branch fires cleanly.
+fn read_raw_frame(
+    reader: &mut BufReader<ChildStdout>,
+    frame_bytes: usize,
+    dims: (u32, u32),
+) -> Result<Vec<u8>, String> {
+    use std::io::Read;
+    let mut out = Vec::with_capacity(8 + frame_bytes);
+    out.extend_from_slice(&dims.0.to_le_bytes());
+    out.extend_from_slice(&dims.1.to_le_bytes());
+    out.resize(8 + frame_bytes, 0);
+    let buf = &mut out[8..];
+    let mut filled = 0usize;
+    while filled < frame_bytes {
+        match reader.read(&mut buf[filled..]) {
+            Ok(0) => {
+                return Err(format!(
+                    "ffmpeg pipe closed mid-frame ({filled}/{frame_bytes} bytes)"
+                ));
+            }
+            Ok(n) => filled += n,
+            Err(e) => return Err(format!("pipe read: {e}")),
+        }
+    }
+    Ok(out)
+}
+
 /// Read one JPEG (SOI 0xFF 0xD8 .. EOI 0xFF 0xD9) from an image2pipe stream.
 ///
 /// Chunked scan: for each `fill_buf` chunk we vector-scan for the marker

sam-tool-tauri/src-tauri/src/lib.rs

@@ -10,7 +10,7 @@ mod stream;
 mod user;
 mod video;
 
-use decoder::DecoderSession;
+use decoder::{DecoderSession, OutputFormat};
 use serde::Serialize;
 use state::AppState;
 use std::collections::HashMap;
@@ -41,12 +41,28 @@ fn open_video(
         .filter(|s| !s.is_empty());
     let preview_max_height = settings.as_ref().map(|s| s.preview_max_height);
 
-    // Two sessions:
-    //  - preview: may downscale via -vf scale=-2:H for cheap playback.
-    //  - extract: always full resolution; what extract_one writes to disk.
-    // Both spawn ffmpeg lazily — opening a video is still cheap.
-    let preview = DecoderSession::open(p.clone(), info.clone(), hwaccel.clone(), preview_max_height);
-    let extract = DecoderSession::open(p, info.clone(), hwaccel, None);
+    // Two sessions, two formats:
+    //  - preview: raw RGBA frames so JS skips the JPEG-decode round-trip
+    //    that bracketed every preview frame in the old pipeline. May also
+    //    downscale via `-s WxH` for cheap playback when preview_max_height
+    //    is set.
+    //  - extract: MJPEG output (full resolution) so `extract_one` writes
+    //    .jpg files unchanged.
+    // Both spawn ffmpeg lazily.
+    let preview = DecoderSession::open(
+        p.clone(),
+        info.clone(),
+        hwaccel.clone(),
+        preview_max_height,
+        OutputFormat::Rgba,
+    );
+    let extract = DecoderSession::open(
+        p,
+        info.clone(),
+        hwaccel,
+        None,
+        OutputFormat::Mjpeg,
+    );
     if let Some(s) = settings.as_ref() {
         preview.set_cache_capacity(s.cache_capacity);
         extract.set_cache_capacity(s.cache_capacity);
@@ -109,6 +125,19 @@ fn import_coco(path: String) -> Result<coco::ImportedCoco, String> {
     coco::import(p)
 }
 
+/// Drain the most recent diagnostic notice from the preview decoder (e.g.
+/// "Hardware decode 'cuda' failed; falling back to software"). One-shot:
+/// a subsequent call returns None until the next event sets it. Frontend
+/// is expected to poll this after first decode and surface as a toast.
+#[tauri::command]
+fn take_decoder_notice(state: State<'_, AppState>) -> Result<Option<String>, String> {
+    let guard = state
+        .preview_decoder
+        .lock()
+        .map_err(|e| format!("lock: {e}"))?;
+    Ok(guard.as_ref().and_then(|d| d.take_notice()))
+}
+
 #[tauri::command]
 fn set_cache_capacity(state: State<'_, AppState>, capacity: usize) -> Result<usize, String> {
     let clamped = capacity.clamp(decoder::MIN_CACHE_CAPACITY, decoder::MAX_CACHE_CAPACITY);
@@ -336,6 +365,7 @@ pub fn run() {
             close_video,
             current_video,
             decode_frame,
+            take_decoder_notice,
             set_cache_capacity,
             import_coco,
             extract_frame,

sam-tool-tauri/src/ipc.ts

@@ -23,6 +23,9 @@ export const api = {
   currentVideo: () => invoke<VideoInfo | null>("current_video"),
   decodeFrame: (frameIdx: number) =>
     invoke<ArrayBuffer>("decode_frame", { frameIdx }),
+  /** One-shot diagnostic notice (e.g. "Hardware decode 'cuda' failed;
+   *  falling back to software"). Returns null if nothing pending. */
+  takeDecoderNotice: () => invoke<string | null>("take_decoder_notice"),
   setCacheCapacity: (capacity: number) =>
     invoke<number>("set_cache_capacity", { capacity }),
 

sam-tool-tauri/src/modes/ExtractMode.tsx

@@ -62,6 +62,25 @@ export function ExtractMode({ username, settings }: Props) {
   useEffect(() => { speedRef.current = speed; }, [speed]);
   useEffect(() => { reviewModeRef.current = reviewMode; }, [reviewMode]);
 
+  // Wall-clock anchor for VLC-style playback. target = anchorIdx + elapsed *
+  // fps * speed. Must be re-pegged whenever the user changes speed mid-play
+  // or seeks during play; otherwise a 1× → 2× bump after 10s would compute
+  // target relative to the ORIGINAL anchor, leaping forward by elapsed × Δspeed
+  // frames in a single tick (and the symmetric symptom on speed-down + seek).
+  const anchorTimeRef = useRef(0);
+  const anchorIdxRef = useRef(0);
+  const rebaseAnchor = useCallback(() => {
+    anchorTimeRef.current = performance.now();
+    anchorIdxRef.current = frameIdxRef.current;
+  }, []);
+  // Re-peg on speed change WHILE PLAYING. If paused, the next play will peg
+  // anew. We don't watch reviewMode/frame_skip because skip rounding is
+  // resilient to mid-play changes (worst case: one tick of misalignment).
+  useEffect(() => {
+    if (playing) rebaseAnchor();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [speed]);
+
   const [coco, setCoco] = useState<ImportedCoco | null>(null);
 
   // Extraction state
@@ -106,6 +125,12 @@ export function ExtractMode({ username, settings }: Props) {
   // stuck in "playing" while the canvas freezes.
   const decodeErrorRef = useRef<string | null>(null);
   const [playbackError, setPlaybackError] = useState<string | null>(null);
+  // Set after the first decode following a video open. We poll the backend
+  // once for a hwaccel-fallback notice (e.g. "Hardware decode 'cuda' failed;
+  // running software") so the user can SEE that their selection didn't
+  // actually engage, instead of testing three identically-behaving options.
+  const noticeCheckedRef = useRef(false);
+  const [decoderNotice, setDecoderNotice] = useState<string | null>(null);
 
   const clearBitmapCache = useCallback(() => {
     for (const bitmap of bitmapCacheRef.current.values()) {
@@ -248,6 +273,15 @@ export function ExtractMode({ username, settings }: Props) {
           if (drew) {
             renderedIdx.current = want;
             decodeErrorRef.current = null;
+            // One-shot hwaccel-fallback notice poll. Runs after the first
+            // successful decode, which is when ffmpeg has finished its
+            // hwaccel init attempt. Fire-and-forget; the IPC is cheap.
+            if (!noticeCheckedRef.current) {
+              noticeCheckedRef.current = true;
+              api.takeDecoderNotice().then((notice) => {
+                if (notice) setDecoderNotice(notice);
+              }).catch(() => {});
+            }
           }
           // !drew means renderFrame was preempted by a newer request; loop
           // continues and the new target gets a fresh decode.
@@ -345,40 +379,33 @@ export function ExtractMode({ username, settings }: Props) {
     return () => ro.disconnect();
   }, [video, drawBitmap, getCachedBitmap, pump, settings?.preview_max_height]);
 
-  // --- playback (invoke-pull, every-frame) ---------------------------------
-  // SAM-Tool is an EXTRACTION tool: the user is reviewing frames to pick
-  // which to extract, so we MUST display every frame in order — dropping
-  // any frame defeats the use case. That rules out Channel-push (which
-  // emits at wall-clock rate and drops frames when the consumer lags).
-  //
-  // Architecture instead: classic pull loop. The loop sets requestedIdx to
-  // the next frame and awaits `pump()`. pump returns the instant the frame
-  // is on the canvas (single-flight, no rAF polling). We then schedule the
-  // next iteration with setTimeout to hit the target wall-clock interval.
-  // If decode/render exceeds the interval, the loop simply runs slower —
-  // every frame still painted. Effective playback rate is
-  // min(fps × speed, decode_capacity). The user can lower the
-  // preview-quality preset to raise decode_capacity on slow machines.
-  //
-  // The React-reconciliation fix is preserved: setFrameIdx is NOT called
-  // per frame. frameIdxRef.current is the authoritative cursor, and the
-  // 10 Hz interval below syncs it to state for the slider/readout. speed,
-  // reviewMode, and frame_skip are read via refs so toggling them mid-play
-  // doesn't tear down the loop.
+  // --- playback (realtime preview) -----------------------------------------
+  // VLC-like playback: wall-clock time is authoritative and the decode pump
+  // coalesces to the latest requested frame. If decode/render cannot keep up,
+  // intermediate frames are skipped so video remains playable. Step/extract
+  // still use exact frame requests.
   useEffect(() => {
     if (!playing || !video) return;
     let cancelled = false;
-    let nextDueAt = performance.now();
+    let raf = 0;
+    rebaseAnchor();
 
-    const loop = async () => {
-      while (!cancelled && playing) {
-        const skip = Math.max(1, frameSkipRef.current | 0);
-        const current = frameIdxRef.current;
-        let target = Math.min(video.total_frames - 1, current + skip);
-        if (target <= current) {
-          setPlaying(false);
-          return;
-        }
+    const tick = () => {
+      if (cancelled) return;
+      if (decodeErrorRef.current) {
+        setPlaybackError(decodeErrorRef.current);
+        setPlaying(false);
+        return;
+      }
+
+      const skip = Math.max(1, frameSkipRef.current | 0);
+      const elapsed = (performance.now() - anchorTimeRef.current) / 1000;
+      const rawAdvance = Math.floor(elapsed * video.fps * speedRef.current);
+      const advance = Math.floor(rawAdvance / skip) * skip;
+      let target = Math.min(video.total_frames - 1, anchorIdxRef.current + advance);
+      const current = frameIdxRef.current;
+
+      if (target > current) {
         // Review-mode scan: with skip > 1 a naive +skip jump can leap past
         // annotated frames between current+1 and current+skip. Walk that
         // window and clamp `target` down to the first annotated frame.
@@ -391,16 +418,10 @@ export function ExtractMode({ username, settings }: Props) {
           }
         }
         requestedIdx.current = target;
-        await pump();
-        if (cancelled) return;
-        if (decodeErrorRef.current) {
-          setPlaybackError(decodeErrorRef.current);
-          setPlaying(false);
-          return;
-        }
-        // Cursor is ref-only here — the 10 Hz syncer below batches updates
-        // to React state, so we don't pay the reconciliation cost per frame.
+        // Cursor is ref-only here; the 10 Hz syncer below batches updates to
+        // React state, so playback does not pay reconciliation cost per frame.
         frameIdxRef.current = target;
+        pump();
 
         if (reviewModeRef.current && annotatedFrames.current.has(target)) {
           // Annotated stop — flush state synchronously so the slider lands
@@ -409,26 +430,20 @@ export function ExtractMode({ username, settings }: Props) {
           setPlaying(false);
           return;
         }
-
-        const advanced = Math.max(1, target - current);
-        const interval = (advanced * 1000) / (video.fps * speedRef.current);
-        const now = performance.now();
-        const due = nextDueAt + interval;
-        if (due > now) {
-          await new Promise((r) => setTimeout(r, due - now));
-          nextDueAt = due;
-        } else {
-          // Decode took longer than the target interval; reset baseline
-          // instead of accumulating debt. Playback runs slower but no
-          // frames are dropped.
-          nextDueAt = now;
-        }
       }
+
+      if (target >= video.total_frames - 1) {
+        setPlaying(false);
+        return;
+      }
+      raf = requestAnimationFrame(tick);
     };
-    loop();
+
+    raf = requestAnimationFrame(tick);
 
     return () => {
       cancelled = true;
+      cancelAnimationFrame(raf);
       // Sync the ref into state so the slider/readout aren't stale after
       // pause.
       setFrameIdx(frameIdxRef.current);
@@ -457,6 +472,11 @@ export function ExtractMode({ username, settings }: Props) {
     const clamped = Math.max(0, Math.min(video.total_frames - 1, idx));
     frameIdxRef.current = clamped;
     requestedIdx.current = clamped;
+    // If a play loop is active, re-peg the wall-clock anchor to here so the
+    // next tick doesn't compute target relative to the pre-seek position
+    // (which would either yank the user forward or stall until wall-clock
+    // catches up).
+    if (playing) rebaseAnchor();
     const cached = getCachedBitmap(clamped);
     if (cached) {
       drawBitmap(cached, clamped);
@@ -465,7 +485,7 @@ export function ExtractMode({ username, settings }: Props) {
       pump();
     }
     setFrameIdx((prev) => (prev === clamped ? prev : clamped));
-  }, [drawBitmap, getCachedBitmap, pump, video, playing]);
+  }, [drawBitmap, getCachedBitmap, pump, video, playing, rebaseAnchor]);
 
   const stepFrames = useCallback((delta: number) => {
     if (!video) return;
@@ -716,6 +736,9 @@ export function ExtractMode({ username, settings }: Props) {
       renderedIdx.current = -1;
       decodeErrorRef.current = null;
       setPlaybackError(null);
+      // Fresh decoder session — re-arm the one-shot notice probe.
+      noticeCheckedRef.current = false;
+      setDecoderNotice(null);
       // Reset per-video extraction context, then auto-adopt the per-video
       // default folder. If the user already extracted frames into it in a
       // prior session, listExtractedFrames rehydrates them immediately.
@@ -952,6 +975,16 @@ export function ExtractMode({ username, settings }: Props) {
           </button>
         </p>
       )}
+
+      {decoderNotice && (
+        <p className="warn" role="status" style={{ whiteSpace: "pre-wrap" }}>
+          ⚠ {decoderNotice}
+          {" "}
+          <button className="link" onClick={() => setDecoderNotice(null)}>
+            dismiss
+          </button>
+        </p>
+      )}
     </div>
   );
 }

sam-tool-tauri/src/services/frameDecoderPool.ts

@@ -1,25 +1,47 @@
-// Worker that takes a packed-frame ArrayBuffer (8-byte LE i64 frame number +
-// JPEG bytes), decodes the JPEG to an ImageBitmap off the main thread, and
-// returns both via transferable postMessage so the main thread only does the
-// final drawImage. Inlined as a Blob URL so we don't need a separate worker
-// build entry.
+// Worker that decodes a packed-frame ArrayBuffer to an ImageBitmap off the
+// main thread. Wire format (LE):
+//   [0..8)    i64 frame number
+//   [8..12)   u32 width   (RGBA wire only)
+//   [12..16)  u32 height  (RGBA wire only)
+//   [16..)    width*height*4 bytes of RGBA pixels
 //
-// Two perf details worth keeping:
-//  * We do NOT copy the JPEG payload before constructing the Blob. A
-//    Uint8Array view onto the transferred buffer is enough — the Blob
-//    takes ownership of the bytes without an intermediate allocation,
-//    saving ~10 MB/s of GC pressure at 30 fps.
-//  * The worker is lazy-initialised on first decode. Mounting ExtractMode
-//    no longer pays a worker spin-up cost.
+// The previous pipeline was [frame#, JPEG bytes] — that JPEG had to be
+// software-decoded by libjpeg-turbo (~3-8 ms per 720p frame on WebKit2GTK)
+// before it could become an ImageBitmap. With raw RGBA we feed pixels
+// straight into ImageData and skip the JPEG round-trip entirely; the cost
+// drops to a single memcpy from the transferred buffer into the bitmap.
+// Inlined as a Blob URL so we don't need a separate worker build entry.
+
+// Skip optional per-frame processing the browser would otherwise do:
+//  - colorSpaceConversion 'none': no ICC profile in ffmpeg raw output.
+//  - imageOrientation 'none': no EXIF rotation in our output.
+//  - premultiplyAlpha 'none': source is already RGBA from ffmpeg.
+const BITMAP_OPTS = `{
+  colorSpaceConversion: "none",
+  imageOrientation: "none",
+  premultiplyAlpha: "none",
+}`;
 
 const WORKER_SOURCE = `
 self.onmessage = async (e) => {
   const { id, buffer } = e.data;
   let frameNumber = -1;
   try {
-    frameNumber = Number(new BigInt64Array(buffer, 0, 1)[0]);
-    const blob = new Blob([new Uint8Array(buffer, 8)], { type: "image/jpeg" });
-    const bitmap = await createImageBitmap(blob);
+    const view = new DataView(buffer);
+    frameNumber = Number(view.getBigInt64(0, true));
+    const width = view.getUint32(8, true);
+    const height = view.getUint32(12, true);
+    if (width <= 0 || height <= 0) {
+      throw new Error("bad frame header w=" + width + " h=" + height);
+    }
+    const need = width * height * 4;
+    const have = buffer.byteLength - 16;
+    if (have !== need) {
+      throw new Error("rgba size mismatch want=" + need + " got=" + have);
+    }
+    const pixels = new Uint8ClampedArray(buffer, 16, need);
+    const imageData = new ImageData(pixels, width, height);
+    const bitmap = await createImageBitmap(imageData, ${BITMAP_OPTS});
     self.postMessage({ id, frameNumber, bitmap }, [bitmap]);
   } catch (err) {
     const msg = err && err.message ? err.message : String(err);