playback

sam-tool-tauri/src-tauri/src/decoder.rs

@@ -2,15 +2,37 @@ use crate::video::VideoInfo;
 use std::collections::{HashMap, VecDeque};
 use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
-use std::process::{Child, ChildStdout, Command, Stdio};
+use std::process::{Child, ChildStderr, ChildStdout, Command, Stdio};
 use std::sync::{Arc, Condvar, Mutex};
 use std::thread::{self, JoinHandle};
 use std::time::{Duration, Instant};
 
+/// Cap on stderr lines retained per ffmpeg process. The hwaccel-init message
+/// is usually 1-3 lines; we keep more to cover later-frame errors too.
+const STDERR_TAIL_LINES: usize = 32;
+
 pub const DEFAULT_CACHE_CAPACITY: usize = 160;
 pub const MIN_CACHE_CAPACITY: usize = 50;
 pub const MAX_CACHE_CAPACITY: usize = 800;
+/// Cache cap for the RGBA preview decoder. Raw frames are 30-80× bigger
+/// than MJPEG so we cap depth for memory headroom. Sized for 1080p (the
+/// most common case): 16 frames × 8.3 MB ≈ 133 MB. At 720p this drops to
+/// ~45 MB. At 4K it's ~528 MB, so users with 4K sources should set
+/// preview_max_height in settings.
+pub const RGBA_PREVIEW_CACHE_CAPACITY: usize = 16;
+pub const RGBA_PREVIEW_LOOKAHEAD: u64 = 8;
 const READ_BUF_BYTES: usize = 256 * 1024;
+
+/// Wire format produced by an ffmpeg child. Preview decoder uses Rgba so
+/// the JS side avoids a CPU MJPEG decode per frame (which was the real
+/// per-frame bottleneck — see review findings A + E). Extract decoder
+/// stays on Mjpeg because `extract_one` writes the decoder output to disk
+/// and users expect .jpg files.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum OutputFormat {
+    Mjpeg,
+    Rgba,
+}
 /// If the requested frame is this far ahead of the worker, kill + reseek.
 /// Higher = fewer ffmpeg restarts during forward scan; the cap is bounded
 /// by lookahead × cache capacity anyway.
@@ -86,6 +108,10 @@ struct SharedState {
     eof: bool,
     error: Option<String>,
     shutdown: bool,
+    /// Last N stderr lines from the most recent ffmpeg child. Populated by
+    /// a non-blocking drain thread; read only when something has gone wrong
+    /// (e.g. hwaccel fallback) to surface the ffmpeg diagnostic.
+    stderr_tail: VecDeque<String>,
 }
 
 struct Shared {
@@ -103,9 +129,19 @@ pub struct DecoderSession {
     /// When Some(H), ffmpeg downscales to height H (aspect preserved) via
     /// `-vf scale=-2:H`. Used by the playback decoder for cheap previews;
     /// the extract decoder always passes None so saved frames are full-res.
-    preview_max_height: Option<u32>,
+    format: OutputFormat,
+    /// Pre-computed (width, height) ffmpeg will emit. For Mjpeg this is the
+    /// source dims (mjpeg encoder doesn't care). For Rgba these MUST be
+    /// even and exact, since the reader frames input by `out_w * out_h * 4`
+    /// byte counts — there's no marker to find frame boundaries in raw.
+    output_dims: (u32, u32),
     shared: Arc<Shared>,
     worker: Option<JoinHandle<()>>,
+    /// One-shot notice surfaced via `take_notice` after a hwaccel fallback.
+    /// Lets the frontend show "Hardware decode 'cuda' failed; running
+    /// software" instead of silently testing the same code path under three
+    /// different settings.
+    last_notice: Mutex<Option<String>>,
 }
 
 impl DecoderSession {
@@ -114,26 +150,61 @@ impl DecoderSession {
         info: VideoInfo,
         hwaccel: Option<String>,
         preview_max_height: Option<u32>,
+        format: OutputFormat,
     ) -> Self {
+        let preview_max_height = preview_max_height.filter(|h| *h > 0);
+        let output_dims = compute_output_dims(&info, preview_max_height);
+        // RGBA frames are huge (1080p = 8.3 MB), so a 160-entry cache would
+        // be ~1.3 GB. Cap depth and lookahead for the Rgba path; Mjpeg
+        // keeps its existing generous defaults.
+        let (init_cap, init_lookahead) = match format {
+            OutputFormat::Mjpeg => (
+                DEFAULT_CACHE_CAPACITY,
+                (DEFAULT_CACHE_CAPACITY as u64) / 2,
+            ),
+            OutputFormat::Rgba => (RGBA_PREVIEW_CACHE_CAPACITY, RGBA_PREVIEW_LOOKAHEAD),
+        };
         Self {
             path,
             info,
             hwaccel: hwaccel.filter(|s| !s.is_empty()),
-            preview_max_height: preview_max_height.filter(|h| *h > 0),
+            format,
+            output_dims,
             shared: Arc::new(Shared {
                 state: Mutex::new(SharedState {
                     child: None,
-                    cache: FrameCache::new(DEFAULT_CACHE_CAPACITY),
+                    cache: FrameCache::new(init_cap),
                     next_idx: 0,
                     reader_idx: 0,
-                    lookahead: (DEFAULT_CACHE_CAPACITY as u64) / 2,
+                    lookahead: init_lookahead,
                     eof: false,
                     error: None,
                     shutdown: false,
+                    stderr_tail: VecDeque::new(),
                 }),
                 cond: Condvar::new(),
             }),
             worker: None,
+            last_notice: Mutex::new(None),
+        }
+    }
+
+    /// Drain and return the most recent fallback / diagnostic notice (if any).
+    /// One-shot: a subsequent call returns None until the next event sets it.
+    pub fn take_notice(&self) -> Option<String> {
+        self.last_notice.lock().ok().and_then(|mut n| n.take())
+    }
+
+    fn snapshot_stderr_tail(&self) -> String {
+        let st = self.shared.state.lock().unwrap();
+        if st.stderr_tail.is_empty() {
+            "(no ffmpeg stderr captured)".into()
+        } else {
+            st.stderr_tail
+                .iter()
+                .cloned()
+                .collect::<Vec<_>>()
+                .join("\n")
         }
     }
 
@@ -142,6 +213,11 @@ impl DecoderSession {
     }
 
     pub fn set_cache_capacity(&self, capacity: usize) {
+        // RGBA preview cap is fixed by memory budget — a user-set 800-frame
+        // cache at 1080p RGBA would be ~6.6 GB.
+        if self.format == OutputFormat::Rgba {
+            return;
+        }
         let clamped = capacity.clamp(MIN_CACHE_CAPACITY, MAX_CACHE_CAPACITY);
         let mut st = self.shared.state.lock().unwrap();
         st.cache.set_capacity(clamped);
@@ -158,9 +234,14 @@ impl DecoderSession {
                 // path that worked on h264 but chokes on an HEVC segment).
                 // Once cleared, subsequent failures propagate.
                 if let Some(prev) = self.hwaccel.take() {
-                    eprintln!(
-                        "[decoder] hwaccel '{prev}' failed ({e}); falling back to software"
+                    let tail = self.snapshot_stderr_tail();
+                    let notice = format!(
+                        "Hardware decode '{prev}' failed; falling back to software.\nffmpeg said:\n{tail}"
                     );
+                    eprintln!("[decoder] {notice}");
+                    if let Ok(mut slot) = self.last_notice.lock() {
+                        *slot = Some(notice);
+                    }
                     self.stop_worker();
                     self.decode_at_inner(idx)
                 } else {
@@ -233,16 +314,18 @@ impl DecoderSession {
     fn seek_and_start_worker(&mut self, idx: u64) -> Result<(), String> {
         self.stop_worker();
 
-        let (child, stdout) = spawn_ffmpeg(
+        let (child, stdout, stderr) = spawn_ffmpeg(
             &self.path,
             &self.info,
             idx,
             self.hwaccel.as_deref(),
-            self.preview_max_height,
+            self.format,
+            self.output_dims,
         )?;
 
         {
             let mut st = self.shared.state.lock().unwrap();
+            st.stderr_tail.clear();
             st.child = Some(child);
             st.next_idx = idx;
             st.reader_idx = idx;
@@ -252,9 +335,14 @@ impl DecoderSession {
         }
         self.shared.cond.notify_all();
 
+        let drain_shared = self.shared.clone();
+        thread::spawn(move || drain_stderr(stderr, drain_shared));
+
+        let format = self.format;
+        let dims = self.output_dims;
         let shared = self.shared.clone();
         let handle = thread::spawn(move || {
-            worker_loop(stdout, shared);
+            worker_loop(stdout, shared, format, dims);
         });
         self.worker = Some(handle);
         Ok(())
@@ -324,13 +412,37 @@ fn vaapi_device() -> String {
     "/dev/dri/renderD128".to_string()
 }
 
+/// Compute the (width, height) ffmpeg will output. Scales down to
+/// preview_max_height only when source is taller; preserves aspect via a
+/// rounded-to-even width. Returns source dims when no scaling applies.
+/// The reader uses these to frame raw video output, so they MUST match
+/// what ffmpeg actually emits — we set them explicitly via `-s WxH` in
+/// the Rgba path to take the guesswork out.
+fn compute_output_dims(info: &VideoInfo, preview_max_height: Option<u32>) -> (u32, u32) {
+    if let Some(h) = preview_max_height {
+        if info.height > h && info.height > 0 {
+            // Maintain aspect; round width to nearest even (ffmpeg requires
+            // even dims for many filters).
+            let w = ((info.width as u64 * h as u64) + (info.height as u64 / 2))
+                / (info.height as u64);
+            let w_even = ((w as u32) + 1) & !1;
+            return (w_even.max(2), h);
+        }
+    }
+    // Even source dims for safety (almost always already even).
+    let w = (info.width + 1) & !1;
+    let h = (info.height + 1) & !1;
+    (w.max(2), h.max(2))
+}
+
 fn spawn_ffmpeg(
     path: &PathBuf,
     info: &VideoInfo,
     idx: u64,
     hwaccel: Option<&str>,
-    preview_max_height: Option<u32>,
-) -> Result<(Child, BufReader<ChildStdout>), String> {
+    format: OutputFormat,
+    out_dims: (u32, u32),
+) -> Result<(Child, BufReader<ChildStdout>, ChildStderr), String> {
     let path_str = path.to_str().ok_or_else(|| "non-utf8 path".to_string())?;
     let t = idx as f64 / info.fps;
     let mut cmd = Command::new("ffmpeg");
@@ -362,39 +474,96 @@ fn spawn_ffmpeg(
     cmd.args(["-ss", &format!("{:.6}", t)]);
     cmd.args(["-i", path_str]);
     cmd.args(["-an"]);
-    // Preview scaling: cap output height to `h`, width auto (`-2` keeps
-    // aspect & ensures even). Skip the filter entirely when the source is
-    // already shorter than the cap — no upscale, no wasted filter step.
-    if let Some(h) = preview_max_height {
-        if info.height > h {
-            let vf = format!("scale=-2:{h}");
-            cmd.args(["-vf", &vf]);
+    // Pin output dimensions exactly. For Mjpeg this is cosmetic (matches
+    // source); for Rgba it MUST match what the reader expects to byte-frame
+    // raw video output.
+    let (ow, oh) = out_dims;
+    if ow != info.width || oh != info.height {
+        cmd.args(["-s", &format!("{ow}x{oh}")]);
+    }
+    match format {
+        OutputFormat::Mjpeg => {
+            cmd.args([
+                "-q:v",
+                "3",
+                "-f",
+                "image2pipe",
+                "-vcodec",
+                "mjpeg",
+                "pipe:1",
+            ]);
+        }
+        OutputFormat::Rgba => {
+            // Raw RGBA, no entropy coding. Worker grabs the bytes verbatim
+            // into ImageData, eliminating the CPU MJPEG encode (ffmpeg)
+            // and the CPU JPEG decode (createImageBitmap on WebKit2GTK)
+            // that bracketed every preview frame in the old pipeline.
+            cmd.args([
+                "-pix_fmt",
+                "rgba",
+                "-f",
+                "rawvideo",
+                "pipe:1",
+            ]);
         }
     }
-    cmd.args([
-        "-q:v",
-        "3",
-        "-f",
-        "image2pipe",
-        "-vcodec",
-        "mjpeg",
-        "pipe:1",
-    ]);
-    // stderr null: capturing it cost a spawn+join drain thread per seek
-    // plus up to 150 ms of polling on EOF. hwaccel-failure messages still
-    // surface via the exit code path through `decode_at`'s fallback.
-    cmd.stdout(Stdio::piped()).stderr(Stdio::null());
+    // stderr is captured (was /dev/null) into a non-blocking drain thread
+    // that pushes lines into shared.state.stderr_tail. Bounded buffer, no
+    // join on EOF — the earlier 150ms wait-for-stderr-tail problem was the
+    // synchronous WAIT, not the capture itself.
+    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
     let mut child = cmd.spawn().map_err(|e| format!("ffmpeg spawn: {e}"))?;
     let stdout = child
         .stdout
         .take()
         .ok_or_else(|| "ffmpeg stdout unavailable".to_string())?;
-    Ok((child, BufReader::with_capacity(READ_BUF_BYTES, stdout)))
+    let stderr = child
+        .stderr
+        .take()
+        .ok_or_else(|| "ffmpeg stderr unavailable".to_string())?;
+    Ok((
+        child,
+        BufReader::with_capacity(READ_BUF_BYTES, stdout),
+        stderr,
+    ))
 }
 
-fn worker_loop(mut stdout: BufReader<ChildStdout>, shared: Arc<Shared>) {
+/// Read ffmpeg stderr line-by-line into a bounded ring buffer on shared
+/// state. Exits when ffmpeg closes the pipe (EOF) or the read errors.
+/// Cheap: no blocking on the main thread, used only to surface diagnostic
+/// info when something has gone wrong.
+fn drain_stderr(stderr: ChildStderr, shared: Arc<Shared>) {
+    let reader = std::io::BufReader::new(stderr);
+    for line_res in reader.lines() {
+        let line = match line_res {
+            Ok(l) => l,
+            Err(_) => return,
+        };
+        let trimmed = line.trim().to_string();
+        if trimmed.is_empty() {
+            continue;
+        }
+        let mut st = match shared.state.lock() {
+            Ok(g) => g,
+            Err(_) => return,
+        };
+        st.stderr_tail.push_back(trimmed);
+        while st.stderr_tail.len() > STDERR_TAIL_LINES {
+            st.stderr_tail.pop_front();
+        }
+    }
+}
+
+fn worker_loop(
+    mut stdout: BufReader<ChildStdout>,
+    shared: Arc<Shared>,
+    format: OutputFormat,
+    dims: (u32, u32),
+) {
+    // Pre-compute the exact frame size for the Rgba path so we don't pay
+    // for marker scanning that doesn't exist in raw video.
+    let raw_frame_bytes = (dims.0 as usize) * (dims.1 as usize) * 4;
     loop {
-        // Backpressure: wait if we're already far enough ahead.
         {
             let st = shared.state.lock().unwrap();
             if st.shutdown {
@@ -413,7 +582,12 @@ fn worker_loop(mut stdout: BufReader<ChildStdout>, shared: Arc<Shared>) {
             }
         }
 
-        match read_jpeg_bytes(&mut stdout) {
+        let read_result = match format {
+            OutputFormat::Mjpeg => read_jpeg_bytes(&mut stdout),
+            OutputFormat::Rgba => read_raw_frame(&mut stdout, raw_frame_bytes, dims),
+        };
+
+        match read_result {
             Ok(bytes) => {
                 let mut st = shared.state.lock().unwrap();
                 if st.shutdown {
@@ -438,6 +612,37 @@ fn worker_loop(mut stdout: BufReader<ChildStdout>, shared: Arc<Shared>) {
     }
 }
 
+/// Read exactly one raw RGBA frame from the pipe and prefix it with
+/// `[width:u32 LE, height:u32 LE]`. The JS worker then knows the
+/// dimensions without having to track them out-of-band. Returns
+/// `Err("pipe closed")` (or "EOF") so the worker loop's EOF detection
+/// branch fires cleanly.
+fn read_raw_frame(
+    reader: &mut BufReader<ChildStdout>,
+    frame_bytes: usize,
+    dims: (u32, u32),
+) -> Result<Vec<u8>, String> {
+    use std::io::Read;
+    let mut out = Vec::with_capacity(8 + frame_bytes);
+    out.extend_from_slice(&dims.0.to_le_bytes());
+    out.extend_from_slice(&dims.1.to_le_bytes());
+    out.resize(8 + frame_bytes, 0);
+    let buf = &mut out[8..];
+    let mut filled = 0usize;
+    while filled < frame_bytes {
+        match reader.read(&mut buf[filled..]) {
+            Ok(0) => {
+                return Err(format!(
+                    "ffmpeg pipe closed mid-frame ({filled}/{frame_bytes} bytes)"
+                ));
+            }
+            Ok(n) => filled += n,
+            Err(e) => return Err(format!("pipe read: {e}")),
+        }
+    }
+    Ok(out)
+}
+
 /// Read one JPEG (SOI 0xFF 0xD8 .. EOI 0xFF 0xD9) from an image2pipe stream.
 ///
 /// Chunked scan: for each `fill_buf` chunk we vector-scan for the marker

sam-tool-tauri/src-tauri/src/lib.rs

@@ -10,7 +10,7 @@ mod stream;
 mod user;
 mod video;
 
-use decoder::DecoderSession;
+use decoder::{DecoderSession, OutputFormat};
 use serde::Serialize;
 use state::AppState;
 use std::collections::HashMap;
@@ -41,12 +41,28 @@ fn open_video(
         .filter(|s| !s.is_empty());
     let preview_max_height = settings.as_ref().map(|s| s.preview_max_height);
 
-    // Two sessions:
-    //  - preview: may downscale via -vf scale=-2:H for cheap playback.
-    //  - extract: always full resolution; what extract_one writes to disk.
-    // Both spawn ffmpeg lazily — opening a video is still cheap.
-    let preview = DecoderSession::open(p.clone(), info.clone(), hwaccel.clone(), preview_max_height);
-    let extract = DecoderSession::open(p, info.clone(), hwaccel, None);
+    // Two sessions, two formats:
+    //  - preview: raw RGBA frames so JS skips the JPEG-decode round-trip
+    //    that bracketed every preview frame in the old pipeline. May also
+    //    downscale via `-s WxH` for cheap playback when preview_max_height
+    //    is set.
+    //  - extract: MJPEG output (full resolution) so `extract_one` writes
+    //    .jpg files unchanged.
+    // Both spawn ffmpeg lazily.
+    let preview = DecoderSession::open(
+        p.clone(),
+        info.clone(),
+        hwaccel.clone(),
+        preview_max_height,
+        OutputFormat::Rgba,
+    );
+    let extract = DecoderSession::open(
+        p,
+        info.clone(),
+        hwaccel,
+        None,
+        OutputFormat::Mjpeg,
+    );
     if let Some(s) = settings.as_ref() {
         preview.set_cache_capacity(s.cache_capacity);
         extract.set_cache_capacity(s.cache_capacity);
@@ -109,6 +125,19 @@ fn import_coco(path: String) -> Result<coco::ImportedCoco, String> {
     coco::import(p)
 }
 
+/// Drain the most recent diagnostic notice from the preview decoder (e.g.
+/// "Hardware decode 'cuda' failed; falling back to software"). One-shot:
+/// a subsequent call returns None until the next event sets it. Frontend
+/// is expected to poll this after first decode and surface as a toast.
+#[tauri::command]
+fn take_decoder_notice(state: State<'_, AppState>) -> Result<Option<String>, String> {
+    let guard = state
+        .preview_decoder
+        .lock()
+        .map_err(|e| format!("lock: {e}"))?;
+    Ok(guard.as_ref().and_then(|d| d.take_notice()))
+}
+
 #[tauri::command]
 fn set_cache_capacity(state: State<'_, AppState>, capacity: usize) -> Result<usize, String> {
     let clamped = capacity.clamp(decoder::MIN_CACHE_CAPACITY, decoder::MAX_CACHE_CAPACITY);
@@ -336,6 +365,7 @@ pub fn run() {
             close_video,
             current_video,
             decode_frame,
+            take_decoder_notice,
             set_cache_capacity,
             import_coco,
             extract_frame,