From 0b5510636e6f6b45e1033eb8c7ff9236cf4899a6 Mon Sep 17 00:00:00 2001 From: ravi Date: Tue, 19 May 2026 14:05:47 +0530 Subject: [PATCH] playback and import issues --- sam-tool-tauri/src-tauri/src/coco.rs | 154 +++++++++++++++++- sam-tool-tauri/src-tauri/src/decoder.rs | 28 +++- sam-tool-tauri/src-tauri/src/lib.rs | 29 ++-- sam-tool-tauri/src-tauri/src/settings.rs | 17 +- .../src/components/shell/SettingsModal.tsx | 11 +- sam-tool-tauri/src/modes/AnnotateMode.tsx | 124 ++++++-------- sam-tool-tauri/src/modes/ExtractMode.tsx | 146 ++++++++++++++--- 7 files changed, 377 insertions(+), 132 deletions(-) diff --git a/sam-tool-tauri/src-tauri/src/coco.rs b/sam-tool-tauri/src-tauri/src/coco.rs index c176ecb..278bed2 100644 --- a/sam-tool-tauri/src-tauri/src/coco.rs +++ b/sam-tool-tauri/src-tauri/src/coco.rs @@ -77,6 +77,10 @@ pub struct ImportedCoco { enum Format { LabelStudioCoco, SamAnno, + /// `{"video": {"width", "height", ...}, "fixed_annotations": [ + /// { "label", "frame_number", "type": "bbox"|"polygon", + /// "vertices": [{"x","y"}, ...] (normalized 0..1) }, ... ]}` + FixedAnno, Unknown, } @@ -87,6 +91,9 @@ fn detect_format(v: &serde_json::Value) -> Format { if obj.contains_key("images") && obj.contains_key("annotations") { return Format::LabelStudioCoco; } + if obj.contains_key("fixed_annotations") { + return Format::FixedAnno; + } if !obj.is_empty() && obj .keys() @@ -111,8 +118,9 @@ pub fn import(path: &Path) -> Result { Ok(build_from_coco(coco)) } Format::SamAnno => build_from_sam_anno(&root), + Format::FixedAnno => build_from_fixed_anno(&root), Format::Unknown => Err( - "unrecognized annotation format — expected Label Studio COCO or SAM anno JSON" + "unrecognized annotation format — expected Label Studio COCO, SAM anno JSON, or fixed_annotations JSON" .into(), ), } @@ -322,6 +330,150 @@ fn build_from_sam_anno(root: &serde_json::Value) -> Result }) } +// ---- Fixed-annotations format ---------------------------------------------- +// +// { +// "video": { "width": 1920, "height": 1080, "fps": 30, ... }, +// "fixed_annotations": [ +// { "label": "...", "frame_number": 1278, +// "type": "bbox", // 4 corner vertices, or +// "type": "polygon", // N >= 3 vertices +// "vertices": [ {"x": 0..1, "y": 0..1}, ... ], +// ... } +// ] +// } +// +// Coordinates are normalized to the video frame, so we scale by video.width +// and video.height to get absolute pixels. The "side" / "annotated_by" / +// "created_at" fields are metadata we don't need for the shape import; we +// only keep label, frame, geometry. +fn build_from_fixed_anno(root: &serde_json::Value) -> Result { + let obj = root + .as_object() + .ok_or_else(|| "fixed_anno root is not an object".to_string())?; + let video = obj + .get("video") + .and_then(|v| v.as_object()) + .ok_or_else(|| "missing `video` block".to_string())?; + let w = video + .get("width") + .and_then(|v| v.as_f64()) + .ok_or_else(|| "video.width missing or non-numeric".to_string())?; + let h = video + .get("height") + .and_then(|v| v.as_f64()) + .ok_or_else(|| "video.height missing or non-numeric".to_string())?; + if w <= 0.0 || h <= 0.0 { + return Err(format!("video dimensions invalid: {w}x{h}")); + } + let items = obj + .get("fixed_annotations") + .and_then(|v| v.as_array()) + .ok_or_else(|| "`fixed_annotations` is not an array".to_string())?; + + let mut class_names = BTreeSet::::new(); + let mut frames: HashMap> = HashMap::new(); + let mut stats = ImportStats { + source_format: "fixed".into(), + ..Default::default() + }; + + for item in items { + let Some(obj) = item.as_object() else { continue }; + let label = obj + .get("label") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + let frame = obj + .get("frame_number") + .and_then(|v| v.as_u64()) + .or_else(|| obj.get("frame").and_then(|v| v.as_u64())); + let Some(frame) = frame else { + stats.skipped_no_image += 1; + continue; + }; + let kind = obj + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or("bbox") + .to_lowercase(); + let Some(verts) = obj.get("vertices").and_then(|v| v.as_array()) else { + continue; + }; + let pts: Vec<[f64; 2]> = verts + .iter() + .filter_map(|v| { + let o = v.as_object()?; + let x = o.get("x")?.as_f64()?; + let y = o.get("y")?.as_f64()?; + Some([x * w, y * h]) + }) + .collect(); + if pts.len() < 2 { + continue; + } + + stats.total_annotations += 1; + class_names.insert(label.clone()); + let bucket = frames.entry(frame.to_string()).or_default(); + + if kind == "bbox" { + // Tight axis-aligned bbox of whatever vertices were given. + let (mut xmin, mut ymin) = (f64::INFINITY, f64::INFINITY); + let (mut xmax, mut ymax) = (f64::NEG_INFINITY, f64::NEG_INFINITY); + for &[x, y] in &pts { + if x < xmin { xmin = x; } + if y < ymin { ymin = y; } + if x > xmax { xmax = x; } + if y > ymax { ymax = y; } + } + bucket.push(Shape::Bbox { + class_id: 0, + class_name: label, + bbox: [xmin, ymin, xmax - xmin, ymax - ymin], + }); + } else if pts.len() >= 3 { + bucket.push(Shape::Polygon { + class_id: 0, + class_name: label, + points: pts, + }); + } + } + + // Assign class ids by sorted class name (same convention as SAM anno). + let categories: Vec = class_names + .iter() + .enumerate() + .map(|(i, name)| Category { + id: i as i64, + name: name.clone(), + }) + .collect(); + let name_to_id: HashMap = + categories.iter().map(|c| (c.name.clone(), c.id)).collect(); + for shapes in frames.values_mut() { + for shape in shapes.iter_mut() { + let (cid, cname) = match shape { + Shape::Bbox { class_id, class_name, .. } + | Shape::Polygon { class_id, class_name, .. } => (class_id, class_name), + }; + if let Some(&id) = name_to_id.get(cname) { + *cid = id; + } + } + } + + stats.categories = categories.len(); + stats.frame_count = frames.len(); + Ok(ImportedCoco { + categories, + frames, + stats, + }) +} + // ---- helpers --------------------------------------------------------------- fn trailing_int(file_name: &str) -> Option { diff --git a/sam-tool-tauri/src-tauri/src/decoder.rs b/sam-tool-tauri/src-tauri/src/decoder.rs index a6f5ae5..889ede3 100644 --- a/sam-tool-tauri/src-tauri/src/decoder.rs +++ b/sam-tool-tauri/src-tauri/src/decoder.rs @@ -5,14 +5,15 @@ use std::path::PathBuf; use std::process::{Child, ChildStdout, Command, Stdio}; use std::sync::{Arc, Condvar, Mutex}; use std::thread::{self, JoinHandle}; -use std::time::Duration; +use std::time::{Duration, Instant}; -pub const DEFAULT_CACHE_CAPACITY: usize = 500; -pub const MIN_CACHE_CAPACITY: usize = 200; -pub const MAX_CACHE_CAPACITY: usize = 2000; +pub const DEFAULT_CACHE_CAPACITY: usize = 160; +pub const MIN_CACHE_CAPACITY: usize = 50; +pub const MAX_CACHE_CAPACITY: usize = 800; const READ_BUF_BYTES: usize = 256 * 1024; /// If the requested frame is this far ahead of the worker, kill + reseek. const SEEK_AHEAD_THRESHOLD: u64 = 60; +const DECODE_TIMEOUT: Duration = Duration::from_secs(15); struct FrameCache { map: HashMap>, @@ -158,6 +159,7 @@ impl DecoderSession { } // Wait for the worker to produce idx. + let wait_started = Instant::now(); let mut st = self.shared.state.lock().unwrap(); loop { if st.cache.contains(idx) { @@ -175,7 +177,17 @@ impl DecoderSession { if st.shutdown { return Err("decoder shut down".into()); } - st = self.shared.cond.wait(st).unwrap(); + let (next_st, timeout) = self + .shared + .cond + .wait_timeout(st, Duration::from_millis(250)) + .unwrap(); + st = next_st; + if timeout.timed_out() && wait_started.elapsed() >= DECODE_TIMEOUT { + drop(st); + self.stop_worker(); + return Err(format!("timed out decoding frame {idx}")); + } } } @@ -239,9 +251,7 @@ fn spawn_ffmpeg( info: &VideoInfo, idx: u64, ) -> Result<(Child, BufReader), String> { - let path_str = path - .to_str() - .ok_or_else(|| "non-utf8 path".to_string())?; + let path_str = path.to_str().ok_or_else(|| "non-utf8 path".to_string())?; let t = idx as f64 / info.fps; let mut cmd = Command::new("ffmpeg"); cmd.args(["-v", "error", "-threads", "0"]); @@ -263,7 +273,7 @@ fn spawn_ffmpeg( "mjpeg", "pipe:1", ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + cmd.stdout(Stdio::piped()).stderr(Stdio::null()); let mut child = cmd.spawn().map_err(|e| format!("ffmpeg spawn: {e}"))?; let stdout = child .stdout diff --git a/sam-tool-tauri/src-tauri/src/lib.rs b/sam-tool-tauri/src-tauri/src/lib.rs index d0af287..3385ea8 100644 --- a/sam-tool-tauri/src-tauri/src/lib.rs +++ b/sam-tool-tauri/src-tauri/src/lib.rs @@ -23,14 +23,22 @@ fn ping() -> String { } #[tauri::command] -fn open_video(state: State<'_, AppState>, path: String) -> Result { +fn open_video( + state: State<'_, AppState>, + app: AppHandle, + path: String, +) -> Result { let p = PathBuf::from(&path); if !p.exists() { return Err(format!("file not found: {}", p.display())); } let info = video::probe(&p)?; + let decoder = DecoderSession::open(p, info.clone()); + if let Ok(settings) = settings::load(&app) { + decoder.set_cache_capacity(settings.cache_capacity); + } let mut guard = state.decoder.lock().map_err(|e| format!("lock: {e}"))?; - *guard = Some(DecoderSession::open(p, info.clone())); + *guard = Some(decoder); Ok(info) } @@ -120,9 +128,7 @@ fn bulk_extract( let empty: Vec = Vec::new(); let mut done: u64 = 0; for idx in start..=end { - let shapes = shapes_by_frame - .get(&idx.to_string()) - .unwrap_or(&empty); + let shapes = shapes_by_frame.get(&idx.to_string()).unwrap_or(&empty); extract::extract_one(decoder, idx, &dir, shapes, &author)?; done += 1; if done == 1 || done % 5 == 0 || done == total { @@ -133,12 +139,7 @@ fn bulk_extract( } #[tauri::command] -fn bulk_undo( - app: AppHandle, - start: u64, - end: u64, - out_dir: String, -) -> Result { +fn bulk_undo(app: AppHandle, start: u64, end: u64, out_dir: String) -> Result { if end < start { return Err("end must be >= start".into()); } @@ -253,10 +254,8 @@ fn save_image_annotations( } } } - let text = serde_json::to_string_pretty(&payload) - .map_err(|e| format!("serialize: {e}"))?; - std::fs::write(&json_path, text) - .map_err(|e| format!("write {}: {e}", json_path.display())) + let text = serde_json::to_string_pretty(&payload).map_err(|e| format!("serialize: {e}"))?; + std::fs::write(&json_path, text).map_err(|e| format!("write {}: {e}", json_path.display())) } #[tauri::command] diff --git a/sam-tool-tauri/src-tauri/src/settings.rs b/sam-tool-tauri/src-tauri/src/settings.rs index 7f0858a..ab0c7c6 100644 --- a/sam-tool-tauri/src-tauri/src/settings.rs +++ b/sam-tool-tauri/src-tauri/src/settings.rs @@ -22,7 +22,15 @@ fn default_sam_url() -> String { } fn default_cache_capacity() -> usize { - 500 + crate::decoder::DEFAULT_CACHE_CAPACITY +} + +fn normalize(mut s: Settings) -> Settings { + s.cache_capacity = s.cache_capacity.clamp( + crate::decoder::MIN_CACHE_CAPACITY, + crate::decoder::MAX_CACHE_CAPACITY, + ); + s } impl Default for Settings { @@ -51,12 +59,15 @@ pub fn load(app: &AppHandle) -> Result { return Ok(Settings::default()); } let text = fs::read_to_string(&path).map_err(|e| format!("read {}: {e}", path.display()))?; - serde_json::from_str(&text).map_err(|e| format!("parse settings: {e}")) + serde_json::from_str(&text) + .map(normalize) + .map_err(|e| format!("parse settings: {e}")) } pub fn save(app: &AppHandle, s: &Settings) -> Result<(), String> { let path = settings_path(app)?; - let text = serde_json::to_string_pretty(s).map_err(|e| format!("serialize: {e}"))?; + let normalized = normalize(s.clone()); + let text = serde_json::to_string_pretty(&normalized).map_err(|e| format!("serialize: {e}"))?; fs::write(&path, text).map_err(|e| format!("write {}: {e}", path.display())) } diff --git a/sam-tool-tauri/src/components/shell/SettingsModal.tsx b/sam-tool-tauri/src/components/shell/SettingsModal.tsx index 40a7f6a..cd59767 100644 --- a/sam-tool-tauri/src/components/shell/SettingsModal.tsx +++ b/sam-tool-tauri/src/components/shell/SettingsModal.tsx @@ -74,6 +74,7 @@ export function SettingsModal({ open, onClose, onSaved }: Props) { setError(null); try { await api.saveSettings(settings); + await api.setCacheCapacity(settings.cache_capacity).catch(() => {}); onSaved?.(settings); onClose(); } catch (e) { @@ -238,8 +239,8 @@ export function SettingsModal({ open, onClose, onSaved }: Props) { @@ -250,9 +251,9 @@ export function SettingsModal({ open, onClose, onSaved }: Props) { } />
- 200 - 1000 - 2000 + 50 + 400 + 800
diff --git a/sam-tool-tauri/src/modes/AnnotateMode.tsx b/sam-tool-tauri/src/modes/AnnotateMode.tsx index cf12282..16888a3 100644 --- a/sam-tool-tauri/src/modes/AnnotateMode.tsx +++ b/sam-tool-tauri/src/modes/AnnotateMode.tsx @@ -15,8 +15,6 @@ import { DeleteConfirmModal } from "../components/annotate/DeleteConfirmModal"; import { drawOverlay, colorForClass } from "../components/extract/overlay"; import type { Settings } from "../types"; -const DEFAULT_CLASS_ID = 0; - interface Props { username: string; settings: Settings | null; @@ -109,7 +107,7 @@ export function AnnotateMode({ username, settings }: Props) { const canvasRef = useRef(null); const bitmapRef = useRef(null); - const loadedForImageRef = useRef(null); + const imageLoadSeqRef = useRef(0); const stageRef = useRef(null); // Mirror of `classes` that can be updated synchronously — needed because // `pickerAddClass` calls setClasses and then immediately calls onConfirm, @@ -180,7 +178,7 @@ export function AnnotateMode({ username, settings }: Props) { setDims(null); bitmapRef.current?.close(); bitmapRef.current = null; - loadedForImageRef.current = null; + imageLoadSeqRef.current += 1; resetZoom(); setDrawing(null); } catch (e) { @@ -366,82 +364,60 @@ export function AnnotateMode({ username, settings }: Props) { } }, [ann, overlayVisible, drawing, drawInProgress, selectedAnnId, bboxDrag, vertexDrag]); - const loadImage = useCallback(async (imgPath: string) => { - try { - const bytes = await api.readImageBytes(imgPath); - const blob = new Blob([bytes], { type: "image/jpeg" }); - const bitmap = await createImageBitmap(blob); + useEffect(() => { + if (!selectedImagePath) { + setAnn(null); + setDims(null); bitmapRef.current?.close(); - bitmapRef.current = bitmap; - loadedForImageRef.current = imgPath; - setDims((prev) => - prev && prev.w === bitmap.width && prev.h === bitmap.height - ? prev - : { w: bitmap.width, h: bitmap.height } - ); - } catch (e) { - setLoadError(`image load: ${e}`); + bitmapRef.current = null; + imageLoadSeqRef.current += 1; + return; } - }, []); - - useEffect(() => { - if (!selectedImagePath) return; - let cancelled = false; - setLoadError(null); - api - .loadImageAnnotations(selectedImagePath) - .then((loaded) => { - if (cancelled) return; - setAnn( - loaded ?? - (dims - ? { - version: 1, - image: selectedImagePath.split("/").pop() ?? "", - width: dims.w, - height: dims.h, - annotations: [], - } - : null) - ); - }) - .catch((e) => { - if (cancelled) return; - setAnn(null); - setLoadError(String(e)); - }); - return () => { - cancelled = true; - }; - // Intentionally not depending on `dims` — first selection may still have - // null dims; subsequent effect from loadImage will fill them and this - // effect reruns on path change only. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [selectedImagePath]); - - // Hold a ref to the latest drawCanvas so the image-change effect can call - // it without subscribing to its identity (which flips on every `drawing` / - // `ann` / `overlayVisible` change and would reset state mid-interaction). - const drawCanvasRef = useRef<() => void>(() => { }); - useEffect(() => { - drawCanvasRef.current = drawCanvas; - }, [drawCanvas]); - - // On image selection change: reset per-image state, load bytes, draw once. - useEffect(() => { - if (!selectedImagePath) return; + const seq = imageLoadSeqRef.current + 1; + imageLoadSeqRef.current = seq; resetZoom(); + setLoadError(null); + setAnn(null); + setDims(null); setDrawing(null); setSelectedAnnId(null); setBboxDrag(null); setVertexDrag(null); undoStackRef.current = []; redoStackRef.current = []; - if (loadedForImageRef.current === selectedImagePath) { - drawCanvasRef.current(); - } else { - loadImage(selectedImagePath).then(() => drawCanvasRef.current()); + + async function loadSelectedImage() { + try { + const [bytes, loaded] = await Promise.all([ + api.readImageBytes(selectedImagePath!), + api.loadImageAnnotations(selectedImagePath!), + ]); + const blob = new Blob([bytes], { type: "image/jpeg" }); + const bitmap = await createImageBitmap(blob); + if (imageLoadSeqRef.current !== seq) { + bitmap.close(); + return; + } + bitmapRef.current?.close(); + bitmapRef.current = bitmap; + const nextDims = { w: bitmap.width, h: bitmap.height }; + setDims(nextDims); + setAnn( + loaded ?? { + version: 1, + image: selectedImagePath!.split("/").pop() ?? "", + width: nextDims.w, + height: nextDims.h, + annotations: [], + } + ); + } catch (e) { + if (imageLoadSeqRef.current !== seq) return; + setAnn(null); + setLoadError(`image load: ${e}`); + } } + void loadSelectedImage(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [selectedImagePath]); @@ -942,12 +918,8 @@ export function AnnotateMode({ username, settings }: Props) { setSelectedAnnId(null); return; } - // if (activeClassId == null) { - // setStatusMsg("Pick a class on the right first"); - // return; - // } - const classId = activeClassId ?? DEFAULT_CLASS_ID; - + // Class id isn't needed until the LabelPicker confirms after the shape + // is drawn — drawing itself doesn't depend on `activeClassId`. e.stopPropagation(); // don't let the pan handler see this const p = canvasToImageCoords(e.clientX, e.clientY); if (!p) return; diff --git a/sam-tool-tauri/src/modes/ExtractMode.tsx b/sam-tool-tauri/src/modes/ExtractMode.tsx index 9821284..48e019c 100644 --- a/sam-tool-tauri/src/modes/ExtractMode.tsx +++ b/sam-tool-tauri/src/modes/ExtractMode.tsx @@ -7,6 +7,7 @@ import { FrameCanvas } from "../components/extract/FrameCanvas"; import { drawOverlay } from "../components/extract/overlay"; const SPEEDS = [0.25, 0.5, 1, 1.5, 2, 3, 4]; +const FRAME_BITMAP_CACHE_LIMIT = 24; interface Props { username: string; @@ -67,20 +68,48 @@ export function ExtractMode({ username }: Props) { const requestedIdx = useRef(0); const renderedIdx = useRef(-1); const decodingInFlight = useRef(false); + const bitmapCacheRef = useRef>(new Map()); - const renderFrame = useCallback( - async (bytes: ArrayBuffer, atFrameIdx: number) => { + const clearBitmapCache = useCallback(() => { + for (const bitmap of bitmapCacheRef.current.values()) { + bitmap.close(); + } + bitmapCacheRef.current.clear(); + }, []); + + const getCachedBitmap = useCallback((idx: number) => { + const cache = bitmapCacheRef.current; + const bitmap = cache.get(idx); + if (!bitmap) return null; + cache.delete(idx); + cache.set(idx, bitmap); + return bitmap; + }, []); + + const rememberBitmap = useCallback((idx: number, bitmap: ImageBitmap) => { + const cache = bitmapCacheRef.current; + const existing = cache.get(idx); + if (existing && existing !== bitmap) { + existing.close(); + } + cache.delete(idx); + cache.set(idx, bitmap); + while (cache.size > FRAME_BITMAP_CACHE_LIMIT) { + const oldest = cache.keys().next().value; + if (oldest == null) break; + cache.get(oldest)?.close(); + cache.delete(oldest); + } + }, []); + + const drawBitmap = useCallback( + (bitmap: ImageBitmap, atFrameIdx: number) => { const canvas = canvasRef.current; if (!canvas) return; - const blob = new Blob([bytes], { type: "image/jpeg" }); - let bitmap: ImageBitmap; - try { bitmap = await createImageBitmap(blob); } - catch (e) { console.error("createImageBitmap failed", e); return; } const ctx = canvas.getContext("2d"); - if (!ctx) { bitmap.close(); return; } + if (!ctx) return; ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height); - bitmap.close(); if (overlayVisibleRef.current) { const shapes = frameMapRef.current.get(atFrameIdx); @@ -92,12 +121,31 @@ export function ExtractMode({ username }: Props) { [] ); + const renderFrame = useCallback( + async (bytes: ArrayBuffer, atFrameIdx: number) => { + const blob = new Blob([bytes], { type: "image/jpeg" }); + let bitmap: ImageBitmap; + try { bitmap = await createImageBitmap(blob); } + catch (e) { console.error("createImageBitmap failed", e); return; } + rememberBitmap(atFrameIdx, bitmap); + if (requestedIdx.current !== atFrameIdx) return; + drawBitmap(bitmap, atFrameIdx); + }, + [drawBitmap, rememberBitmap] + ); + const pump = useCallback(async () => { if (decodingInFlight.current) return; decodingInFlight.current = true; try { while (requestedIdx.current !== renderedIdx.current) { const want = requestedIdx.current; + const cached = getCachedBitmap(want); + if (cached) { + drawBitmap(cached, want); + renderedIdx.current = want; + continue; + } try { const bytes = await api.decodeFrame(want); if (requestedIdx.current !== want) continue; @@ -112,7 +160,7 @@ export function ExtractMode({ username }: Props) { } finally { decodingInFlight.current = false; } - }, [renderFrame]); + }, [drawBitmap, getCachedBitmap, renderFrame]); useEffect(() => { if (!video) return; @@ -121,9 +169,17 @@ export function ExtractMode({ username }: Props) { }, [frameIdx, pump, video]); useEffect(() => { - renderedIdx.current = -1; - pump(); - }, [overlayVisible, frameMap, pump]); + const cached = getCachedBitmap(frameIdxRef.current); + if (cached) { + drawBitmap(cached, frameIdxRef.current); + renderedIdx.current = frameIdxRef.current; + } else { + renderedIdx.current = -1; + pump(); + } + }, [overlayVisible, frameMap, drawBitmap, getCachedBitmap, pump]); + + useEffect(() => clearBitmapCache, [clearBitmapCache]); // --- playback loop --------------------------------------------------- useEffect(() => { @@ -142,7 +198,16 @@ export function ExtractMode({ username }: Props) { video.total_frames - 1, anchor.idx + Math.floor(elapsed * video.fps * speed) ); - if (reviewMode && annotatedFrames.current.has(want)) { + // Only trigger the review-mode auto-pause once we've actually advanced + // past the resume point. Otherwise pressing play while parked on an + // annotated frame would re-pause on the very first tick, making the + // play button look broken — the user would have to step a frame + // forward manually before play could "stick". + if ( + reviewMode && + want > anchor.idx && + annotatedFrames.current.has(want) + ) { setFrameIdx(want); setPlaying(false); return; @@ -162,19 +227,28 @@ export function ExtractMode({ username }: Props) { const seekToFrame = useCallback((idx: number) => { if (!video) return; const clamped = Math.max(0, Math.min(video.total_frames - 1, idx)); - setFrameIdx(clamped); + frameIdxRef.current = clamped; + requestedIdx.current = clamped; + const cached = getCachedBitmap(clamped); + if (cached) { + drawBitmap(cached, clamped); + renderedIdx.current = clamped; + } else { + pump(); + } + setFrameIdx((prev) => (prev === clamped ? prev : clamped)); // If a playback tick is active, rebase its anchor so the loop continues // from the new position instead of snapping back. if (playAnchorRef.current) { playAnchorRef.current = { time: performance.now(), idx: clamped }; } - }, [video]); + }, [drawBitmap, getCachedBitmap, pump, video]); const stepFrames = useCallback((delta: number) => { if (!video) return; setPlaying(false); - setFrameIdx((i) => Math.max(0, Math.min(video.total_frames - 1, i + delta))); - }, [video]); + seekToFrame(frameIdxRef.current + delta); + }, [seekToFrame, video]); const togglePlay = useCallback(() => { if (!video) return; @@ -213,6 +287,7 @@ export function ExtractMode({ username }: Props) { const extractCurrentFrame = useCallback(async () => { if (!outputFolder || !video) return; + setPlaying(false); setBulkError(null); try { const shapes = frameMap.get(frameIdx) ?? []; @@ -225,6 +300,7 @@ export function ExtractMode({ username }: Props) { const undoCurrentFrame = useCallback(async () => { if (!outputFolder) return; + setPlaying(false); setBulkError(null); try { await api.undoExtract(frameIdx, outputFolder); @@ -252,18 +328,28 @@ export function ExtractMode({ username }: Props) { async function runBulkExtract() { if (!outputFolder || !bulkRange) return; + setPlaying(false); setBulkError(null); setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "extract" }); const unlisten = await listen<{ done: number; total: number }>( "bulk-extract-progress", (e) => setBulkProgress({ ...e.payload, phase: "extract" }) ); + const shapesByFrame: Record = {}; + if (coco) { + for (const [key, shapes] of Object.entries(coco.frames)) { + const idx = Number(key); + if (idx >= bulkRange.start && idx <= bulkRange.end) { + shapesByFrame[key] = shapes; + } + } + } try { await api.bulkExtract( bulkRange.start, bulkRange.end, outputFolder, - coco?.frames ?? {}, + shapesByFrame, username ); const existing = await api.listExtractedFrames(outputFolder); @@ -278,6 +364,7 @@ export function ExtractMode({ username }: Props) { async function runBulkUndo() { if (!outputFolder || !bulkRange) return; + setPlaying(false); setBulkError(null); setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "undo" }); const unlisten = await listen<{ done: number; total: number }>( @@ -296,6 +383,18 @@ export function ExtractMode({ username }: Props) { } } + const extractCurrentFrameRef = useRef(extractCurrentFrame); + const undoCurrentFrameRef = useRef(undoCurrentFrame); + const setRangeStartRef = useRef(setRangeStart); + const setRangeEndRef = useRef(setRangeEnd); + + useEffect(() => { + extractCurrentFrameRef.current = extractCurrentFrame; + undoCurrentFrameRef.current = undoCurrentFrame; + setRangeStartRef.current = setRangeStart; + setRangeEndRef.current = setRangeEnd; + }, [extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]); + // --- keyboard -------------------------------------------------------- useEffect(() => { function onKey(e: KeyboardEvent) { @@ -316,18 +415,18 @@ export function ExtractMode({ username }: Props) { case "h": case "H": e.preventDefault(); setOverlayVisible((v) => !v); break; case "e": case "E": - e.preventDefault(); extractCurrentFrame(); break; + e.preventDefault(); extractCurrentFrameRef.current(); break; case "u": case "U": - e.preventDefault(); undoCurrentFrame(); break; + e.preventDefault(); undoCurrentFrameRef.current(); break; case "[": - e.preventDefault(); setRangeStart(); break; + e.preventDefault(); setRangeStartRef.current(); break; case "]": - e.preventDefault(); setRangeEnd(); break; + e.preventDefault(); setRangeEndRef.current(); break; } } window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); - }, [togglePlay, stepFrames, bumpSpeed, extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]); + }, [togglePlay, stepFrames, bumpSpeed]); // --- video/coco open ------------------------------------------------- useEffect(() => { @@ -358,6 +457,7 @@ export function ExtractMode({ username }: Props) { if (!selected || typeof selected !== "string") return; try { const info = await api.openVideo(selected); + clearBitmapCache(); setVideo(info); setFrameIdx(0); setPlaying(false);