playback and import issues

2026-05-19 14:05:47 +05:30
parent f744747167
commit 0b5510636e
7 changed files with 377 additions and 132 deletions
--- a/sam-tool-tauri/src/modes/ExtractMode.tsx
+++ b/sam-tool-tauri/src/modes/ExtractMode.tsx
@@ -7,6 +7,7 @@ import { FrameCanvas } from "../components/extract/FrameCanvas";
 import { drawOverlay } from "../components/extract/overlay";

 const SPEEDS = [0.25, 0.5, 1, 1.5, 2, 3, 4];
+const FRAME_BITMAP_CACHE_LIMIT = 24;

 interface Props {
  username: string;
@@ -67,20 +68,48 @@ export function ExtractMode({ username }: Props) {
  const requestedIdx = useRef(0);
  const renderedIdx = useRef(-1);
  const decodingInFlight = useRef(false);
+  const bitmapCacheRef = useRef<Map<number, ImageBitmap>>(new Map());

-  const renderFrame = useCallback(
-    async (bytes: ArrayBuffer, atFrameIdx: number) => {
+  const clearBitmapCache = useCallback(() => {
+    for (const bitmap of bitmapCacheRef.current.values()) {
+      bitmap.close();
+    }
+    bitmapCacheRef.current.clear();
+  }, []);
+
+  const getCachedBitmap = useCallback((idx: number) => {
+    const cache = bitmapCacheRef.current;
+    const bitmap = cache.get(idx);
+    if (!bitmap) return null;
+    cache.delete(idx);
+    cache.set(idx, bitmap);
+    return bitmap;
+  }, []);
+
+  const rememberBitmap = useCallback((idx: number, bitmap: ImageBitmap) => {
+    const cache = bitmapCacheRef.current;
+    const existing = cache.get(idx);
+    if (existing && existing !== bitmap) {
+      existing.close();
+    }
+    cache.delete(idx);
+    cache.set(idx, bitmap);
+    while (cache.size > FRAME_BITMAP_CACHE_LIMIT) {
+      const oldest = cache.keys().next().value;
+      if (oldest == null) break;
+      cache.get(oldest)?.close();
+      cache.delete(oldest);
+    }
+  }, []);
+
+  const drawBitmap = useCallback(
+    (bitmap: ImageBitmap, atFrameIdx: number) => {
      const canvas = canvasRef.current;
      if (!canvas) return;
-      const blob = new Blob([bytes], { type: "image/jpeg" });
-      let bitmap: ImageBitmap;
-      try { bitmap = await createImageBitmap(blob); }
-      catch (e) { console.error("createImageBitmap failed", e); return; }
      const ctx = canvas.getContext("2d");
-      if (!ctx) { bitmap.close(); return; }
+      if (!ctx) return;
      ctx.clearRect(0, 0, canvas.width, canvas.height);
      ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
-      bitmap.close();

      if (overlayVisibleRef.current) {
        const shapes = frameMapRef.current.get(atFrameIdx);
@@ -92,12 +121,31 @@ export function ExtractMode({ username }: Props) {
    []
  );

+  const renderFrame = useCallback(
+    async (bytes: ArrayBuffer, atFrameIdx: number) => {
+      const blob = new Blob([bytes], { type: "image/jpeg" });
+      let bitmap: ImageBitmap;
+      try { bitmap = await createImageBitmap(blob); }
+      catch (e) { console.error("createImageBitmap failed", e); return; }
+      rememberBitmap(atFrameIdx, bitmap);
+      if (requestedIdx.current !== atFrameIdx) return;
+      drawBitmap(bitmap, atFrameIdx);
+    },
+    [drawBitmap, rememberBitmap]
+  );
+
  const pump = useCallback(async () => {
    if (decodingInFlight.current) return;
    decodingInFlight.current = true;
    try {
      while (requestedIdx.current !== renderedIdx.current) {
        const want = requestedIdx.current;
+        const cached = getCachedBitmap(want);
+        if (cached) {
+          drawBitmap(cached, want);
+          renderedIdx.current = want;
+          continue;
+        }
        try {
          const bytes = await api.decodeFrame(want);
          if (requestedIdx.current !== want) continue;
@@ -112,7 +160,7 @@ export function ExtractMode({ username }: Props) {
    } finally {
      decodingInFlight.current = false;
    }
-  }, [renderFrame]);
+  }, [drawBitmap, getCachedBitmap, renderFrame]);

  useEffect(() => {
    if (!video) return;
@@ -121,9 +169,17 @@ export function ExtractMode({ username }: Props) {
  }, [frameIdx, pump, video]);

  useEffect(() => {
-    renderedIdx.current = -1;
-    pump();
-  }, [overlayVisible, frameMap, pump]);
+    const cached = getCachedBitmap(frameIdxRef.current);
+    if (cached) {
+      drawBitmap(cached, frameIdxRef.current);
+      renderedIdx.current = frameIdxRef.current;
+    } else {
+      renderedIdx.current = -1;
+      pump();
+    }
+  }, [overlayVisible, frameMap, drawBitmap, getCachedBitmap, pump]);
+
+  useEffect(() => clearBitmapCache, [clearBitmapCache]);

  // --- playback loop ---------------------------------------------------
  useEffect(() => {
@@ -142,7 +198,16 @@ export function ExtractMode({ username }: Props) {
        video.total_frames - 1,
        anchor.idx + Math.floor(elapsed * video.fps * speed)
      );
-      if (reviewMode && annotatedFrames.current.has(want)) {
+      // Only trigger the review-mode auto-pause once we've actually advanced
+      // past the resume point. Otherwise pressing play while parked on an
+      // annotated frame would re-pause on the very first tick, making the
+      // play button look broken — the user would have to step a frame
+      // forward manually before play could "stick".
+      if (
+        reviewMode &&
+        want > anchor.idx &&
+        annotatedFrames.current.has(want)
+      ) {
        setFrameIdx(want);
        setPlaying(false);
        return;
@@ -162,19 +227,28 @@ export function ExtractMode({ username }: Props) {
  const seekToFrame = useCallback((idx: number) => {
    if (!video) return;
    const clamped = Math.max(0, Math.min(video.total_frames - 1, idx));
-    setFrameIdx(clamped);
+    frameIdxRef.current = clamped;
+    requestedIdx.current = clamped;
+    const cached = getCachedBitmap(clamped);
+    if (cached) {
+      drawBitmap(cached, clamped);
+      renderedIdx.current = clamped;
+    } else {
+      pump();
+    }
+    setFrameIdx((prev) => (prev === clamped ? prev : clamped));
    // If a playback tick is active, rebase its anchor so the loop continues
    // from the new position instead of snapping back.
    if (playAnchorRef.current) {
      playAnchorRef.current = { time: performance.now(), idx: clamped };
    }
-  }, [video]);
+  }, [drawBitmap, getCachedBitmap, pump, video]);

  const stepFrames = useCallback((delta: number) => {
    if (!video) return;
    setPlaying(false);
-    setFrameIdx((i) => Math.max(0, Math.min(video.total_frames - 1, i + delta)));
-  }, [video]);
+    seekToFrame(frameIdxRef.current + delta);
+  }, [seekToFrame, video]);

  const togglePlay = useCallback(() => {
    if (!video) return;
@@ -213,6 +287,7 @@ export function ExtractMode({ username }: Props) {

  const extractCurrentFrame = useCallback(async () => {
    if (!outputFolder || !video) return;
+    setPlaying(false);
    setBulkError(null);
    try {
      const shapes = frameMap.get(frameIdx) ?? [];
@@ -225,6 +300,7 @@ export function ExtractMode({ username }: Props) {

  const undoCurrentFrame = useCallback(async () => {
    if (!outputFolder) return;
+    setPlaying(false);
    setBulkError(null);
    try {
      await api.undoExtract(frameIdx, outputFolder);
@@ -252,18 +328,28 @@ export function ExtractMode({ username }: Props) {

  async function runBulkExtract() {
    if (!outputFolder || !bulkRange) return;
+    setPlaying(false);
    setBulkError(null);
    setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "extract" });
    const unlisten = await listen<{ done: number; total: number }>(
      "bulk-extract-progress",
      (e) => setBulkProgress({ ...e.payload, phase: "extract" })
    );
+    const shapesByFrame: Record<string, CocoShape[]> = {};
+    if (coco) {
+      for (const [key, shapes] of Object.entries(coco.frames)) {
+        const idx = Number(key);
+        if (idx >= bulkRange.start && idx <= bulkRange.end) {
+          shapesByFrame[key] = shapes;
+        }
+      }
+    }
    try {
      await api.bulkExtract(
        bulkRange.start,
        bulkRange.end,
        outputFolder,
-        coco?.frames ?? {},
+        shapesByFrame,
        username
      );
      const existing = await api.listExtractedFrames(outputFolder);
@@ -278,6 +364,7 @@ export function ExtractMode({ username }: Props) {

  async function runBulkUndo() {
    if (!outputFolder || !bulkRange) return;
+    setPlaying(false);
    setBulkError(null);
    setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "undo" });
    const unlisten = await listen<{ done: number; total: number }>(
@@ -296,6 +383,18 @@ export function ExtractMode({ username }: Props) {
    }
  }

+  const extractCurrentFrameRef = useRef(extractCurrentFrame);
+  const undoCurrentFrameRef = useRef(undoCurrentFrame);
+  const setRangeStartRef = useRef(setRangeStart);
+  const setRangeEndRef = useRef(setRangeEnd);
+
+  useEffect(() => {
+    extractCurrentFrameRef.current = extractCurrentFrame;
+    undoCurrentFrameRef.current = undoCurrentFrame;
+    setRangeStartRef.current = setRangeStart;
+    setRangeEndRef.current = setRangeEnd;
+  }, [extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]);
+
  // --- keyboard --------------------------------------------------------
  useEffect(() => {
    function onKey(e: KeyboardEvent) {
@@ -316,18 +415,18 @@ export function ExtractMode({ username }: Props) {
        case "h": case "H":
          e.preventDefault(); setOverlayVisible((v) => !v); break;
        case "e": case "E":
-          e.preventDefault(); extractCurrentFrame(); break;
+          e.preventDefault(); extractCurrentFrameRef.current(); break;
        case "u": case "U":
-          e.preventDefault(); undoCurrentFrame(); break;
+          e.preventDefault(); undoCurrentFrameRef.current(); break;
        case "[":
-          e.preventDefault(); setRangeStart(); break;
+          e.preventDefault(); setRangeStartRef.current(); break;
        case "]":
-          e.preventDefault(); setRangeEnd(); break;
+          e.preventDefault(); setRangeEndRef.current(); break;
      }
    }
    window.addEventListener("keydown", onKey);
    return () => window.removeEventListener("keydown", onKey);
-  }, [togglePlay, stepFrames, bumpSpeed, extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]);
+  }, [togglePlay, stepFrames, bumpSpeed]);

  // --- video/coco open -------------------------------------------------
  useEffect(() => {
@@ -358,6 +457,7 @@ export function ExtractMode({ username }: Props) {
    if (!selected || typeof selected !== "string") return;
    try {
      const info = await api.openVideo(selected);
+      clearBitmapCache();
      setVideo(info);
      setFrameIdx(0);
      setPlaying(false);