playback and import issues

2026-05-19 14:05:47 +05:30
parent f744747167
commit 0b5510636e
7 changed files with 377 additions and 132 deletions
--- a/sam-tool-tauri/src/components/shell/SettingsModal.tsx
+++ b/sam-tool-tauri/src/components/shell/SettingsModal.tsx
@@ -74,6 +74,7 @@ export function SettingsModal({ open, onClose, onSaved }: Props) {
    setError(null);
    try {
      await api.saveSettings(settings);
+      await api.setCacheCapacity(settings.cache_capacity).catch(() => {});
      onSaved?.(settings);
      onClose();
    } catch (e) {
@@ -238,8 +239,8 @@ export function SettingsModal({ open, onClose, onSaved }: Props) {
              </div>
              <input
                type="range"
-                min={200}
-                max={2000}
+                min={50}
+                max={800}
                step={50}
                value={settings.cache_capacity}
                onChange={(e) =>
@@ -250,9 +251,9 @@ export function SettingsModal({ open, onClose, onSaved }: Props) {
                }
              />
              <div className="range-ticks">
-                <span>200</span>
-                <span>1000</span>
-                <span>2000</span>
+                <span>50</span>
+                <span>400</span>
+                <span>800</span>
              </div>
            </label>
          </div>
--- a/sam-tool-tauri/src/modes/AnnotateMode.tsx
+++ b/sam-tool-tauri/src/modes/AnnotateMode.tsx
@@ -15,8 +15,6 @@ import { DeleteConfirmModal } from "../components/annotate/DeleteConfirmModal";
 import { drawOverlay, colorForClass } from "../components/extract/overlay";
 import type { Settings } from "../types";

-const DEFAULT_CLASS_ID = 0;
-
 interface Props {
  username: string;
  settings: Settings | null;
@@ -109,7 +107,7 @@ export function AnnotateMode({ username, settings }: Props) {

  const canvasRef = useRef<HTMLCanvasElement>(null);
  const bitmapRef = useRef<ImageBitmap | null>(null);
-  const loadedForImageRef = useRef<string | null>(null);
+  const imageLoadSeqRef = useRef(0);
  const stageRef = useRef<HTMLDivElement>(null);
  // Mirror of `classes` that can be updated synchronously — needed because
  // `pickerAddClass` calls setClasses and then immediately calls onConfirm,
@@ -180,7 +178,7 @@ export function AnnotateMode({ username, settings }: Props) {
      setDims(null);
      bitmapRef.current?.close();
      bitmapRef.current = null;
-      loadedForImageRef.current = null;
+      imageLoadSeqRef.current += 1;
      resetZoom();
      setDrawing(null);
    } catch (e) {
@@ -366,82 +364,60 @@ export function AnnotateMode({ username, settings }: Props) {
    }
  }, [ann, overlayVisible, drawing, drawInProgress, selectedAnnId, bboxDrag, vertexDrag]);

-  const loadImage = useCallback(async (imgPath: string) => {
-    try {
-      const bytes = await api.readImageBytes(imgPath);
-      const blob = new Blob([bytes], { type: "image/jpeg" });
-      const bitmap = await createImageBitmap(blob);
+  useEffect(() => {
+    if (!selectedImagePath) {
+      setAnn(null);
+      setDims(null);
      bitmapRef.current?.close();
-      bitmapRef.current = bitmap;
-      loadedForImageRef.current = imgPath;
-      setDims((prev) =>
-        prev && prev.w === bitmap.width && prev.h === bitmap.height
-          ? prev
-          : { w: bitmap.width, h: bitmap.height }
-      );
-    } catch (e) {
-      setLoadError(`image load: ${e}`);
+      bitmapRef.current = null;
+      imageLoadSeqRef.current += 1;
+      return;
    }
-  }, []);
-
-  useEffect(() => {
-    if (!selectedImagePath) return;
-    let cancelled = false;
-    setLoadError(null);
-    api
-      .loadImageAnnotations(selectedImagePath)
-      .then((loaded) => {
-        if (cancelled) return;
-        setAnn(
-          loaded ??
-          (dims
-            ? {
-              version: 1,
-              image: selectedImagePath.split("/").pop() ?? "",
-              width: dims.w,
-              height: dims.h,
-              annotations: [],
-            }
-            : null)
-        );
-      })
-      .catch((e) => {
-        if (cancelled) return;
-        setAnn(null);
-        setLoadError(String(e));
-      });
-    return () => {
-      cancelled = true;
-    };
-    // Intentionally not depending on `dims` — first selection may still have
-    // null dims; subsequent effect from loadImage will fill them and this
-    // effect reruns on path change only.
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [selectedImagePath]);
-
-  // Hold a ref to the latest drawCanvas so the image-change effect can call
-  // it without subscribing to its identity (which flips on every `drawing` /
-  // `ann` / `overlayVisible` change and would reset state mid-interaction).
-  const drawCanvasRef = useRef<() => void>(() => { });
-  useEffect(() => {
-    drawCanvasRef.current = drawCanvas;
-  }, [drawCanvas]);
-
-  // On image selection change: reset per-image state, load bytes, draw once.
-  useEffect(() => {
-    if (!selectedImagePath) return;
+    const seq = imageLoadSeqRef.current + 1;
+    imageLoadSeqRef.current = seq;
    resetZoom();
+    setLoadError(null);
+    setAnn(null);
+    setDims(null);
    setDrawing(null);
    setSelectedAnnId(null);
    setBboxDrag(null);
    setVertexDrag(null);
    undoStackRef.current = [];
    redoStackRef.current = [];
-    if (loadedForImageRef.current === selectedImagePath) {
-      drawCanvasRef.current();
-    } else {
-      loadImage(selectedImagePath).then(() => drawCanvasRef.current());
+
+    async function loadSelectedImage() {
+      try {
+        const [bytes, loaded] = await Promise.all([
+          api.readImageBytes(selectedImagePath!),
+          api.loadImageAnnotations(selectedImagePath!),
+        ]);
+        const blob = new Blob([bytes], { type: "image/jpeg" });
+        const bitmap = await createImageBitmap(blob);
+        if (imageLoadSeqRef.current !== seq) {
+          bitmap.close();
+          return;
+        }
+        bitmapRef.current?.close();
+        bitmapRef.current = bitmap;
+        const nextDims = { w: bitmap.width, h: bitmap.height };
+        setDims(nextDims);
+        setAnn(
+          loaded ?? {
+            version: 1,
+            image: selectedImagePath!.split("/").pop() ?? "",
+            width: nextDims.w,
+            height: nextDims.h,
+            annotations: [],
+          }
+        );
+      } catch (e) {
+        if (imageLoadSeqRef.current !== seq) return;
+        setAnn(null);
+        setLoadError(`image load: ${e}`);
+      }
    }
+    void loadSelectedImage();
    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [selectedImagePath]);

@@ -942,12 +918,8 @@ export function AnnotateMode({ username, settings }: Props) {
      setSelectedAnnId(null);
      return;
    }
-    // if (activeClassId == null) {
-    //   setStatusMsg("Pick a class on the right first");
-    //   return;
-    // }
-    const classId = activeClassId ?? DEFAULT_CLASS_ID;
-
+    // Class id isn't needed until the LabelPicker confirms after the shape
+    // is drawn — drawing itself doesn't depend on `activeClassId`.
    e.stopPropagation(); // don't let the pan handler see this
    const p = canvasToImageCoords(e.clientX, e.clientY);
    if (!p) return;
--- a/sam-tool-tauri/src/modes/ExtractMode.tsx
+++ b/sam-tool-tauri/src/modes/ExtractMode.tsx
@@ -7,6 +7,7 @@ import { FrameCanvas } from "../components/extract/FrameCanvas";
 import { drawOverlay } from "../components/extract/overlay";

 const SPEEDS = [0.25, 0.5, 1, 1.5, 2, 3, 4];
+const FRAME_BITMAP_CACHE_LIMIT = 24;

 interface Props {
  username: string;
@@ -67,20 +68,48 @@ export function ExtractMode({ username }: Props) {
  const requestedIdx = useRef(0);
  const renderedIdx = useRef(-1);
  const decodingInFlight = useRef(false);
+  const bitmapCacheRef = useRef<Map<number, ImageBitmap>>(new Map());

-  const renderFrame = useCallback(
-    async (bytes: ArrayBuffer, atFrameIdx: number) => {
+  const clearBitmapCache = useCallback(() => {
+    for (const bitmap of bitmapCacheRef.current.values()) {
+      bitmap.close();
+    }
+    bitmapCacheRef.current.clear();
+  }, []);
+
+  const getCachedBitmap = useCallback((idx: number) => {
+    const cache = bitmapCacheRef.current;
+    const bitmap = cache.get(idx);
+    if (!bitmap) return null;
+    cache.delete(idx);
+    cache.set(idx, bitmap);
+    return bitmap;
+  }, []);
+
+  const rememberBitmap = useCallback((idx: number, bitmap: ImageBitmap) => {
+    const cache = bitmapCacheRef.current;
+    const existing = cache.get(idx);
+    if (existing && existing !== bitmap) {
+      existing.close();
+    }
+    cache.delete(idx);
+    cache.set(idx, bitmap);
+    while (cache.size > FRAME_BITMAP_CACHE_LIMIT) {
+      const oldest = cache.keys().next().value;
+      if (oldest == null) break;
+      cache.get(oldest)?.close();
+      cache.delete(oldest);
+    }
+  }, []);
+
+  const drawBitmap = useCallback(
+    (bitmap: ImageBitmap, atFrameIdx: number) => {
      const canvas = canvasRef.current;
      if (!canvas) return;
-      const blob = new Blob([bytes], { type: "image/jpeg" });
-      let bitmap: ImageBitmap;
-      try { bitmap = await createImageBitmap(blob); }
-      catch (e) { console.error("createImageBitmap failed", e); return; }
      const ctx = canvas.getContext("2d");
-      if (!ctx) { bitmap.close(); return; }
+      if (!ctx) return;
      ctx.clearRect(0, 0, canvas.width, canvas.height);
      ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
-      bitmap.close();

      if (overlayVisibleRef.current) {
        const shapes = frameMapRef.current.get(atFrameIdx);
@@ -92,12 +121,31 @@ export function ExtractMode({ username }: Props) {
    []
  );

+  const renderFrame = useCallback(
+    async (bytes: ArrayBuffer, atFrameIdx: number) => {
+      const blob = new Blob([bytes], { type: "image/jpeg" });
+      let bitmap: ImageBitmap;
+      try { bitmap = await createImageBitmap(blob); }
+      catch (e) { console.error("createImageBitmap failed", e); return; }
+      rememberBitmap(atFrameIdx, bitmap);
+      if (requestedIdx.current !== atFrameIdx) return;
+      drawBitmap(bitmap, atFrameIdx);
+    },
+    [drawBitmap, rememberBitmap]
+  );
+
  const pump = useCallback(async () => {
    if (decodingInFlight.current) return;
    decodingInFlight.current = true;
    try {
      while (requestedIdx.current !== renderedIdx.current) {
        const want = requestedIdx.current;
+        const cached = getCachedBitmap(want);
+        if (cached) {
+          drawBitmap(cached, want);
+          renderedIdx.current = want;
+          continue;
+        }
        try {
          const bytes = await api.decodeFrame(want);
          if (requestedIdx.current !== want) continue;
@@ -112,7 +160,7 @@ export function ExtractMode({ username }: Props) {
    } finally {
      decodingInFlight.current = false;
    }
-  }, [renderFrame]);
+  }, [drawBitmap, getCachedBitmap, renderFrame]);

  useEffect(() => {
    if (!video) return;
@@ -121,9 +169,17 @@ export function ExtractMode({ username }: Props) {
  }, [frameIdx, pump, video]);

  useEffect(() => {
-    renderedIdx.current = -1;
-    pump();
-  }, [overlayVisible, frameMap, pump]);
+    const cached = getCachedBitmap(frameIdxRef.current);
+    if (cached) {
+      drawBitmap(cached, frameIdxRef.current);
+      renderedIdx.current = frameIdxRef.current;
+    } else {
+      renderedIdx.current = -1;
+      pump();
+    }
+  }, [overlayVisible, frameMap, drawBitmap, getCachedBitmap, pump]);
+
+  useEffect(() => clearBitmapCache, [clearBitmapCache]);

  // --- playback loop ---------------------------------------------------
  useEffect(() => {
@@ -142,7 +198,16 @@ export function ExtractMode({ username }: Props) {
        video.total_frames - 1,
        anchor.idx + Math.floor(elapsed * video.fps * speed)
      );
-      if (reviewMode && annotatedFrames.current.has(want)) {
+      // Only trigger the review-mode auto-pause once we've actually advanced
+      // past the resume point. Otherwise pressing play while parked on an
+      // annotated frame would re-pause on the very first tick, making the
+      // play button look broken — the user would have to step a frame
+      // forward manually before play could "stick".
+      if (
+        reviewMode &&
+        want > anchor.idx &&
+        annotatedFrames.current.has(want)
+      ) {
        setFrameIdx(want);
        setPlaying(false);
        return;
@@ -162,19 +227,28 @@ export function ExtractMode({ username }: Props) {
  const seekToFrame = useCallback((idx: number) => {
    if (!video) return;
    const clamped = Math.max(0, Math.min(video.total_frames - 1, idx));
-    setFrameIdx(clamped);
+    frameIdxRef.current = clamped;
+    requestedIdx.current = clamped;
+    const cached = getCachedBitmap(clamped);
+    if (cached) {
+      drawBitmap(cached, clamped);
+      renderedIdx.current = clamped;
+    } else {
+      pump();
+    }
+    setFrameIdx((prev) => (prev === clamped ? prev : clamped));
    // If a playback tick is active, rebase its anchor so the loop continues
    // from the new position instead of snapping back.
    if (playAnchorRef.current) {
      playAnchorRef.current = { time: performance.now(), idx: clamped };
    }
-  }, [video]);
+  }, [drawBitmap, getCachedBitmap, pump, video]);

  const stepFrames = useCallback((delta: number) => {
    if (!video) return;
    setPlaying(false);
-    setFrameIdx((i) => Math.max(0, Math.min(video.total_frames - 1, i + delta)));
-  }, [video]);
+    seekToFrame(frameIdxRef.current + delta);
+  }, [seekToFrame, video]);

  const togglePlay = useCallback(() => {
    if (!video) return;
@@ -213,6 +287,7 @@ export function ExtractMode({ username }: Props) {

  const extractCurrentFrame = useCallback(async () => {
    if (!outputFolder || !video) return;
+    setPlaying(false);
    setBulkError(null);
    try {
      const shapes = frameMap.get(frameIdx) ?? [];
@@ -225,6 +300,7 @@ export function ExtractMode({ username }: Props) {

  const undoCurrentFrame = useCallback(async () => {
    if (!outputFolder) return;
+    setPlaying(false);
    setBulkError(null);
    try {
      await api.undoExtract(frameIdx, outputFolder);
@@ -252,18 +328,28 @@ export function ExtractMode({ username }: Props) {

  async function runBulkExtract() {
    if (!outputFolder || !bulkRange) return;
+    setPlaying(false);
    setBulkError(null);
    setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "extract" });
    const unlisten = await listen<{ done: number; total: number }>(
      "bulk-extract-progress",
      (e) => setBulkProgress({ ...e.payload, phase: "extract" })
    );
+    const shapesByFrame: Record<string, CocoShape[]> = {};
+    if (coco) {
+      for (const [key, shapes] of Object.entries(coco.frames)) {
+        const idx = Number(key);
+        if (idx >= bulkRange.start && idx <= bulkRange.end) {
+          shapesByFrame[key] = shapes;
+        }
+      }
+    }
    try {
      await api.bulkExtract(
        bulkRange.start,
        bulkRange.end,
        outputFolder,
-        coco?.frames ?? {},
+        shapesByFrame,
        username
      );
      const existing = await api.listExtractedFrames(outputFolder);
@@ -278,6 +364,7 @@ export function ExtractMode({ username }: Props) {

  async function runBulkUndo() {
    if (!outputFolder || !bulkRange) return;
+    setPlaying(false);
    setBulkError(null);
    setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "undo" });
    const unlisten = await listen<{ done: number; total: number }>(
@@ -296,6 +383,18 @@ export function ExtractMode({ username }: Props) {
    }
  }

+  const extractCurrentFrameRef = useRef(extractCurrentFrame);
+  const undoCurrentFrameRef = useRef(undoCurrentFrame);
+  const setRangeStartRef = useRef(setRangeStart);
+  const setRangeEndRef = useRef(setRangeEnd);
+
+  useEffect(() => {
+    extractCurrentFrameRef.current = extractCurrentFrame;
+    undoCurrentFrameRef.current = undoCurrentFrame;
+    setRangeStartRef.current = setRangeStart;
+    setRangeEndRef.current = setRangeEnd;
+  }, [extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]);
+
  // --- keyboard --------------------------------------------------------
  useEffect(() => {
    function onKey(e: KeyboardEvent) {
@@ -316,18 +415,18 @@ export function ExtractMode({ username }: Props) {
        case "h": case "H":
          e.preventDefault(); setOverlayVisible((v) => !v); break;
        case "e": case "E":
-          e.preventDefault(); extractCurrentFrame(); break;
+          e.preventDefault(); extractCurrentFrameRef.current(); break;
        case "u": case "U":
-          e.preventDefault(); undoCurrentFrame(); break;
+          e.preventDefault(); undoCurrentFrameRef.current(); break;
        case "[":
-          e.preventDefault(); setRangeStart(); break;
+          e.preventDefault(); setRangeStartRef.current(); break;
        case "]":
-          e.preventDefault(); setRangeEnd(); break;
+          e.preventDefault(); setRangeEndRef.current(); break;
      }
    }
    window.addEventListener("keydown", onKey);
    return () => window.removeEventListener("keydown", onKey);
-  }, [togglePlay, stepFrames, bumpSpeed, extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]);
+  }, [togglePlay, stepFrames, bumpSpeed]);

  // --- video/coco open -------------------------------------------------
  useEffect(() => {
@@ -358,6 +457,7 @@ export function ExtractMode({ username }: Props) {
    if (!selected || typeof selected !== "string") return;
    try {
      const info = await api.openVideo(selected);
+      clearBitmapCache();
      setVideo(info);
      setFrameIdx(0);
      setPlaying(false);