1000 lines
36 KiB
TypeScript
1000 lines
36 KiB
TypeScript
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||
import { open as openDialog } from "@tauri-apps/plugin-dialog";
|
||
import { listen } from "@tauri-apps/api/event";
|
||
import { api } from "../ipc";
|
||
import type { CocoShape, ImportedCoco, Settings, VideoInfo } from "../types";
|
||
import { FrameCanvas } from "../components/extract/FrameCanvas";
|
||
import { drawOverlay } from "../components/extract/overlay";
|
||
|
||
const SPEEDS = [0.25, 0.5, 1, 1.5, 2, 3, 4];
|
||
const FRAME_BITMAP_CACHE_LIMIT = 24;
|
||
|
||
interface Props {
|
||
username: string;
|
||
settings: Settings | null;
|
||
}
|
||
|
||
interface BulkProgress {
|
||
done: number;
|
||
total: number;
|
||
phase: "extract" | "undo";
|
||
}
|
||
|
||
export function ExtractMode({ username, settings }: Props) {
|
||
// frame_skip ≥ 1 means: during playback advance by N frames per tick, and
|
||
// bulk-extract takes every Nth frame across [start, end]. Kept in a ref so
|
||
// the long-lived playback loop sees changes without re-running its effect.
|
||
const frameSkipRef = useRef<number>(Math.max(1, settings?.frame_skip ?? 1));
|
||
useEffect(() => {
|
||
frameSkipRef.current = Math.max(1, settings?.frame_skip ?? 1);
|
||
}, [settings?.frame_skip]);
|
||
// Preview-quality cap (in px height; 0 = original). Read per-draw via ref
|
||
// so a settings change triggers an immediate redraw of the current frame
|
||
// without rebuilding the playback effect.
|
||
const previewMaxHRef = useRef<number>(settings?.preview_max_height ?? 0);
|
||
useEffect(() => {
|
||
previewMaxHRef.current = settings?.preview_max_height ?? 0;
|
||
}, [settings?.preview_max_height]);
|
||
const [video, setVideo] = useState<VideoInfo | null>(null);
|
||
const [frameIdx, setFrameIdx] = useState(0);
|
||
const [playing, setPlaying] = useState(false);
|
||
const [speed, setSpeed] = useState(1);
|
||
const [reviewMode, setReviewMode] = useState(false);
|
||
const [overlayVisible, setOverlayVisible] = useState(true);
|
||
const [loadError, setLoadError] = useState<string | null>(null);
|
||
|
||
const [coco, setCoco] = useState<ImportedCoco | null>(null);
|
||
|
||
// Extraction state
|
||
const [outputFolder, setOutputFolder] = useState<string | null>(null);
|
||
const [extractedFrames, setExtractedFrames] = useState<Set<number>>(new Set());
|
||
const [bulkStart, setBulkStart] = useState<number | null>(null);
|
||
const [bulkEnd, setBulkEnd] = useState<number | null>(null);
|
||
const [bulkProgress, setBulkProgress] = useState<BulkProgress | null>(null);
|
||
const [bulkError, setBulkError] = useState<string | null>(null);
|
||
const [infoCollapsed, setInfoCollapsed] = useState(false);
|
||
|
||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||
|
||
const frameMap = useMemo(() => {
|
||
const m = new Map<number, CocoShape[]>();
|
||
if (!coco) return m;
|
||
for (const [k, v] of Object.entries(coco.frames)) m.set(Number(k), v);
|
||
return m;
|
||
}, [coco]);
|
||
|
||
// Refs mirroring state for effects that shouldn't depend on fast-changing values.
|
||
const frameIdxRef = useRef(0);
|
||
const frameMapRef = useRef<Map<number, CocoShape[]>>(new Map());
|
||
const overlayVisibleRef = useRef(overlayVisible);
|
||
const annotatedFrames = useRef<Set<number>>(new Set());
|
||
|
||
useEffect(() => { frameIdxRef.current = frameIdx; }, [frameIdx]);
|
||
useEffect(() => {
|
||
frameMapRef.current = frameMap;
|
||
annotatedFrames.current = new Set(frameMap.keys());
|
||
}, [frameMap]);
|
||
useEffect(() => { overlayVisibleRef.current = overlayVisible; }, [overlayVisible]);
|
||
|
||
// --- decode pump (latest-wins coalescing) ----------------------------
|
||
const requestedIdx = useRef(0);
|
||
const renderedIdx = useRef(-1);
|
||
const decodingInFlight = useRef(false);
|
||
const bitmapCacheRef = useRef<Map<number, ImageBitmap>>(new Map());
|
||
// Last decode/render error message. Set in `pump` on failure, cleared on
|
||
// any successful render, and observed by the playback loop so a mid-play
|
||
// failure stops playback with a visible banner instead of leaving the UI
|
||
// stuck in "playing" while the canvas freezes.
|
||
const decodeErrorRef = useRef<string | null>(null);
|
||
const [playbackError, setPlaybackError] = useState<string | null>(null);
|
||
// Cached canvas CSS rect. drawBitmap used to call getBoundingClientRect()
|
||
// every frame, which forces a synchronous layout flush each time. We now
|
||
// refresh it from the ResizeObserver below and fall back to a one-shot
|
||
// query if it isn't populated yet.
|
||
const canvasRectRef = useRef<{ width: number; height: number } | null>(null);
|
||
|
||
const clearBitmapCache = useCallback(() => {
|
||
for (const bitmap of bitmapCacheRef.current.values()) {
|
||
bitmap.close();
|
||
}
|
||
bitmapCacheRef.current.clear();
|
||
}, []);
|
||
|
||
const getCachedBitmap = useCallback((idx: number) => {
|
||
const cache = bitmapCacheRef.current;
|
||
const bitmap = cache.get(idx);
|
||
if (!bitmap) return null;
|
||
cache.delete(idx);
|
||
cache.set(idx, bitmap);
|
||
return bitmap;
|
||
}, []);
|
||
|
||
const rememberBitmap = useCallback((idx: number, bitmap: ImageBitmap) => {
|
||
const cache = bitmapCacheRef.current;
|
||
const existing = cache.get(idx);
|
||
if (existing && existing !== bitmap) {
|
||
existing.close();
|
||
}
|
||
cache.delete(idx);
|
||
cache.set(idx, bitmap);
|
||
while (cache.size > FRAME_BITMAP_CACHE_LIMIT) {
|
||
const oldest = cache.keys().next().value;
|
||
if (oldest == null) break;
|
||
cache.get(oldest)?.close();
|
||
cache.delete(oldest);
|
||
}
|
||
}, []);
|
||
|
||
const drawBitmap = useCallback(
|
||
(bitmap: ImageBitmap, atFrameIdx: number) => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas || !video) return;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
|
||
// Backing store = displayed CSS rect × DPR, capped at video native and
|
||
// optionally at the user's Preview-quality preset. Halves drawImage
|
||
// cost on 4K-into-1080p (the common slow-system case). Coords below
|
||
// are in image-pixel space — the transform maps to backing pixels so
|
||
// overlay.ts is unchanged. The rect is cached by the ResizeObserver
|
||
// effect; we only fall back to a live query on the first frame.
|
||
let rect = canvasRectRef.current;
|
||
if (!rect) {
|
||
const dom = canvas.getBoundingClientRect();
|
||
rect = { width: dom.width, height: dom.height };
|
||
canvasRectRef.current = rect;
|
||
}
|
||
const dpr = window.devicePixelRatio || 1;
|
||
let maxW = video.width;
|
||
let maxH = video.height;
|
||
const previewMaxH = previewMaxHRef.current;
|
||
if (previewMaxH > 0 && previewMaxH < video.height) {
|
||
maxH = previewMaxH;
|
||
maxW = Math.round(previewMaxH * (video.width / video.height));
|
||
}
|
||
const targetW = Math.max(
|
||
1,
|
||
Math.min(maxW, Math.round(rect.width * dpr))
|
||
);
|
||
const targetH = Math.max(
|
||
1,
|
||
Math.min(maxH, Math.round(rect.height * dpr))
|
||
);
|
||
if (canvas.width !== targetW) canvas.width = targetW;
|
||
if (canvas.height !== targetH) canvas.height = targetH;
|
||
|
||
ctx.setTransform(
|
||
targetW / video.width,
|
||
0,
|
||
0,
|
||
targetH / video.height,
|
||
0,
|
||
0
|
||
);
|
||
ctx.clearRect(0, 0, video.width, video.height);
|
||
ctx.drawImage(bitmap, 0, 0, video.width, video.height);
|
||
|
||
if (overlayVisibleRef.current) {
|
||
const shapes = frameMapRef.current.get(atFrameIdx);
|
||
if (shapes && shapes.length > 0) {
|
||
drawOverlay(ctx, shapes, video.width, video.height);
|
||
}
|
||
}
|
||
ctx.setTransform(1, 0, 0, 1, 0, 0);
|
||
},
|
||
[video]
|
||
);
|
||
|
||
// Returns true if we drew, false if we were preempted by a newer request,
|
||
// and throws if bitmap creation failed — pump uses this to decide whether
|
||
// to mark the frame rendered or surface the error to the playback loop.
|
||
const renderFrame = useCallback(
|
||
async (bytes: ArrayBuffer, atFrameIdx: number): Promise<boolean> => {
|
||
// Diagnostic timing log: set `window.SAM_FRAME_DEBUG = true` in devtools
|
||
// on a slow machine to see per-frame bitmap and draw timings — confirms
|
||
// where the bottleneck lives (decode vs createImageBitmap vs drawImage).
|
||
const debug = (window as unknown as { SAM_FRAME_DEBUG?: boolean })
|
||
.SAM_FRAME_DEBUG === true;
|
||
const t0 = debug ? performance.now() : 0;
|
||
const blob = new Blob([bytes], { type: "image/jpeg" });
|
||
let bitmap: ImageBitmap;
|
||
try {
|
||
bitmap = await createImageBitmap(blob);
|
||
} catch (e) {
|
||
throw new Error(
|
||
`createImageBitmap failed: ${e instanceof Error ? e.message : String(e)}`
|
||
);
|
||
}
|
||
const t1 = debug ? performance.now() : 0;
|
||
rememberBitmap(atFrameIdx, bitmap);
|
||
if (requestedIdx.current !== atFrameIdx) return false;
|
||
drawBitmap(bitmap, atFrameIdx);
|
||
if (debug) {
|
||
const t2 = performance.now();
|
||
// eslint-disable-next-line no-console
|
||
console.debug(
|
||
`[SAM] f${atFrameIdx} bytes=${bytes.byteLength} bitmap=${(t1 - t0).toFixed(0)}ms draw=${(t2 - t1).toFixed(0)}ms`
|
||
);
|
||
}
|
||
return true;
|
||
},
|
||
[drawBitmap, rememberBitmap]
|
||
);
|
||
|
||
const pump = useCallback(async () => {
|
||
if (decodingInFlight.current) return;
|
||
decodingInFlight.current = true;
|
||
try {
|
||
while (requestedIdx.current !== renderedIdx.current) {
|
||
const want = requestedIdx.current;
|
||
const cached = getCachedBitmap(want);
|
||
if (cached) {
|
||
drawBitmap(cached, want);
|
||
renderedIdx.current = want;
|
||
continue;
|
||
}
|
||
try {
|
||
const bytes = await api.decodeFrame(want);
|
||
if (requestedIdx.current !== want) continue;
|
||
const drew = await renderFrame(bytes, want);
|
||
if (drew) {
|
||
renderedIdx.current = want;
|
||
decodeErrorRef.current = null;
|
||
}
|
||
// !drew means renderFrame was preempted by a newer request; loop
|
||
// continues and the new target gets a fresh decode.
|
||
} catch (e) {
|
||
if (requestedIdx.current !== want) continue;
|
||
console.error("decode/render failed", e);
|
||
decodeErrorRef.current = e instanceof Error ? e.message : String(e);
|
||
break;
|
||
}
|
||
}
|
||
} finally {
|
||
decodingInFlight.current = false;
|
||
}
|
||
}, [drawBitmap, getCachedBitmap, renderFrame]);
|
||
|
||
useEffect(() => {
|
||
if (!video) return;
|
||
requestedIdx.current = frameIdx;
|
||
pump();
|
||
}, [frameIdx, pump, video]);
|
||
|
||
useEffect(() => {
|
||
const cached = getCachedBitmap(frameIdxRef.current);
|
||
if (cached) {
|
||
drawBitmap(cached, frameIdxRef.current);
|
||
renderedIdx.current = frameIdxRef.current;
|
||
} else {
|
||
renderedIdx.current = -1;
|
||
pump();
|
||
}
|
||
}, [overlayVisible, frameMap, drawBitmap, getCachedBitmap, pump]);
|
||
|
||
useEffect(() => clearBitmapCache, [clearBitmapCache]);
|
||
|
||
// Live re-render when the preview-quality cap changes so the user sees the
|
||
// resolution swap immediately on Settings save instead of having to seek.
|
||
useEffect(() => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas || !video) return;
|
||
const idx = frameIdxRef.current;
|
||
const cached = getCachedBitmap(idx);
|
||
if (cached) drawBitmap(cached, idx);
|
||
// If not cached, the next pump invocation picks up the new cap naturally.
|
||
}, [settings?.preview_max_height, video, drawBitmap, getCachedBitmap]);
|
||
|
||
// Redraw on canvas resize: backing-store is sized from CSS rect × DPR, so
|
||
// a window resize/maximize would otherwise leave the old backing dimensions
|
||
// until the next decode landed. Cheap — just re-runs drawBitmap on the
|
||
// already-cached frame.
|
||
useEffect(() => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas || !video) return;
|
||
let scheduled = false;
|
||
// Seed the rect cache once on mount so the first drawBitmap doesn't
|
||
// have to force a layout flush itself.
|
||
{
|
||
const dom = canvas.getBoundingClientRect();
|
||
canvasRectRef.current = { width: dom.width, height: dom.height };
|
||
}
|
||
const ro = new ResizeObserver((entries) => {
|
||
// Update the cached rect from the observer entry — it carries the
|
||
// CSS-box size without forcing a sync layout.
|
||
const last = entries[entries.length - 1];
|
||
const box = last?.contentBoxSize?.[0];
|
||
if (box) {
|
||
canvasRectRef.current = {
|
||
width: box.inlineSize,
|
||
height: box.blockSize,
|
||
};
|
||
} else if (last?.contentRect) {
|
||
canvasRectRef.current = {
|
||
width: last.contentRect.width,
|
||
height: last.contentRect.height,
|
||
};
|
||
}
|
||
if (scheduled) return;
|
||
scheduled = true;
|
||
requestAnimationFrame(() => {
|
||
scheduled = false;
|
||
const idx = frameIdxRef.current;
|
||
const cached = getCachedBitmap(idx);
|
||
if (cached) drawBitmap(cached, idx);
|
||
else { renderedIdx.current = -1; pump(); }
|
||
});
|
||
});
|
||
ro.observe(canvas);
|
||
return () => ro.disconnect();
|
||
}, [video, drawBitmap, getCachedBitmap, pump]);
|
||
|
||
// --- playback loop ---------------------------------------------------
|
||
// Render-paced: advance ONLY after the previous frame actually drew. On a
|
||
// fast box this hits target fps; on a slow box it plays smoothly at the
|
||
// achievable decode/render rate instead of dropping 9-of-10 frames at the
|
||
// canvas while the wall-clock UI counter races ahead.
|
||
useEffect(() => {
|
||
if (!playing || !video) return;
|
||
let cancelled = false;
|
||
let nextDueAt = performance.now();
|
||
|
||
const waitForFrame = (target: number) =>
|
||
new Promise<void>((resolve) => {
|
||
const check = () => {
|
||
if (
|
||
cancelled ||
|
||
renderedIdx.current === target ||
|
||
decodeErrorRef.current !== null
|
||
) {
|
||
resolve();
|
||
return;
|
||
}
|
||
requestAnimationFrame(check);
|
||
};
|
||
check();
|
||
});
|
||
|
||
const loop = async () => {
|
||
while (!cancelled && playing) {
|
||
const skip = Math.max(1, frameSkipRef.current | 0);
|
||
const current = frameIdxRef.current;
|
||
let target = Math.min(video.total_frames - 1, current + skip);
|
||
if (target <= current) {
|
||
setPlaying(false);
|
||
return;
|
||
}
|
||
// Review-mode scan: with skip > 1 a naive +skip jump can leap past
|
||
// annotations between current+1 and current+skip. Walk that window
|
||
// and clamp `target` down to the first annotated frame.
|
||
if (reviewMode) {
|
||
for (let i = current + 1; i <= target; i++) {
|
||
if (annotatedFrames.current.has(i)) {
|
||
target = i;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
requestedIdx.current = target;
|
||
// Fast path: await pump directly. When pump owns the queue it
|
||
// resolves the instant target is drawn (no 16ms rAF latency tax).
|
||
// If another caller (ResizeObserver / overlay-change effect) was
|
||
// already running pump, our call short-circuits — fall back to
|
||
// rAF polling in that case so we still wait correctly.
|
||
await pump();
|
||
if (
|
||
!cancelled &&
|
||
renderedIdx.current !== target &&
|
||
decodeErrorRef.current === null
|
||
) {
|
||
await waitForFrame(target);
|
||
}
|
||
if (cancelled) return;
|
||
// Decode failed mid-play: stop and surface so the UI doesn't claim to
|
||
// be playing while the canvas is frozen on the last good frame.
|
||
if (decodeErrorRef.current) {
|
||
setPlaybackError(decodeErrorRef.current);
|
||
setPlaying(false);
|
||
return;
|
||
}
|
||
const advanced = Math.max(1, target - current);
|
||
// Real-time pacing regardless of skip: skip controls density,
|
||
// `speed` controls speed. Without the advanced-frame factor here,
|
||
// skip=3 would play at 3x wall-clock, which is what the speed slider
|
||
// is for. Compute after review-mode clamping so annotated stops do not
|
||
// wait for the full skip window.
|
||
const interval = (advanced * 1000) / (video.fps * speed);
|
||
// Keep the fast playback loop's cursor in sync immediately. Waiting
|
||
// for React's frameIdx effect can make a slow decode path replay the
|
||
// same target once and add an extra frame interval per rendered frame.
|
||
frameIdxRef.current = target;
|
||
setFrameIdx(target);
|
||
// Pause AFTER rendering so the user actually sees the annotated frame.
|
||
if (reviewMode && annotatedFrames.current.has(target)) {
|
||
setPlaying(false);
|
||
return;
|
||
}
|
||
|
||
const now = performance.now();
|
||
const due = nextDueAt + interval;
|
||
if (due > now) {
|
||
await new Promise((r) => setTimeout(r, due - now));
|
||
nextDueAt = due;
|
||
} else {
|
||
// Behind real-time — don't accumulate debt and then "catch up" by
|
||
// skipping frames; just reset the clock to now.
|
||
nextDueAt = now;
|
||
}
|
||
}
|
||
};
|
||
loop();
|
||
return () => {
|
||
cancelled = true;
|
||
};
|
||
}, [playing, video, speed, reviewMode, pump]);
|
||
|
||
// --- navigation helpers ----------------------------------------------
|
||
const seekToFrame = useCallback((idx: number) => {
|
||
if (!video) return;
|
||
const clamped = Math.max(0, Math.min(video.total_frames - 1, idx));
|
||
frameIdxRef.current = clamped;
|
||
requestedIdx.current = clamped;
|
||
const cached = getCachedBitmap(clamped);
|
||
if (cached) {
|
||
drawBitmap(cached, clamped);
|
||
renderedIdx.current = clamped;
|
||
} else {
|
||
pump();
|
||
}
|
||
setFrameIdx((prev) => (prev === clamped ? prev : clamped));
|
||
}, [drawBitmap, getCachedBitmap, pump, video]);
|
||
|
||
const stepFrames = useCallback((delta: number) => {
|
||
if (!video) return;
|
||
setPlaying(false);
|
||
seekToFrame(frameIdxRef.current + delta);
|
||
}, [seekToFrame, video]);
|
||
|
||
const togglePlay = useCallback(() => {
|
||
if (!video) return;
|
||
setPlaying((p) => {
|
||
if (!p) {
|
||
// Fresh play attempt — clear any stale decode error so the loop can
|
||
// re-arm. If it fails again, the banner will reappear.
|
||
decodeErrorRef.current = null;
|
||
setPlaybackError(null);
|
||
}
|
||
return !p;
|
||
});
|
||
}, [video]);
|
||
|
||
const bumpSpeed = useCallback((dir: 1 | -1) => {
|
||
setSpeed((s) => {
|
||
const i = SPEEDS.indexOf(s);
|
||
const next = Math.max(0, Math.min(SPEEDS.length - 1, (i < 0 ? 2 : i) + dir));
|
||
return SPEEDS[next];
|
||
});
|
||
}, []);
|
||
|
||
// --- extraction helpers ----------------------------------------------
|
||
const applyOutputFolder = useCallback(async (path: string) => {
|
||
setOutputFolder(path);
|
||
try {
|
||
const existing = await api.listExtractedFrames(path);
|
||
setExtractedFrames(new Set(existing));
|
||
} catch (e) {
|
||
// Folder may not exist yet — that's fine, we'll create it on extract.
|
||
setExtractedFrames(new Set());
|
||
console.warn("listExtractedFrames failed (non-fatal):", e);
|
||
}
|
||
if (coco && coco.categories.length > 0) {
|
||
try { await api.writeClassesTxt(path, coco.categories); } catch (e) { console.error(e); }
|
||
}
|
||
}, [coco]);
|
||
|
||
async function pickOutputFolder() {
|
||
const selected = await openDialog({ directory: true, multiple: false });
|
||
if (!selected || typeof selected !== "string") return;
|
||
await applyOutputFolder(selected);
|
||
}
|
||
|
||
const extractCurrentFrame = useCallback(async () => {
|
||
if (!outputFolder || !video) return;
|
||
setPlaying(false);
|
||
setBulkError(null);
|
||
try {
|
||
const shapes = frameMap.get(frameIdx) ?? [];
|
||
await api.extractFrame(frameIdx, outputFolder, shapes, username);
|
||
setExtractedFrames((prev) => new Set(prev).add(frameIdx));
|
||
} catch (e) {
|
||
setBulkError(String(e));
|
||
}
|
||
}, [outputFolder, video, frameIdx, frameMap, username]);
|
||
|
||
const undoCurrentFrame = useCallback(async () => {
|
||
if (!outputFolder) return;
|
||
setPlaying(false);
|
||
setBulkError(null);
|
||
try {
|
||
await api.undoExtract(frameIdx, outputFolder);
|
||
setExtractedFrames((prev) => {
|
||
const n = new Set(prev);
|
||
n.delete(frameIdx);
|
||
return n;
|
||
});
|
||
} catch (e) {
|
||
setBulkError(String(e));
|
||
}
|
||
}, [outputFolder, frameIdx]);
|
||
|
||
const setRangeStart = useCallback(() => setBulkStart(frameIdx), [frameIdx]);
|
||
const setRangeEnd = useCallback(() => setBulkEnd(frameIdx), [frameIdx]);
|
||
const clearRange = useCallback(() => { setBulkStart(null); setBulkEnd(null); }, []);
|
||
|
||
const bulkRange = useMemo(() => {
|
||
if (bulkStart == null || bulkEnd == null) return null;
|
||
return {
|
||
start: Math.min(bulkStart, bulkEnd),
|
||
end: Math.max(bulkStart, bulkEnd),
|
||
};
|
||
}, [bulkStart, bulkEnd]);
|
||
|
||
const bulkCount = useMemo(() => {
|
||
if (!bulkRange) return 0;
|
||
const skip = Math.max(1, settings?.frame_skip ?? 1);
|
||
return Math.floor((bulkRange.end - bulkRange.start) / skip) + 1;
|
||
}, [bulkRange, settings?.frame_skip]);
|
||
|
||
async function runBulkExtract() {
|
||
if (!outputFolder || !bulkRange) return;
|
||
setPlaying(false);
|
||
setBulkError(null);
|
||
const skip = Math.max(1, settings?.frame_skip ?? 1);
|
||
const total = Math.floor((bulkRange.end - bulkRange.start) / skip) + 1;
|
||
setBulkProgress({ done: 0, total, phase: "extract" });
|
||
const unlisten = await listen<{ done: number; total: number }>(
|
||
"bulk-extract-progress",
|
||
(e) => setBulkProgress({ ...e.payload, phase: "extract" })
|
||
);
|
||
// Only include annotations for frames we'll actually emit; backend will
|
||
// step start..end by `skip` and shapesByFrame is keyed by absolute idx.
|
||
const shapesByFrame: Record<string, CocoShape[]> = {};
|
||
if (coco) {
|
||
for (let idx = bulkRange.start; idx <= bulkRange.end; idx += skip) {
|
||
const key = String(idx);
|
||
const shapes = coco.frames[key];
|
||
if (shapes) shapesByFrame[key] = shapes;
|
||
}
|
||
}
|
||
try {
|
||
await api.bulkExtract(
|
||
bulkRange.start,
|
||
bulkRange.end,
|
||
outputFolder,
|
||
shapesByFrame,
|
||
username,
|
||
skip
|
||
);
|
||
const existing = await api.listExtractedFrames(outputFolder);
|
||
setExtractedFrames(new Set(existing));
|
||
} catch (e) {
|
||
setBulkError(String(e));
|
||
} finally {
|
||
unlisten();
|
||
setBulkProgress(null);
|
||
}
|
||
}
|
||
|
||
async function runBulkUndo() {
|
||
if (!outputFolder || !bulkRange) return;
|
||
setPlaying(false);
|
||
setBulkError(null);
|
||
setBulkProgress({ done: 0, total: bulkRange.end - bulkRange.start + 1, phase: "undo" });
|
||
const unlisten = await listen<{ done: number; total: number }>(
|
||
"bulk-undo-progress",
|
||
(e) => setBulkProgress({ ...e.payload, phase: "undo" })
|
||
);
|
||
try {
|
||
await api.bulkUndo(bulkRange.start, bulkRange.end, outputFolder);
|
||
const existing = await api.listExtractedFrames(outputFolder);
|
||
setExtractedFrames(new Set(existing));
|
||
} catch (e) {
|
||
setBulkError(String(e));
|
||
} finally {
|
||
unlisten();
|
||
setBulkProgress(null);
|
||
}
|
||
}
|
||
|
||
const extractCurrentFrameRef = useRef(extractCurrentFrame);
|
||
const undoCurrentFrameRef = useRef(undoCurrentFrame);
|
||
const setRangeStartRef = useRef(setRangeStart);
|
||
const setRangeEndRef = useRef(setRangeEnd);
|
||
|
||
useEffect(() => {
|
||
extractCurrentFrameRef.current = extractCurrentFrame;
|
||
undoCurrentFrameRef.current = undoCurrentFrame;
|
||
setRangeStartRef.current = setRangeStart;
|
||
setRangeEndRef.current = setRangeEnd;
|
||
}, [extractCurrentFrame, undoCurrentFrame, setRangeStart, setRangeEnd]);
|
||
|
||
// --- keyboard --------------------------------------------------------
|
||
useEffect(() => {
|
||
function onKey(e: KeyboardEvent) {
|
||
const tag = (e.target as HTMLElement | null)?.tagName;
|
||
if (tag === "INPUT" || tag === "TEXTAREA") return;
|
||
const step = e.ctrlKey || e.metaKey ? 10 : 1;
|
||
// Adaptive throttle: drop OS key-repeats when the previous step is
|
||
// still decoding. On fast systems this is a no-op; on slow systems it
|
||
// prevents a held arrow from queueing 30 seeks/sec the pump can't keep
|
||
// up with, which is what causes the "huge gap between keypress and
|
||
// frame" feel.
|
||
const isHeldRepeat =
|
||
e.repeat &&
|
||
(e.key === "ArrowRight" || e.key === "ArrowLeft") &&
|
||
requestedIdx.current !== renderedIdx.current;
|
||
if (isHeldRepeat) { e.preventDefault(); return; }
|
||
switch (e.key) {
|
||
case " ":
|
||
e.preventDefault(); togglePlay(); break;
|
||
case "ArrowRight":
|
||
e.preventDefault(); stepFrames(step); break;
|
||
case "ArrowLeft":
|
||
e.preventDefault(); stepFrames(-step); break;
|
||
case "ArrowUp":
|
||
e.preventDefault(); bumpSpeed(1); break;
|
||
case "ArrowDown":
|
||
e.preventDefault(); bumpSpeed(-1); break;
|
||
case "h": case "H":
|
||
e.preventDefault(); setOverlayVisible((v) => !v); break;
|
||
case "e": case "E":
|
||
e.preventDefault(); extractCurrentFrameRef.current(); break;
|
||
case "u": case "U":
|
||
e.preventDefault(); undoCurrentFrameRef.current(); break;
|
||
case "[":
|
||
e.preventDefault(); setRangeStartRef.current(); break;
|
||
case "]":
|
||
e.preventDefault(); setRangeEndRef.current(); break;
|
||
}
|
||
}
|
||
window.addEventListener("keydown", onKey);
|
||
return () => window.removeEventListener("keydown", onKey);
|
||
}, [togglePlay, stepFrames, bumpSpeed]);
|
||
|
||
// --- video/coco open -------------------------------------------------
|
||
useEffect(() => {
|
||
api.currentVideo().then((v) => {
|
||
if (!v) return;
|
||
setVideo(v);
|
||
// Re-adopt the default folder on mount too, so switching modes and
|
||
// coming back doesn't drop the extracted-frames overlay.
|
||
applyOutputFolder(v.default_output_folder);
|
||
});
|
||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||
}, []);
|
||
|
||
async function chooseVideo() {
|
||
setLoadError(null);
|
||
const selected = await openDialog({
|
||
multiple: false,
|
||
filters: [
|
||
{
|
||
name: "Video",
|
||
extensions: [
|
||
"mp4","MP4","mov","MOV","avi","AVI","mkv","MKV","webm","WEBM","flv","FLV","wmv","WMV",
|
||
],
|
||
},
|
||
{ name: "All files", extensions: ["*"] },
|
||
],
|
||
});
|
||
if (!selected || typeof selected !== "string") return;
|
||
try {
|
||
const info = await api.openVideo(selected);
|
||
clearBitmapCache();
|
||
setVideo(info);
|
||
setFrameIdx(0);
|
||
setPlaying(false);
|
||
setSpeed(1);
|
||
renderedIdx.current = -1;
|
||
decodeErrorRef.current = null;
|
||
setPlaybackError(null);
|
||
// Reset per-video extraction context, then auto-adopt the per-video
|
||
// default folder. If the user already extracted frames into it in a
|
||
// prior session, listExtractedFrames rehydrates them immediately.
|
||
setBulkStart(null);
|
||
setBulkEnd(null);
|
||
await applyOutputFolder(info.default_output_folder);
|
||
} catch (e) {
|
||
setLoadError(String(e));
|
||
}
|
||
}
|
||
|
||
async function chooseCoco() {
|
||
const selected = await openDialog({
|
||
multiple: false,
|
||
filters: [{ name: "Annotations JSON", extensions: ["json"] }, { name: "All files", extensions: ["*"] }],
|
||
});
|
||
if (!selected || typeof selected !== "string") return;
|
||
try {
|
||
const imported = await api.importCoco(selected);
|
||
setCoco(imported);
|
||
// If a folder is already picked, write (or refresh) classes.txt.
|
||
if (outputFolder && imported.categories.length > 0) {
|
||
try { await api.writeClassesTxt(outputFolder, imported.categories); } catch (e) { console.error(e); }
|
||
}
|
||
} catch (e) {
|
||
setLoadError(String(e));
|
||
}
|
||
}
|
||
|
||
function clearCoco() { setCoco(null); }
|
||
|
||
// --- render ----------------------------------------------------------
|
||
|
||
if (!video) {
|
||
return (
|
||
<div className="upload-pane">
|
||
<button className="primary" onClick={chooseVideo}>Upload Video</button>
|
||
{loadError && <p className="error">{loadError}</p>}
|
||
<p className="dim">
|
||
Supported: mp4, mov, avi, mkv, webm, flv, wmv. Decoded via <code>ffmpeg</code>.
|
||
</p>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
const t = frameIdx / video.fps;
|
||
const currentShapes = frameMap.get(frameIdx);
|
||
const isCurrentExtracted = extractedFrames.has(frameIdx);
|
||
const pct = bulkProgress ? Math.round((bulkProgress.done / bulkProgress.total) * 100) : 0;
|
||
|
||
return (
|
||
<div className="extract-mode">
|
||
{!infoCollapsed && (
|
||
<>
|
||
<div className="video-info">
|
||
<span className="path" title={video.path}>{video.path.split("/").pop()}</span>
|
||
<span className="dim">
|
||
{video.width}×{video.height} · {video.fps.toFixed(2)} fps · {video.total_frames} frames · {video.duration_s.toFixed(1)}s
|
||
</span>
|
||
<button className="link" onClick={chooseVideo}>change…</button>
|
||
</div>
|
||
|
||
<div className="coco-info">
|
||
{coco ? (
|
||
<>
|
||
<span>
|
||
<span className="dim">{coco.stats.source_format === "sam" ? "SAM" : "COCO"}:</span>{" "}
|
||
<strong>{coco.stats.total_annotations}</strong> annotations ·{" "}
|
||
<strong>{coco.stats.frame_count}</strong> frames ·{" "}
|
||
<strong>{coco.stats.categories}</strong> categories
|
||
{coco.stats.skipped_rle > 0 && <span className="dim"> · {coco.stats.skipped_rle} RLE skipped</span>}
|
||
{coco.stats.skipped_no_image > 0 && <span className="dim"> · {coco.stats.skipped_no_image} unmappable</span>}
|
||
</span>
|
||
<button className="link" onClick={chooseCoco}>replace…</button>
|
||
<button className="link" onClick={clearCoco}>clear</button>
|
||
</>
|
||
) : (
|
||
<>
|
||
<span className="dim">no annotations loaded</span>
|
||
<button className="link" onClick={chooseCoco}>import annotations…</button>
|
||
</>
|
||
)}
|
||
</div>
|
||
</>
|
||
)}
|
||
|
||
<div className={`extract-info${infoCollapsed ? " collapsed" : ""}`}>
|
||
{outputFolder ? (
|
||
<>
|
||
<span>Output: <strong title={outputFolder}>{outputFolder.split("/").pop()}</strong></span>
|
||
<span className="dim"> · {extractedFrames.size} extracted</span>
|
||
<button className="link" onClick={pickOutputFolder}>change…</button>
|
||
</>
|
||
) : (
|
||
<button className="link" onClick={pickOutputFolder}>choose output folder…</button>
|
||
)}
|
||
<div className="extract-actions">
|
||
<button
|
||
className={isCurrentExtracted ? "extracted" : "primary-ghost"}
|
||
onClick={extractCurrentFrame}
|
||
disabled={!outputFolder}
|
||
title="E"
|
||
>
|
||
{isCurrentExtracted ? "✓ re-extract (E)" : "Extract (E)"}
|
||
</button>
|
||
<button
|
||
onClick={undoCurrentFrame}
|
||
disabled={!outputFolder || !isCurrentExtracted}
|
||
title="U"
|
||
>
|
||
Undo (U)
|
||
</button>
|
||
<div className="range-picker">
|
||
<button onClick={setRangeStart} title="[">
|
||
Start [ {bulkStart ?? "–"}
|
||
</button>
|
||
<button onClick={setRangeEnd} title="]">
|
||
End ] {bulkEnd ?? "–"}
|
||
</button>
|
||
{(bulkStart != null || bulkEnd != null) && (
|
||
<button className="link" onClick={clearRange}>clear</button>
|
||
)}
|
||
</div>
|
||
<button
|
||
onClick={runBulkExtract}
|
||
disabled={!outputFolder || !bulkRange || !!bulkProgress}
|
||
>
|
||
Bulk Extract{bulkRange && ` (${bulkCount})`}
|
||
</button>
|
||
<button
|
||
onClick={runBulkUndo}
|
||
disabled={!outputFolder || !bulkRange || !!bulkProgress}
|
||
>
|
||
Bulk Undo
|
||
</button>
|
||
<button
|
||
className="collapse-toggle"
|
||
onClick={() => setInfoCollapsed((v) => !v)}
|
||
title={infoCollapsed ? "show info" : "hide info"}
|
||
>
|
||
{infoCollapsed ? "v" : "^"}
|
||
</button>
|
||
</div>
|
||
{bulkProgress && (
|
||
<div className="progress">
|
||
<div className="progress-bar" style={{ width: `${pct}%` }} />
|
||
<span>
|
||
{bulkProgress.phase === "extract" ? "extracting" : "undoing"} ·{" "}
|
||
{bulkProgress.done} / {bulkProgress.total}
|
||
</span>
|
||
</div>
|
||
)}
|
||
{bulkError && <p className="error">{bulkError}</p>}
|
||
</div>
|
||
|
||
<FrameCanvas ref={canvasRef} video={video} extracted={isCurrentExtracted} />
|
||
|
||
<div className="timeline">
|
||
<div className="timeline-track-wrap">
|
||
<input
|
||
type="range"
|
||
min={0}
|
||
max={video.total_frames - 1}
|
||
value={frameIdx}
|
||
onChange={(e) => seekToFrame(Number(e.target.value))}
|
||
style={
|
||
{
|
||
"--val": `${(frameIdx / Math.max(1, video.total_frames - 1)) * 100}%`,
|
||
} as React.CSSProperties
|
||
}
|
||
/>
|
||
{extractedFrames.size > 0 && (
|
||
<ExtractedTicks
|
||
extracted={extractedFrames}
|
||
totalFrames={video.total_frames}
|
||
/>
|
||
)}
|
||
</div>
|
||
<div className="timeline-readout">
|
||
frame <strong>{frameIdx}</strong> / {video.total_frames - 1}
|
||
<span className="dim"> · {formatTime(t)}</span>
|
||
{currentShapes && currentShapes.length > 0 && (
|
||
<span className="badge">
|
||
{currentShapes.length} annotation{currentShapes.length === 1 ? "" : "s"}
|
||
</span>
|
||
)}
|
||
{isCurrentExtracted && <span className="badge extracted">✓ extracted</span>}
|
||
</div>
|
||
</div>
|
||
|
||
<div className="controls">
|
||
<button className={playing ? "active" : ""} onClick={togglePlay}>
|
||
{playing ? "❚❚ pause" : "▶ play"}
|
||
</button>
|
||
<button onClick={() => stepFrames(-1)} title="←">◀ 1</button>
|
||
<button onClick={() => stepFrames(1)} title="→">1 ▶</button>
|
||
<button onClick={() => stepFrames(-10)} title="Ctrl+←">◀◀ 10</button>
|
||
<button onClick={() => stepFrames(10)} title="Ctrl+→">10 ▶▶</button>
|
||
<div className="speed">
|
||
<button onClick={() => bumpSpeed(-1)} title="↓">–</button>
|
||
<span className="speed-readout">{speed}×</span>
|
||
<button onClick={() => bumpSpeed(1)} title="↑">+</button>
|
||
</div>
|
||
<label className={`review-toggle ${reviewMode ? "on" : ""}`}>
|
||
<input
|
||
type="checkbox"
|
||
checked={reviewMode}
|
||
onChange={(e) => setReviewMode(e.target.checked)}
|
||
/>
|
||
Review mode
|
||
</label>
|
||
<button
|
||
className={`overlay-toggle ${overlayVisible ? "on" : ""}`}
|
||
onClick={() => setOverlayVisible((v) => !v)}
|
||
title="H"
|
||
>
|
||
{overlayVisible ? "👁 overlay on" : "◌ overlay off"}
|
||
</button>
|
||
<span className="dim shortcuts">space · ← → · ctrl+← → · ↑ ↓ · H · E · U · [ ]</span>
|
||
</div>
|
||
|
||
{playbackError && (
|
||
<p className="error" role="alert">
|
||
Playback stopped: {playbackError}
|
||
{" "}
|
||
<button
|
||
className="link"
|
||
onClick={() => {
|
||
decodeErrorRef.current = null;
|
||
setPlaybackError(null);
|
||
}}
|
||
>
|
||
dismiss
|
||
</button>
|
||
</p>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
function formatTime(s: number) {
|
||
const m = Math.floor(s / 60);
|
||
const r = s - m * 60;
|
||
return `${m}:${r.toFixed(2).padStart(5, "0")}`;
|
||
}
|
||
|
||
/**
|
||
* Canvas overlay that draws one green tick per pixel-column of the timeline
|
||
* track where at least one extracted frame falls. Canvas avoids the DOM
|
||
* blow-up of 1-div-per-frame on long videos, and collapses dense neighboring
|
||
* frames into a single visible stripe at narrow timeline widths.
|
||
*/
|
||
function ExtractedTicks({
|
||
extracted,
|
||
totalFrames,
|
||
}: {
|
||
extracted: Set<number>;
|
||
totalFrames: number;
|
||
}) {
|
||
const ref = useRef<HTMLCanvasElement>(null);
|
||
useEffect(() => {
|
||
const canvas = ref.current;
|
||
if (!canvas) return;
|
||
let raf = 0;
|
||
|
||
const draw = () => {
|
||
raf = 0;
|
||
const rect = canvas.getBoundingClientRect();
|
||
const dpr = window.devicePixelRatio || 1;
|
||
const w = Math.max(1, Math.floor(rect.width * dpr));
|
||
const h = Math.max(1, Math.floor(rect.height * dpr));
|
||
if (canvas.width !== w) canvas.width = w;
|
||
if (canvas.height !== h) canvas.height = h;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
ctx.clearRect(0, 0, w, h);
|
||
if (extracted.size === 0 || totalFrames <= 0) return;
|
||
ctx.fillStyle = "rgba(92, 207, 117, 0.85)";
|
||
const max = Math.max(1, totalFrames - 1);
|
||
for (const f of extracted) {
|
||
const x = Math.round((f / max) * (w - 1));
|
||
ctx.fillRect(x, 0, Math.max(1, Math.floor(dpr)), h);
|
||
}
|
||
};
|
||
|
||
const schedule = () => {
|
||
if (raf) return;
|
||
raf = requestAnimationFrame(draw);
|
||
};
|
||
schedule();
|
||
const ro = new ResizeObserver(schedule);
|
||
ro.observe(canvas);
|
||
return () => {
|
||
if (raf) cancelAnimationFrame(raf);
|
||
ro.disconnect();
|
||
};
|
||
}, [extracted, totalFrames]);
|
||
return <canvas ref={ref} className="timeline-ticks" aria-hidden="true" />;
|
||
}
|