Ravi/SR-Label-SamTool
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

playback issues

Browse Source
This commit is contained in:
ravi
2026-06-04 17:05:58 +05:30
parent 436afcae4c
commit 85b090a8b9
7 changed files with 524 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
sam-tool-tauri/src/ipc.ts
View File

@@ -1,4 +1,4 @@
import { invoke } from "@tauri-apps/api/core";
import { invoke, Channel } from "@tauri-apps/api/core";
import type {
CocoCategory,
CocoShape,
@@ -70,6 +70,25 @@ export const api = {
readFolderSource: (folder: string) =>
invoke<string | null>("read_folder_source", { folder }),
// Forward playback uses a Rust producer thread that pushes packed-frame
// ArrayBuffers via this channel. Pass `new Channel<ArrayBuffer>()` from the
// caller and attach `.onmessage` before invoking — Rust starts sending
// immediately.
startStream: (
channel: Channel<ArrayBuffer>,
fromIdx: number,
speed: number,
stride: number
) => invoke<number>("start_stream", { channel, fromIdx, speed, stride }),
setStreamState: (state: {
playing?: boolean;
speed?: number;
targetFrame?: number;
stride?: number;
}) => invoke<void>("set_stream_state", state),
stopStream: () => invoke<void>("stop_stream"),
streamCurrentFrame: () => invoke<number>("stream_current_frame"),
getSettings: () => invoke<Settings>("get_settings"),
saveSettings: (settings: Settings) =>
invoke<void>("save_settings", { settings }),

249
sam-tool-tauri/src/modes/ExtractMode.tsx
View File

@@ -1,10 +1,12 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { open as openDialog } from "@tauri-apps/plugin-dialog";
import { listen } from "@tauri-apps/api/event";
import { Channel } from "@tauri-apps/api/core";
import { api } from "../ipc";
import type { CocoShape, ImportedCoco, Settings, VideoInfo } from "../types";
import { FrameCanvas } from "../components/extract/FrameCanvas";
import { drawOverlay } from "../components/extract/overlay";
import { sharedFrameDecoderPool } from "../services/frameDecoderPool";
const SPEEDS = [0.25, 0.5, 1, 1.5, 2, 3, 4];
const FRAME_BITMAP_CACHE_LIMIT = 24;
@@ -186,34 +188,44 @@ export function ExtractMode({ username, settings }: Props) {
);
// Returns true if we drew, false if we were preempted by a newer request,
// and throws if bitmap creation failed — pump uses this to decide whether
// to mark the frame rendered or surface the error to the playback loop.
// and throws if worker decode failed. The packed buffer carries an 8-byte
// LE frame-number prefix; the worker is authoritative on which frame
// actually decoded, so we use that for caching + drawing rather than the
// optimistic `atFrameIdx` (`atFrameIdx` only stays useful as a fallback
// for the preempt check).
const renderFrame = useCallback(
async (bytes: ArrayBuffer, atFrameIdx: number): Promise<boolean> => {
// Diagnostic timing log: set `window.SAM_FRAME_DEBUG = true` in devtools
// on a slow machine to see per-frame bitmap and draw timings — confirms
// where the bottleneck lives (decode vs createImageBitmap vs drawImage).
const debug = (window as unknown as { SAM_FRAME_DEBUG?: boolean })
.SAM_FRAME_DEBUG === true;
const t0 = debug ? performance.now() : 0;
const blob = new Blob([bytes], { type: "image/jpeg" });
let frameNumber: number;
let bitmap: ImageBitmap;
try {
bitmap = await createImageBitmap(blob);
const result = await sharedFrameDecoderPool.decode(bytes);
frameNumber = result.frameNumber;
bitmap = result.bitmap;
} catch (e) {
throw new Error(
`createImageBitmap failed: ${e instanceof Error ? e.message : String(e)}`
`worker decode failed: ${e instanceof Error ? e.message : String(e)}`
);
}
const t1 = debug ? performance.now() : 0;
rememberBitmap(atFrameIdx, bitmap);
if (requestedIdx.current !== atFrameIdx) return false;
drawBitmap(bitmap, atFrameIdx);
rememberBitmap(frameNumber, bitmap);
// Preempt check: if a newer request landed during decode, drop this
// paint. Match against either the prefix frame number (authoritative)
// or the original request — keyframe snap can make them disagree.
if (
requestedIdx.current !== frameNumber &&
requestedIdx.current !== atFrameIdx
) {
return false;
}
drawBitmap(bitmap, frameNumber);
if (debug) {
const t2 = performance.now();
// eslint-disable-next-line no-console
console.debug(
`[SAM] f${atFrameIdx} bytes=${bytes.byteLength} bitmap=${(t1 - t0).toFixed(0)}ms draw=${(t2 - t1).toFixed(0)}ms`
`[SAM] f${frameNumber} bytes=${bytes.byteLength} decode=${(t1 - t0).toFixed(0)}ms draw=${(t2 - t1).toFixed(0)}ms`
);
}
return true;
@@ -329,109 +341,126 @@ export function ExtractMode({ username, settings }: Props) {
return () => ro.disconnect();
}, [video, drawBitmap, getCachedBitmap, pump]);
// --- playback loop ---------------------------------------------------
// Render-paced: advance ONLY after the previous frame actually drew. On a
// fast box this hits target fps; on a slow box it plays smoothly at the
// achievable decode/render rate instead of dropping 9-of-10 frames at the
// canvas while the wall-clock UI counter races ahead.
// --- playback (Channel-push transport) -----------------------------------
// Forward playback runs through a Rust producer thread that pushes packed
// frames over a Tauri Channel — eliminates the per-frame IPC round-trip
// that was the main stutter source. Backward / step / scrub / pause all
// stay on the invoke-pull path (pump() above). Three guards live here:
// * `paused` — set true on review-mode stop so any in-flight messages
// queued behind us don't paint;
// * `lastPainted` — direction-aware out-of-order drop in case the worker
// pool returns N+1 before N;
// * cleanup stops the stream and re-fetches the last painted frame via
// invoke so the canvas exactly matches `frameIdx` on pause.
useEffect(() => {
if (!playing || !video) return;
let cancelled = false;
let nextDueAt = performance.now();
let paused = false;
let lastPainted = frameIdxRef.current;
const skip = Math.max(1, frameSkipRef.current | 0);
const startFrom = Math.min(
video.total_frames - 1,
frameIdxRef.current + skip
);
if (startFrom <= frameIdxRef.current) {
setPlaying(false);
return;
}
const waitForFrame = (target: number) =>
new Promise<void>((resolve) => {
const check = () => {
if (
cancelled ||
renderedIdx.current === target ||
decodeErrorRef.current !== null
) {
resolve();
return;
}
requestAnimationFrame(check);
};
check();
});
const loop = async () => {
while (!cancelled && playing) {
const skip = Math.max(1, frameSkipRef.current | 0);
const current = frameIdxRef.current;
let target = Math.min(video.total_frames - 1, current + skip);
if (target <= current) {
setPlaying(false);
return;
}
// Review-mode scan: with skip > 1 a naive +skip jump can leap past
// annotations between current+1 and current+skip. Walk that window
// and clamp `target` down to the first annotated frame.
if (reviewMode) {
for (let i = current + 1; i <= target; i++) {
if (annotatedFrames.current.has(i)) {
target = i;
break;
}
}
}
requestedIdx.current = target;
// Fast path: await pump directly. When pump owns the queue it
// resolves the instant target is drawn (no 16ms rAF latency tax).
// If another caller (ResizeObserver / overlay-change effect) was
// already running pump, our call short-circuits — fall back to
// rAF polling in that case so we still wait correctly.
await pump();
if (
!cancelled &&
renderedIdx.current !== target &&
decodeErrorRef.current === null
) {
await waitForFrame(target);
}
const channel = new Channel<ArrayBuffer>();
channel.onmessage = async (buffer: ArrayBuffer) => {
if (cancelled || paused) return;
// Empty-frame sentinel from the producer = decode failed.
if (buffer.byteLength <= 8) {
setPlaybackError("decode failed during playback");
setPlaying(false);
return;
}
let frameNumber: number;
let bitmap: ImageBitmap;
try {
const result = await sharedFrameDecoderPool.decode(buffer);
frameNumber = result.frameNumber;
bitmap = result.bitmap;
} catch (e) {
if (cancelled) return;
// Decode failed mid-play: stop and surface so the UI doesn't claim to
// be playing while the canvas is frozen on the last good frame.
if (decodeErrorRef.current) {
setPlaybackError(decodeErrorRef.current);
setPlaying(false);
return;
}
const advanced = Math.max(1, target - current);
// Real-time pacing regardless of skip: skip controls density,
// `speed` controls speed. Without the advanced-frame factor here,
// skip=3 would play at 3x wall-clock, which is what the speed slider
// is for. Compute after review-mode clamping so annotated stops do not
// wait for the full skip window.
const interval = (advanced * 1000) / (video.fps * speed);
// Keep the fast playback loop's cursor in sync immediately. Waiting
// for React's frameIdx effect can make a slow decode path replay the
// same target once and add an extra frame interval per rendered frame.
frameIdxRef.current = target;
setFrameIdx(target);
// Pause AFTER rendering so the user actually sees the annotated frame.
if (reviewMode && annotatedFrames.current.has(target)) {
setPlaying(false);
return;
}
setPlaybackError(e instanceof Error ? e.message : String(e));
setPlaying(false);
return;
}
if (cancelled || paused) {
bitmap.close();
return;
}
// Direction-aware out-of-order drop: forward play is monotonic, so
// any frame number < the last painted came from a parallel worker
// and is stale. Cheap defence against worker-pool races.
if (frameNumber < lastPainted) {
bitmap.close();
return;
}
lastPainted = frameNumber;
rememberBitmap(frameNumber, bitmap);
drawBitmap(bitmap, frameNumber);
renderedIdx.current = frameNumber;
requestedIdx.current = frameNumber;
frameIdxRef.current = frameNumber;
setFrameIdx(frameNumber);
decodeErrorRef.current = null;
const now = performance.now();
const due = nextDueAt + interval;
if (due > now) {
await new Promise((r) => setTimeout(r, due - now));
nextDueAt = due;
} else {
// Behind real-time — don't accumulate debt and then "catch up" by
// skipping frames; just reset the clock to now.
nextDueAt = now;
}
// Review-mode stops AT the annotated frame, so the canvas already
// shows it by the time we ask the producer to halt.
if (reviewMode && annotatedFrames.current.has(frameNumber)) {
paused = true;
try { await api.stopStream(); } catch { /* ignore */ }
setPlaying(false);
}
};
loop();
api
.startStream(channel, startFrom, speed, skip)
.catch((e) => {
if (cancelled) return;
setPlaybackError(`start_stream failed: ${e}`);
setPlaying(false);
});
return () => {
cancelled = true;
// Fire-and-forget: stop the producer, then re-fetch the last painted
// frame via the authoritative invoke path so the canvas matches
// `frameIdx` exactly (the producer may have been mid-flight).
const lastFrame = frameIdxRef.current;
api
.stopStream()
.catch(() => { /* ignore */ })
.then(() => api.decodeFrame(lastFrame))
.then((buf) => sharedFrameDecoderPool.decode(buf))
.then(({ frameNumber, bitmap }) => {
// Newer request may have started by the time this resolves —
// only paint if it still matches.
if (requestedIdx.current !== frameNumber) {
bitmap.close();
return;
}
rememberBitmap(frameNumber, bitmap);
drawBitmap(bitmap, frameNumber);
renderedIdx.current = frameNumber;
})
.catch(() => { /* invoke path will recover on next seek */ });
};
}, [playing, video, speed, reviewMode, pump]);
}, [playing, video, speed, reviewMode, drawBitmap, rememberBitmap]);
// Live-update the producer when speed or frame-skip changes mid-play, so
// the user doesn't have to pause + resume to see the new setting.
useEffect(() => {
if (!playing) return;
api.setStreamState({ speed }).catch(() => { /* ignore */ });
}, [playing, speed]);
useEffect(() => {
if (!playing) return;
api.setStreamState({ stride: Math.max(1, settings?.frame_skip ?? 1) }).catch(() => { /* ignore */ });
}, [playing, settings?.frame_skip]);
// --- navigation helpers ----------------------------------------------
const seekToFrame = useCallback((idx: number) => {
@@ -443,11 +472,19 @@ export function ExtractMode({ username, settings }: Props) {
if (cached) {
drawBitmap(cached, clamped);
renderedIdx.current = clamped;
} else {
} else if (!playing) {
// Pump only handles seeks while paused. During Channel-push play the
// producer is the source of frames; we re-target it instead.
pump();
}
if (playing) {
// Hand the new target to the running producer so it resumes from the
// scrub destination instead of where it was. The producer's monotonic
// session_id keeps the existing channel valid.
api.setStreamState({ targetFrame: clamped }).catch(() => { /* ignore */ });
}
setFrameIdx((prev) => (prev === clamped ? prev : clamped));
}, [drawBitmap, getCachedBitmap, pump, video]);
}, [drawBitmap, getCachedBitmap, pump, video, playing]);
const stepFrames = useCallback((delta: number) => {
if (!video) return;

118
sam-tool-tauri/src/services/frameDecoderPool.ts Normal file
View File

@@ -0,0 +1,118 @@
// Worker pool that takes a packed-frame ArrayBuffer (8-byte LE i64 frame
// number + JPEG bytes), decodes the JPEG to an ImageBitmap off the main
// thread, and returns both via transferable postMessage so the main thread
// only does the final drawImage. Inlined as a Blob URL so we don't need a
// separate build entry for the worker file.
const WORKER_SOURCE = `
self.onmessage = async (e) => {
const { id, buffer } = e.data;
let frameNumber = -1;
try {
const headerView = new BigInt64Array(buffer, 0, 1);
frameNumber = Number(headerView[0]);
const jpegView = new Uint8Array(buffer, 8);
// Copy out of the transferred buffer so Blob ownership is independent.
const jpegCopy = jpegView.slice();
const blob = new Blob([jpegCopy], { type: "image/jpeg" });
const bitmap = await createImageBitmap(blob);
self.postMessage({ id, frameNumber, bitmap }, [bitmap]);
} catch (err) {
const msg = err && err.message ? err.message : String(err);
self.postMessage({ id, frameNumber, error: msg });
}
};
`;
const workerBlob = new Blob([WORKER_SOURCE], {
type: "application/javascript",
});
const workerUrl = URL.createObjectURL(workerBlob);
export interface DecodeResult {
frameNumber: number;
bitmap: ImageBitmap;
}
interface Pending {
resolve: (r: DecodeResult) => void;
reject: (e: Error) => void;
}
interface WorkerMessage {
id: number;
frameNumber: number;
bitmap?: ImageBitmap;
error?: string;
}
export class FrameDecoderPool {
private workers: Worker[] = [];
private next = 0;
private pending = new Map<number, Pending>();
private nextId = 1;
constructor(size: number = 2) {
const count = Math.max(1, Math.min(size, 8));
for (let i = 0; i < count; i++) {
const worker = new Worker(workerUrl);
worker.onmessage = (e: MessageEvent<WorkerMessage>) => {
const { id, frameNumber, bitmap, error } = e.data;
const slot = this.pending.get(id);
if (!slot) {
// The caller discarded this request — close the bitmap so it
// doesn't leak (an ImageBitmap holds GPU memory).
bitmap?.close?.();
return;
}
this.pending.delete(id);
if (error || !bitmap) {
slot.reject(new Error(error ?? "decode failed"));
} else {
slot.resolve({ frameNumber, bitmap });
}
};
worker.onerror = (e) => {
// Surface worker-thread errors to any pending request on this
// worker. We don't know which id failed, so reject the oldest.
const first = this.pending.entries().next().value;
if (first) {
const [id, slot] = first;
this.pending.delete(id);
slot.reject(new Error(`worker error: ${e.message}`));
}
};
this.workers.push(worker);
}
}
/**
* Decode a packed-frame buffer. The buffer ownership transfers to the
* worker — do not read it after this call returns.
*/
decode(buffer: ArrayBuffer): Promise<DecodeResult> {
return new Promise((resolve, reject) => {
const id = this.nextId++;
this.pending.set(id, { resolve, reject });
const worker = this.workers[this.next];
this.next = (this.next + 1) % this.workers.length;
worker.postMessage({ id, buffer }, [buffer]);
});
}
destroy() {
for (const w of this.workers) w.terminate();
this.workers = [];
for (const slot of this.pending.values()) {
slot.reject(new Error("decoder pool destroyed"));
}
this.pending.clear();
}
}
/**
* Shared pool used by ExtractMode (and later, AnnotateMode). Two workers is
* enough for sequential playback decode; raising it helps the seek path
* when several scrub requests overlap.
*/
export const sharedFrameDecoderPool = new FrameDecoderPool(2);