video annotations

This commit is contained in:
2026-07-26 22:01:34 +05:30
parent 4166f202b0
commit 9685d16794
4 changed files with 384 additions and 6 deletions

View File

@@ -101,6 +101,14 @@ export const activityFeedsService = {
return response;
},
// Save the record's edited bounding-box annotations (dbName is auto-attached by
// the axios interceptor). Instant JSONB write; baked into the archived media
// at confirm time.
saveAnnotations: async (payload: { id: number; table: 'audits' | 'rectification'; annotations: any }) => {
const response = await axiosClient.post('/api/audit/auditor/anomaly/annotations', payload);
return response;
},
getOrganizationSites: async (org_id: string) => {
const response = await axiosClient.get('/api/audit/Master/site', {
params: { org_id }

View File

@@ -0,0 +1,261 @@
import React, { useRef, useEffect, useState, useCallback } from 'react';
// Bounding-box overlay for the audit-session ANOMALY IMAGE (Frame_Test), shown
// when the global Annotation Mode is on.
// - VIEW (default): the image zooms/pans (scroll wheel + drag, like the normal
// viewer) and the boxes are drawn on a canvas that shares the SAME transform,
// so they stay aligned while you inspect. No accidental drawing.
// - EDIT (click "Edit"): fit scale, the canvas captures the pointer, and you can
// draw (drag), select (click), and delete boxes. Edits are held locally and
// only persisted on "Save changes" (one API call).
//
// Format: { "<frame>": { "<Label>": [ ["<trackId>", [x1,y1], [x2,y2] ], ... ] } }
type Box = [string, [number, number], [number, number]];
export type Annotations = { [frame: string]: { [label: string]: Box[] } };
interface Props {
imageUrl: string;
annotations: Annotations | null | undefined;
onSave?: (a: Annotations) => void;
}
const MINZ = 1, MAXZ = 6, WHEEL_STEP = 0.3, BTN_STEP = 0.5;
const colorFor = (label: string) => {
let h = 0;
for (let i = 0; i < label.length; i++) h = (h * 31 + label.charCodeAt(i)) % 360;
return `hsl(${h}, 90%, 58%)`;
};
type Kind = 'primary' | 'danger' | 'ghost';
const btn = (kind: Kind, disabled = false): React.CSSProperties => ({
fontSize: 11, padding: '4px 10px', borderRadius: 4, minWidth: 28,
border: '1px solid ' + (kind === 'danger' ? '#ef4444' : '#475569'),
background: disabled ? 'rgba(30,41,59,0.6)' : kind === 'primary' ? '#2563eb' : 'rgba(15,23,42,0.9)',
color: disabled ? '#64748b' : kind === 'danger' ? '#fca5a5' : '#fff',
cursor: disabled ? 'default' : 'pointer', fontWeight: kind === 'primary' ? 700 : 400,
});
export const AnnotationOverlay: React.FC<Props> = ({ imageUrl, annotations, onSave }) => {
const wrapRef = useRef<HTMLDivElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const flashTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const [nat, setNat] = useState<{ w: number; h: number } | null>(null);
const [local, setLocal] = useState<Annotations>(annotations || {});
const [dirty, setDirty] = useState(false);
const [flash, setFlash] = useState(false);
const [edit, setEdit] = useState(false);
const [zoom, setZoom] = useState(1);
const [origin, setOrigin] = useState({ x: 50, y: 50 });
const [drag, setDrag] = useState<{ x0: number; y0: number; x1: number; y1: number } | null>(null);
const [selected, setSelected] = useState<{ frame: string; label: string; idx: number } | null>(null);
useEffect(() => { setLocal(annotations || {}); setSelected(null); setDirty(false); }, [annotations, imageUrl]);
useEffect(() => { setZoom(1); setOrigin({ x: 50, y: 50 }); setEdit(false); }, [imageUrl]);
// wheel zoom — VIEW mode only (native non-passive so we can preventDefault)
useEffect(() => {
const el = wrapRef.current;
if (!el) return;
const onWheel = (e: WheelEvent) => {
if (edit) return;
e.preventDefault();
const rect = el.getBoundingClientRect();
setOrigin({ x: ((e.clientX - rect.left) / rect.width) * 100, y: ((e.clientY - rect.top) / rect.height) * 100 });
setZoom((z) => Math.min(MAXZ, Math.max(MINZ, +(z + (e.deltaY < 0 ? WHEEL_STEP : -WHEEL_STEP)).toFixed(2))));
};
el.addEventListener('wheel', onWheel, { passive: false });
return () => el.removeEventListener('wheel', onWheel);
}, [edit]);
const getFit = useCallback(() => {
const wrap = wrapRef.current;
if (!wrap || !nat) return null;
const cw = wrap.clientWidth, ch = wrap.clientHeight;
const scale = Math.min(cw / nat.w, ch / nat.h);
return { ox: (cw - nat.w * scale) / 2, oy: (ch - nat.h * scale) / 2, scale, cw, ch };
}, [nat]);
const draw = useCallback(() => {
const cv = canvasRef.current, fit = getFit();
if (!cv || !fit) return;
if (cv.width !== fit.cw) cv.width = fit.cw;
if (cv.height !== fit.ch) cv.height = fit.ch;
const ctx = cv.getContext('2d');
if (!ctx) return;
ctx.clearRect(0, 0, cv.width, cv.height);
for (const frame of Object.keys(local)) {
for (const label of Object.keys(local[frame])) {
const col = colorFor(label);
local[frame][label].forEach((box, idx) => {
const [, [x1, y1], [x2, y2]] = box;
const X = fit.ox + x1 * fit.scale, Y = fit.oy + y1 * fit.scale;
const W = (x2 - x1) * fit.scale, H = (y2 - y1) * fit.scale;
const sel = selected && selected.frame === frame && selected.label === label && selected.idx === idx;
ctx.strokeStyle = col;
ctx.lineWidth = sel ? 3 : 2;
ctx.strokeRect(X, Y, W, H);
ctx.fillStyle = col;
ctx.font = '12px sans-serif';
const tw = ctx.measureText(label).width;
ctx.fillRect(X, Math.max(0, Y - 15), tw + 6, 15);
ctx.fillStyle = '#0b1220';
ctx.fillText(label, X + 3, Math.max(11, Y - 4));
});
}
}
if (drag) {
ctx.strokeStyle = '#ffffff';
ctx.setLineDash([5, 4]);
ctx.strokeRect(drag.x0, drag.y0, drag.x1 - drag.x0, drag.y1 - drag.y0);
ctx.setLineDash([]);
}
}, [getFit, local, selected, drag]);
useEffect(() => { draw(); }, [draw]);
useEffect(() => {
const onResize = () => draw();
window.addEventListener('resize', onResize);
return () => window.removeEventListener('resize', onResize);
}, [draw]);
const point = (e: React.MouseEvent) => {
const r = canvasRef.current!.getBoundingClientRect();
return { x: e.clientX - r.left, y: e.clientY - r.top };
};
const toNatural = (cx: number, cy: number) => {
const fit = getFit();
if (!fit) return [0, 0];
return [(cx - fit.ox) / fit.scale, (cy - fit.oy) / fit.scale];
};
const change = (next: Annotations) => { setLocal(next); setDirty(true); };
const save = () => {
onSave && onSave(local);
setDirty(false);
setFlash(true);
if (flashTimer.current) clearTimeout(flashTimer.current);
flashTimer.current = setTimeout(() => setFlash(false), 1600);
};
const deleteSelected = () => {
if (!selected) return;
const next: Annotations = JSON.parse(JSON.stringify(local));
next[selected.frame]?.[selected.label]?.splice(selected.idx, 1);
if (next[selected.frame]?.[selected.label]?.length === 0) delete next[selected.frame][selected.label];
if (next[selected.frame] && Object.keys(next[selected.frame]).length === 0) delete next[selected.frame];
setSelected(null);
change(next);
};
// VIEW-mode pan (mouse move while zoomed) — mirrors ZoomableImage
const onWrapMove = (e: React.MouseEvent) => {
if (edit || zoom === 1) return;
const el = wrapRef.current;
if (!el) return;
const rect = el.getBoundingClientRect();
setOrigin({ x: ((e.clientX - rect.left) / rect.width) * 100, y: ((e.clientY - rect.top) / rect.height) * 100 });
};
// EDIT-mode draw/select (canvas is at fit scale, so coords map directly)
const onMouseDown = (e: React.MouseEvent) => {
const { x, y } = point(e);
const fit = getFit();
if (fit) {
for (const frame of Object.keys(local)) {
for (const label of Object.keys(local[frame])) {
const boxes = local[frame][label];
for (let idx = 0; idx < boxes.length; idx++) {
const [, [x1, y1], [x2, y2]] = boxes[idx];
const X = fit.ox + x1 * fit.scale, Y = fit.oy + y1 * fit.scale;
const W = (x2 - x1) * fit.scale, H = (y2 - y1) * fit.scale;
const PAD = 8;
if (x >= X - PAD && x <= X + W + PAD && y >= Y - PAD && y <= Y + H + PAD) { setSelected({ frame, label, idx }); return; }
}
}
}
}
setSelected(null);
setDrag({ x0: x, y0: y, x1: x, y1: y });
};
const onMouseMove = (e: React.MouseEvent) => {
if (!drag) return;
const { x, y } = point(e);
setDrag({ ...drag, x1: x, y1: y });
};
const onMouseUp = () => {
if (!drag) return;
const d = drag; setDrag(null);
if (Math.abs(d.x1 - d.x0) < 6 || Math.abs(d.y1 - d.y0) < 6) return;
const [nx1, ny1] = toNatural(Math.min(d.x0, d.x1), Math.min(d.y0, d.y1));
const [nx2, ny2] = toNatural(Math.max(d.x0, d.x1), Math.max(d.y0, d.y1));
const label = window.prompt('Label for this box:', 'anomaly');
if (!label) return;
const frame = Object.keys(local)[0] || '0';
const next: Annotations = JSON.parse(JSON.stringify(local));
next[frame] = next[frame] || {};
next[frame][label] = next[frame][label] || [];
next[frame][label].push([`e${Date.now()}`, [Math.round(nx1), Math.round(ny1)], [Math.round(nx2), Math.round(ny2)]]);
change(next);
};
useEffect(() => {
const onKey = (e: KeyboardEvent) => {
if (!edit || !selected || (e.key !== 'Delete' && e.key !== 'Backspace')) return;
e.preventDefault();
deleteSelected();
};
window.addEventListener('keydown', onKey);
return () => window.removeEventListener('keydown', onKey);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [edit, selected, local]);
const enterEdit = () => { setZoom(1); setOrigin({ x: 50, y: 50 }); setEdit(true); };
const xf = { transform: `scale(${zoom})`, transformOrigin: `${origin.x}% ${origin.y}%`, transition: 'transform 0.08s ease-out' } as React.CSSProperties;
return (
<div
ref={wrapRef}
onMouseMove={onWrapMove}
style={{ position: 'relative', width: '100%', height: '100%', overflow: 'hidden', cursor: edit ? 'crosshair' : zoom > 1 ? 'move' : 'zoom-in' }}
>
<img
src={imageUrl}
alt="Anomaly"
draggable={false}
onLoad={(e) => setNat({ w: e.currentTarget.naturalWidth, h: e.currentTarget.naturalHeight })}
style={{ width: '100%', height: '100%', objectFit: 'contain', display: 'block', ...xf }}
/>
<canvas
ref={canvasRef}
onMouseDown={edit ? onMouseDown : undefined}
onMouseMove={edit ? onMouseMove : undefined}
onMouseUp={edit ? onMouseUp : undefined}
style={{ position: 'absolute', top: 0, left: 0, width: '100%', height: '100%', pointerEvents: edit ? 'auto' : 'none', ...xf }}
/>
<div style={{ position: 'absolute', top: 8, right: 8, display: 'flex', gap: 6, zIndex: 5, alignItems: 'center' }}>
{flash && <span style={{ fontSize: 11, color: '#22c55e', background: 'rgba(15,23,42,0.9)', padding: '4px 8px', borderRadius: 4 }}>Saved </span>}
{edit ? (
<>
{selected && <button type="button" onClick={deleteSelected} style={btn('danger')}>Delete box</button>}
<button type="button" onClick={save} disabled={!dirty} style={btn('primary', !dirty)}>{dirty ? 'Save changes' : 'Saved'}</button>
<button type="button" onClick={() => setEdit(false)} style={btn('ghost')}>Done</button>
</>
) : (
<>
<button type="button" onClick={() => setZoom((z) => Math.min(MAXZ, +(z + BTN_STEP).toFixed(2)))} style={btn('ghost')}></button>
<button type="button" onClick={() => setZoom((z) => Math.max(MINZ, +(z - BTN_STEP).toFixed(2)))} style={btn('ghost')}></button>
<button type="button" onClick={enterEdit} style={btn('primary')}>Edit</button>
</>
)}
</div>
{edit && (
<div style={{ position: 'absolute', bottom: 8, left: 8, zIndex: 5, fontSize: 11, color: '#cbd5e1', background: 'rgba(15,23,42,0.85)', padding: '4px 8px', borderRadius: 4 }}>
Drag to add a box · click a box then <b>Delete box</b> · then <b>Save changes</b>
</div>
)}
{!edit && zoom > 1 && (
<div style={{ position: 'absolute', bottom: 8, right: 8, zIndex: 5, fontSize: 11, color: '#fff', background: 'rgba(0,0,0,0.6)', padding: '2px 8px', borderRadius: 4, pointerEvents: 'none' }}>
{zoom.toFixed(1)}×
</div>
)}
</div>
);
};

View File

@@ -3,6 +3,8 @@ import { Box, Typography, Button, Chip, Switch, Dialog, DialogTitle, DialogConte
import { useNavigate, useLocation } from 'react-router-dom';
import ArrowBackIcon from '@mui/icons-material/ArrowBack';
import { ZoomableImage } from './ZoomableImage';
import { AnnotationOverlay } from './AnnotationOverlay';
import { VideoAnnotationOverlay } from './VideoAnnotationOverlay';
import { TEST_IMAGE_PREFIX } from '../../constants/media';
import SearchIcon from '@mui/icons-material/Search';
import { activityFeedsService } from '../../api/activityFeedsService';
@@ -104,6 +106,9 @@ export const AuditSession: React.FC = () => {
const [currentIndex, setCurrentIndex] = useState(0);
const [loading, setLoading] = useState(true);
const [carouselIndex, setCarouselIndex] = useState(0);
// Annotation (bounding-box) mode for the anomaly image pane. When on, the
// image renders with the editable AnnotationOverlay instead of ZoomableImage.
const [annotateMode, setAnnotateMode] = useState(false);
const [masterCarouselIndex, setMasterCarouselIndex] = useState(0);
// Manual test-frame capture: ref to read the playing video's currentTime,
// and a flag for the in-flight extraction round-trip.
@@ -490,6 +495,20 @@ export const AuditSession: React.FC = () => {
const currentAnomaly = anomalies[currentIndex];
// Persist edited bounding-box annotations for the current record: optimistic
// local update (keeps the overlay + feed in sync) + a fire-and-forget save to
// the JSONB column. Never blocks the audit flow; the boxes are baked into the
// archived media at confirm time.
const handleSaveAnnotations = useCallback((next: any) => {
const rec = anomalies[currentIndex];
if (!rec) return;
setAnomalies((prev) => prev.map((a, i) => (i === currentIndex ? { ...a, annotations: next } : a)));
activityFeedsService
.saveAnnotations({ id: rec.id, table: isRectActive ? 'rectification' : 'audits', annotations: next })
.catch(() => useToastStore.getState().showToast('Failed to save annotations.', 'error'));
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [currentIndex, anomalies, isRectActive]);
// Audited records are read-only: either it arrived from the server already
// audited (IsAudited = 1), or it was completed in this session. The backend
// enforces the same rule (POST /anomaly rejects audited rows with a 409),
@@ -1598,10 +1617,6 @@ export const AuditSession: React.FC = () => {
);
}
const getSeverityBadge = () => {
return <Chip label="HIGH" color="error" size="small" sx={{ borderRadius: 1, fontWeight: 'bold' }} />;
};
return (
<Box sx={{
display: 'flex',
@@ -1635,7 +1650,6 @@ export const AuditSession: React.FC = () => {
<Typography sx={{ fontWeight: 'bold', fontSize: '0.9rem' }}>
Item {currentIndex + 1} of {anomalies.length} <span style={{ color: '#64748b', fontWeight: 'normal' }}>(session)</span>
</Typography>
{getSeverityBadge()}
{isBulkMode && (
<Chip
label={`${stagedCount} staged`}
@@ -1810,7 +1824,18 @@ export const AuditSession: React.FC = () => {
<Box sx={{ flex: 1, bgcolor: '#0f172a', borderRadius: 2, border: '1px solid #1e293b', overflow: 'hidden', display: 'flex', flexDirection: 'column' }}>
<Box sx={{ px: 2, py: 1, borderBottom: '1px solid #1e293b', display: 'flex', justifyContent: 'space-between', alignItems: 'center', bgcolor: '#1e3a8a' }}>
<Typography sx={{ fontSize: '0.75rem', fontWeight: 'bold', color: '#60a5fa', letterSpacing: 1 }}>ANOMALY IMAGE</Typography>
<Typography sx={{ fontSize: '0.75rem', color: '#93c5fd' }}>{currentAnomaly?.date_of_audit || currentAnomaly?.Created_on}</Typography>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }} title="Show & edit bounding boxes on every record">
<Typography sx={{ fontSize: '0.7rem', color: '#c7d2fe' }}>Annotations</Typography>
<Switch
size="small"
checked={annotateMode}
onChange={(e) => setAnnotateMode(e.target.checked)}
sx={{ '& .MuiSwitch-switchBase.Mui-checked': { color: '#34d399' }, '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { backgroundColor: '#34d399' } }}
/>
</Box>
<Typography sx={{ fontSize: '0.75rem', color: '#93c5fd' }}>{currentAnomaly?.date_of_audit || currentAnomaly?.Created_on}</Typography>
</Box>
</Box>
<Box sx={{ flex: 1, position: 'relative', overflow: 'hidden', display: 'flex', alignItems: 'center', justifyContent: 'center' }}>
{mediaItems.length > 0 ? (
@@ -1822,6 +1847,8 @@ export const AuditSession: React.FC = () => {
height: '100%',
objectFit: 'contain'
}} />
{annotateMode && <VideoAnnotationOverlay videoRef={videoRef} annotations={currentAnomaly?.video_annotations || null} />}
<Button
size="small"
variant="contained"
@@ -1833,6 +1860,12 @@ export const AuditSession: React.FC = () => {
{capturing ? 'Capturing…' : 'Use this frame'}
</Button>
</>
) : (annotateMode && !!currentAnomaly?.Frame_Test && mediaItems[carouselIndex]?.url?.endsWith(currentAnomaly.Frame_Test)) ? (
<AnnotationOverlay
imageUrl={mediaItems[carouselIndex]?.url || ''}
annotations={currentAnomaly?.annotations || null}
onSave={handleSaveAnnotations}
/>
) : (
<ZoomableImage
src={mediaItems[carouselIndex]?.url}

View File

@@ -0,0 +1,76 @@
import React, { useRef, useEffect, useCallback } from 'react';
// Read-only bounding-box overlay for the audit-session VIDEO. Draws the current
// frame's boxes on a <canvas> over the existing <video>, synced to playback, with
// a show/hide toggle. Video boxes are frame-indexed and NOT editable (editing is
// image-only, by design — no 231 MB re-muxes). Source is the record's
// `video_annotations` column (seeded from the video's embedded metadata).
//
// Format: { "<frame>": { "<Label>": [ ["<trackId>", [x1,y1], [x2,y2] ], ... ] } }
type Box = [string, [number, number], [number, number]];
type FrameAnnotations = { [frame: string]: { [label: string]: Box[] } };
interface Props {
videoRef: React.RefObject<HTMLVideoElement | null>;
annotations: FrameAnnotations | null | undefined;
fps?: number;
}
const colorFor = (label: string) => {
let h = 0;
for (let i = 0; i < label.length; i++) h = (h * 31 + label.charCodeAt(i)) % 360;
return `hsl(${h}, 90%, 58%)`;
};
export const VideoAnnotationOverlay: React.FC<Props> = ({ videoRef, annotations, fps = 30 }) => {
const canvasRef = useRef<HTMLCanvasElement>(null);
const rafRef = useRef<number>(0);
const draw = useCallback(() => {
const cv = canvasRef.current, v = videoRef.current;
if (!cv || !v) return;
const cw = v.clientWidth, ch = v.clientHeight;
if (cw === 0 || ch === 0) return;
if (cv.width !== cw) cv.width = cw;
if (cv.height !== ch) cv.height = ch;
const ctx = cv.getContext('2d');
if (!ctx) return;
ctx.clearRect(0, 0, cw, ch);
if (!annotations || !v.videoWidth) return;
// object-fit: contain rect of the video within its element
const scale = Math.min(cw / v.videoWidth, ch / v.videoHeight);
const ox = (cw - v.videoWidth * scale) / 2, oy = (ch - v.videoHeight * scale) / 2;
// current frame → nearest annotated frame within ±3
const f = Math.round(v.currentTime * fps);
let boxes = annotations[String(f)];
for (let d = 1; d <= 3 && !boxes; d++) boxes = annotations[String(f - d)] || annotations[String(f + d)];
if (!boxes) return;
for (const label of Object.keys(boxes)) {
const col = colorFor(label);
for (const box of boxes[label]) {
const [, [x1, y1], [x2, y2]] = box;
const X = ox + x1 * scale, Y = oy + y1 * scale, W = (x2 - x1) * scale, H = (y2 - y1) * scale;
ctx.strokeStyle = col;
ctx.lineWidth = 2;
ctx.strokeRect(X, Y, W, H);
ctx.fillStyle = col;
ctx.font = '12px sans-serif';
const tw = ctx.measureText(label).width;
ctx.fillRect(X, Math.max(0, Y - 15), tw + 6, 15);
ctx.fillStyle = '#0b1220';
ctx.fillText(label, X + 3, Math.max(11, Y - 4));
}
}
}, [videoRef, annotations, fps]);
// continuous redraw while mounted (cheap canvas op) — tracks play, seek, resize
useEffect(() => {
const loop = () => { draw(); rafRef.current = requestAnimationFrame(loop); };
rafRef.current = requestAnimationFrame(loop);
return () => cancelAnimationFrame(rafRef.current);
}, [draw]);
return (
<canvas ref={canvasRef} style={{ position: 'absolute', top: 0, left: 0, width: '100%', height: '100%', pointerEvents: 'none', zIndex: 3 }} />
);
};