Central Verification Dashboard

server/src/ingest.rs

@@ -0,0 +1,254 @@
+use crate::models::ExportDoc;
+use chrono::{DateTime, NaiveDateTime, Utc};
+use sqlx::PgPool;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use walkdir::WalkDir;
+
+const VIDEO_EXTS: &[&str] = &["mp4", "mkv", "avi", "mov", "m4v", "webm"];
+
+fn is_video(p: &Path) -> bool {
+    p.extension()
+        .and_then(|e| e.to_str())
+        .map(|e| VIDEO_EXTS.contains(&e.to_lowercase().as_str()))
+        .unwrap_or(false)
+}
+
+/// Find the sibling export-JSON for a video. Supports `<file>_annotations.json`
+/// (video-annotator's default export name), `<file>.json`, and `<stem>.json`.
+fn sibling_json(video: &Path) -> Option<PathBuf> {
+    let dir = video.parent()?;
+    let file_name = video.file_name()?.to_str()?;
+    let stem = video.file_stem()?.to_str()?;
+    let candidates = [
+        format!("{file_name}_annotations.json"),
+        format!("{file_name}.json"),
+        format!("{stem}.json"),
+        format!("{stem}_annotations.json"),
+    ];
+    for c in candidates {
+        let p = dir.join(c);
+        if p.is_file() {
+            return Some(p);
+        }
+    }
+    None
+}
+
+fn parse_ts(s: &str) -> Option<DateTime<Utc>> {
+    if s.is_empty() {
+        return None;
+    }
+    if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
+        return Some(dt.with_timezone(&Utc));
+    }
+    NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S")
+        .ok()
+        .map(|n| n.and_utc())
+}
+
+/// Scan `dir`, upsert every video (and its annotations) into the project.
+/// Idempotent: re-running updates rows; never downgrades a 'verified' video.
+pub async fn ingest_project(db: &PgPool, project_id: i32, dir: &Path) -> anyhow::Result<usize> {
+    if !dir.is_dir() {
+        anyhow::bail!("ingest dir not found: {}", dir.display());
+    }
+    let mut count = 0usize;
+
+    for entry in WalkDir::new(dir).into_iter().filter_map(|e| e.ok()) {
+        let path = entry.path();
+        if !entry.file_type().is_file() || !is_video(path) {
+            continue;
+        }
+        let rel_path = path
+            .strip_prefix(dir)
+            .unwrap_or(path)
+            .to_string_lossy()
+            .to_string();
+        let file_name = path
+            .file_name()
+            .map(|n| n.to_string_lossy().to_string())
+            .unwrap_or_default();
+
+        // Never clobber a verified push (Phase 3): a worker's pushed annotations +
+        // completer/verify-time are authoritative, so skip re-ingesting this video.
+        let existing_status: Option<String> =
+            sqlx::query_scalar("SELECT status FROM videos WHERE project_id=$1 AND rel_path=$2")
+                .bind(project_id)
+                .bind(&rel_path)
+                .fetch_optional(db)
+                .await?;
+        if existing_status.as_deref() == Some("verified") {
+            tracing::debug!("skip re-ingest of verified video: {rel_path}");
+            continue;
+        }
+
+        let json_path = sibling_json(path);
+        let has_json = json_path.is_some();
+
+        // Parse the sibling JSON if present.
+        let doc: Option<ExportDoc> = json_path.as_ref().and_then(|jp| {
+            std::fs::read_to_string(jp)
+                .ok()
+                .and_then(|t| serde_json::from_str::<ExportDoc>(&t).ok())
+        });
+
+        let (width, height, fps, frame_count, time_ms) = match doc.as_ref().and_then(|d| d.video.as_ref()) {
+            Some(v) => (
+                v.width.map(|x| x as i32),
+                v.height.map(|x| x as i32),
+                v.fps,
+                v.frame_count.map(|x| x as i32),
+                v.annotation_time_ms,
+            ),
+            None => (None, None, None, None, 0i64),
+        };
+
+        // Aggregate annotation_count, primary annotator, annotated_at.
+        let mut ann_count: i32 = 0;
+        let mut by_freq: HashMap<String, i32> = HashMap::new();
+        let mut latest: Option<DateTime<Utc>> = None;
+        if let Some(d) = doc.as_ref() {
+            for a in &d.fixed_annotations {
+                ann_count += 1;
+                if !a.annotated_by.is_empty() {
+                    *by_freq.entry(a.annotated_by.clone()).or_default() += 1;
+                }
+                if let Some(t) = parse_ts(&a.created_at) {
+                    latest = Some(latest.map_or(t, |l| l.max(t)));
+                }
+            }
+            for a in &d.range_annotations {
+                ann_count += 1;
+                if !a.annotated_by.is_empty() {
+                    *by_freq.entry(a.annotated_by.clone()).or_default() += 1;
+                }
+                if let Some(t) = parse_ts(&a.created_at) {
+                    latest = Some(latest.map_or(t, |l| l.max(t)));
+                }
+            }
+        }
+        let primary_annotator = by_freq
+            .into_iter()
+            .max_by_key(|(_, c)| *c)
+            .map(|(u, _)| u)
+            .unwrap_or_default();
+        let status = if has_json && ann_count > 0 { "annotated" } else { "pending" };
+        let raw_json: Option<serde_json::Value> = json_path.as_ref().and_then(|jp| {
+            std::fs::read_to_string(jp).ok().and_then(|t| serde_json::from_str(&t).ok())
+        });
+
+        // Upsert the video. Keep 'verified' status sticky (Phase 3 pushes).
+        let video_id: i32 = sqlx::query_scalar(
+            r#"
+            INSERT INTO videos
+              (project_id, file_name, rel_path, has_json, width, height, fps, frame_count,
+               annotation_count, annotation_time_ms, primary_annotator, status, annotated_at, raw_json,
+               imported_count)
+            VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
+            ON CONFLICT (project_id, rel_path) DO UPDATE SET
+              file_name=EXCLUDED.file_name,
+              has_json=EXCLUDED.has_json,
+              width=EXCLUDED.width, height=EXCLUDED.height, fps=EXCLUDED.fps,
+              frame_count=EXCLUDED.frame_count,
+              annotation_count=EXCLUDED.annotation_count,
+              annotation_time_ms=EXCLUDED.annotation_time_ms,
+              primary_annotator=EXCLUDED.primary_annotator,
+              status = CASE WHEN videos.status='verified' THEN videos.status ELSE EXCLUDED.status END,
+              annotated_at=EXCLUDED.annotated_at,
+              raw_json=EXCLUDED.raw_json,
+              -- imported baseline = the NAS-ingested count (skipped for verified videos
+              -- via the early continue above, so a pushed pass's baseline is preserved).
+              imported_count=EXCLUDED.imported_count,
+              ingested_at=now()
+            RETURNING id
+            "#,
+        )
+        .bind(project_id)
+        .bind(&file_name)
+        .bind(&rel_path)
+        .bind(has_json)
+        .bind(width)
+        .bind(height)
+        .bind(fps)
+        .bind(frame_count)
+        .bind(ann_count)
+        .bind(time_ms)
+        .bind(&primary_annotator)
+        .bind(status)
+        .bind(latest)
+        .bind(raw_json)
+        .bind(ann_count)
+        .fetch_one(db)
+        .await?;
+
+        // Replace detailed annotation rows for this video.
+        if let Some(d) = doc.as_ref() {
+            replace_annotations(db, video_id, d).await?;
+        } else {
+            sqlx::query("DELETE FROM annotations WHERE video_id=$1")
+                .bind(video_id)
+                .execute(db)
+                .await?;
+        }
+        count += 1;
+    }
+    Ok(count)
+}
+
+/// Replace all annotation rows for a video with the contents of an export doc.
+/// Shared by ingest (NAS JSON) and the Phase 3 worker push. Returns the row count
+/// written (each range annotation contributes 1–2 rows: start, and end if present).
+pub async fn replace_annotations(
+    db: &PgPool,
+    video_id: i32,
+    d: &ExportDoc,
+) -> anyhow::Result<i32> {
+    sqlx::query("DELETE FROM annotations WHERE video_id=$1")
+        .bind(video_id)
+        .execute(db)
+        .await?;
+    let mut written = 0i32;
+    for a in &d.fixed_annotations {
+        insert_ann(db, video_id, a.frame_number, &a.label, &a.side, &a.shape_type,
+            &a.vertices, &a.annotated_by, &a.review_status, &a.remark, &a.subclass, &a.created_at).await?;
+        written += 1;
+    }
+    for a in &d.range_annotations {
+        insert_ann(db, video_id, a.start_frame, &a.label, &a.side, &a.shape_type,
+            &a.start_vertices, &a.annotated_by, &a.review_status, &a.remark, &a.subclass, &a.created_at).await?;
+        written += 1;
+        if let (Some(ef), Some(ev)) = (a.end_frame, a.end_vertices.as_ref()) {
+            insert_ann(db, video_id, ef, &a.label, &a.side, &a.shape_type,
+                ev, &a.annotated_by, &a.review_status, &a.remark, &a.subclass, &a.created_at).await?;
+            written += 1;
+        }
+    }
+    Ok(written)
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn insert_ann(
+    db: &PgPool, video_id: i32, frame: i64, label: &str, side: &str, shape: &str,
+    vertices: &serde_json::Value, by: &str, review: &str, remark: &str, subclass: &str, created_at: &str,
+) -> anyhow::Result<()> {
+    sqlx::query(
+        r#"INSERT INTO annotations
+           (video_id, frame_number, label, side, shape_type, vertices, annotated_by, review_status, remark, subclass, created_at)
+           VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)"#,
+    )
+    .bind(video_id)
+    .bind(frame as i32)
+    .bind(label)
+    .bind(side)
+    .bind(if shape.is_empty() { "bbox" } else { shape })
+    .bind(vertices)
+    .bind(by)
+    .bind(if review.is_empty() { "none" } else { review })
+    .bind(remark)
+    .bind(subclass)
+    .bind(created_at)
+    .execute(db)
+    .await?;
+    Ok(())
+}