Ravi/Centralised_VideoVerification_Dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

map and chainage calculation

Browse Source
This commit is contained in:
ravi
2026-07-07 22:42:47 +05:30
parent 05ce1f2cdb
commit 308d6bc129
17 changed files with 4178 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/Dockerfile
View File

@@ -10,7 +10,9 @@ RUN cargo build --release
FROM debian:bookworm-slim
# nfs-common provides mount.nfs for `nfs`-kind projects (mounted on demand at sync/download time).
# libimage-exiftool-perl provides exiftool for the GPS-track extraction sweep (map/chainage).
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates nfs-common \
libimage-exiftool-perl \
&& rm -rf /var/lib/apt/lists/*
COPY --from=build /build/server/target/release/central-server /usr/local/bin/central-server
ENV BIND_ADDR=0.0.0.0:8080

1057
server/src/chainage.rs Normal file
View File

File diff suppressed because it is too large Load Diff

888
server/src/gps.rs Normal file
View File

@@ -0,0 +1,888 @@
//! GPS pipeline: exiftool track extraction (background sweep), shared track math,
//! and the Stage-1 "Viterbi" smoother with its preview/keep/discard endpoint.
//!
//! Track representation everywhere: `[[lat, lon, t_s], …]` (t_s = seconds since the
//! first GPS sample). The RAW extracted track (`videos.gps_track`) is immutable once
//! scanned; a kept correction lives in `gps_track_corrected` and is preferred by the
//! map + chainage. Raw is never modified — discarding a correction restores raw.
//!
//! Lifecycle (`videos.gps_status`): 'pending' → 'ok' | 'none' (no GPS metadata) |
//! 'error' (exiftool/mount failure, message in `gps_error`). The sweep processes ONE
//! video at a time (exiftool -ee reads the whole file over NFS — never hammer the NAS).
use crate::auth::AuthUser;
use crate::{err, ApiResult, AppState};
use axum::{
extract::{Path as AxPath, State},
http::StatusCode,
Json,
};
use serde::Deserialize;
use serde_json::{json, Value};
use std::collections::BTreeMap;
use std::time::Duration;
/// One GPS sample: [lat, lon, t_s].
pub type Pt = [f64; 3];
/// Max samples stored per track (1 Hz ⇒ ~83 min of video at full fidelity).
const MAX_TRACK_PTS: usize = 5000;
/// Hard deadline for one exiftool scan (-ee reads the whole file over NFS).
const EXIFTOOL_TIMEOUT: Duration = Duration::from_secs(600);
/// Sweep pacing: idle poll when nothing is pending / pause between videos / back-off
/// after a failure (so an unreachable NAS isn't probed in a hot loop).
const SWEEP_IDLE: Duration = Duration::from_secs(60);
const SWEEP_BETWEEN: Duration = Duration::from_millis(500);
const SWEEP_AFTER_ERROR: Duration = Duration::from_secs(5);
// ---- track math (shared with chainage) ----
/// WGS84 ellipsoid: semi-major axis + first eccentricity squared.
const WGS84_A: f64 = 6_378_137.0;
const WGS84_E2: f64 = 0.006_694_379_990_14;
/// Meters per degree of latitude / longitude at a given latitude, from the WGS84
/// meridian (M) and prime-vertical (N) radii. Distance and snapping both use this,
/// so every meter in the system is measured the same way.
pub fn wgs84_m_per_deg(lat_deg: f64) -> (f64, f64) {
let phi = lat_deg.to_radians();
let w = (1.0 - WGS84_E2 * phi.sin().powi(2)).sqrt();
let m = WGS84_A * (1.0 - WGS84_E2) / (w * w * w);
let n = WGS84_A / w;
let rad = std::f64::consts::PI / 180.0;
(m * rad, n * phi.cos() * rad)
}
/// WGS84 local-ellipsoid distance between two nearby points. Essentially exact at
/// GPS sample spacing (meters–tens of meters) and <2e-5 relative for spans up to
/// ~100 km — unlike spherical haversine, whose 0.1–0.3% systematic bias would land
/// directly in sr_chainage. (Not valid across the ±180° meridian — irrelevant here.)
pub fn dist_m(lat1: f64, lon1: f64, lat2: f64, lon2: f64) -> f64 {
let (mlat, mlon) = wgs84_m_per_deg((lat1 + lat2) * 0.5);
let dy = (lat2 - lat1) * mlat;
let dx = (lon2 - lon1) * mlon;
(dx * dx + dy * dy).sqrt()
}
pub fn polyline_len_m(track: &[Pt]) -> f64 {
track
.windows(2)
.map(|w| dist_m(w[0][0], w[0][1], w[1][0], w[1][1]))
.sum()
}
/// Parse a stored JSONB track (`[[lat,lon,t_s],…]`) back into points. Tolerant:
/// skips malformed entries.
pub fn parse_track(v: &Value) -> Vec<Pt> {
v.as_array()
.map(|a| {
a.iter()
.filter_map(|p| {
let p = p.as_array()?;
Some([p.first()?.as_f64()?, p.get(1)?.as_f64()?, p.get(2).and_then(|t| t.as_f64()).unwrap_or(0.0)])
})
.collect()
})
.unwrap_or_default()
}
/// Serialize a track for storage/transport, rounding to keep the JSON small
/// (1e-6 deg ≈ 0.1 m; 0.1 s time resolution).
pub fn track_value(track: &[Pt]) -> Value {
Value::Array(
track
.iter()
.map(|p| {
json!([
(p[0] * 1e6).round() / 1e6,
(p[1] * 1e6).round() / 1e6,
(p[2] * 10.0).round() / 10.0
])
})
.collect(),
)
}
/// Cap a track at `max` points (uniform stride, always keeping the last point).
pub fn downsample(track: Vec<Pt>, max: usize) -> Vec<Pt> {
if track.len() <= max || max < 2 {
return track;
}
let stride = track.len().div_ceil(max);
let last = *track.last().unwrap();
let mut out: Vec<Pt> = track.into_iter().step_by(stride).collect();
if out.last() != Some(&last) {
out.push(last);
}
out
}
// ---- exiftool extraction ----
/// What one exiftool scan yields: the GPS samples (t_s relative to the first fix),
/// the first fix's absolute epoch (None when the camera wrote no GPSDateTime), the
/// container's media duration in seconds, and the video frame rate. Empty samples =
/// no GPS metadata. Duration + fps let us do frame→position math on raw NAS videos
/// that were never annotated (no sibling JSON ⇒ no fps in our DB otherwise).
struct ScanResult {
samples: Vec<Pt>,
t0_abs: Option<f64>,
duration_s: Option<f64>,
fps: Option<f64>,
}
/// Run exiftool over a video and collect its embedded GPS time-series (Doc{N}
/// groups, the format dashcams/DJI write at ~1 Hz). Same flags as the proven
/// desktop extractor, plus container Duration + VideoFrameRate.
async fn run_exiftool(path: &std::path::Path) -> Result<ScanResult, String> {
let child = tokio::process::Command::new("exiftool")
.args([
"-ee", "-api", "LargeFileSupport=1", "-G3", "-json", "-n",
"-GPSLatitude", "-GPSLongitude", "-GPSDateTime", "-Duration", "-VideoFrameRate",
])
.arg(path)
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true)
.spawn()
.map_err(|e| format!("failed to run exiftool (is it installed?): {e}"))?;
let output = match tokio::time::timeout(EXIFTOOL_TIMEOUT, child.wait_with_output()).await {
Ok(r) => r.map_err(|e| format!("exiftool failed: {e}"))?,
// kill_on_drop terminates the scan when the timeout drops the child.
Err(_) => return Err("exiftool timed out (file too slow to read?)".into()),
};
let stdout = String::from_utf8_lossy(&output.stdout);
let parsed: Value = serde_json::from_str(stdout.trim()).map_err(|_| {
let se = String::from_utf8_lossy(&output.stderr);
let msg: String = se.trim().chars().take(300).collect();
if msg.is_empty() {
"exiftool produced no output".to_string()
} else {
format!("exiftool: {msg}")
}
})?;
let obj = parsed
.as_array()
.and_then(|a| a.first())
.and_then(|v| v.as_object())
.ok_or_else(|| "exiftool output was not a non-empty array".to_string())?;
// exiftool reports per-file problems inline (e.g. "Error: File is empty").
if let Some((_, e)) = obj.iter().find(|(k, _)| *k == "Error" || k.ends_with(":Error")) {
if let Some(msg) = e.as_str() {
return Err(format!("exiftool: {msg}"));
}
}
// Numeric field across any -G3 group (container + per-track may each report one);
// `reduce` picks the max, which is the sensible choice for both Duration and fps.
let numeric = |name: &str| -> Option<f64> {
obj.iter()
.filter(|(k, _)| k.as_str() == name || k.ends_with(&format!(":{name}")))
.filter_map(|(_, v)| v.as_f64().or_else(|| v.as_str().and_then(|s| s.parse().ok())))
.reduce(f64::max)
};
let duration = numeric("Duration");
let fps = numeric("VideoFrameRate").filter(|f| *f > 0.0);
let (samples, t0_abs) = collect_samples(obj);
Ok(ScanResult { samples, t0_abs, duration_s: duration, fps })
}
/// Group `Doc{N}:GPSLatitude/GPSLongitude/GPSDateTime` keys into ordered samples
/// with relative timestamps (falls back to 1 Hz when GPSDateTime is missing).
/// Also returns the first fix's ABSOLUTE epoch — GPS time is UTC on every camera,
/// so consecutive files can be matched by timestamp (start-of-video stitching).
fn collect_samples(obj: &serde_json::Map<String, Value>) -> (Vec<Pt>, Option<f64>) {
#[derive(Default)]
struct Partial {
lat: Option<f64>,
lon: Option<f64>,
t: Option<f64>,
}
let mut by_doc: BTreeMap<u32, Partial> = BTreeMap::new();
for (k, v) in obj {
let Some((doc_idx, field)) = split_doc_key(k) else { continue };
let entry = by_doc.entry(doc_idx).or_default();
match field {
"GPSLatitude" => entry.lat = v.as_f64(),
"GPSLongitude" => entry.lon = v.as_f64(),
"GPSDateTime" => entry.t = v.as_str().and_then(parse_gps_datetime),
_ => {}
}
}
let mut samples: Vec<Pt> = Vec::with_capacity(by_doc.len());
let mut t0: Option<f64> = None;
let mut t0_abs: Option<f64> = None;
for (idx, p) in by_doc {
let (Some(lat), Some(lon)) = (p.lat, p.lon) else { continue };
if t0.is_none() {
t0_abs = p.t; // real epoch only when the camera stamped it
}
let t_abs = p.t.unwrap_or(idx as f64);
let t_rel = t_abs - *t0.get_or_insert(t_abs);
samples.push([lat, lon, t_rel]);
}
(samples, t0_abs)
}
fn split_doc_key(k: &str) -> Option<(u32, &str)> {
let (group, field) = k.split_once(':')?;
let idx: u32 = group.strip_prefix("Doc")?.parse().ok()?;
Some((idx, field))
}
/// "2025:08:02 09:33:31Z" → seconds since epoch (approx; only relative deltas are
/// used). Civil-to-days via Howard Hinnant's algorithm — no chrono parse needed for
/// exiftool's colon-separated dates.
fn parse_gps_datetime(s: &str) -> Option<f64> {
let s = s.trim_end_matches('Z').trim();
let (date, time) = s.split_once(' ')?;
let mut dp = date.split(':');
let y: i64 = dp.next()?.parse().ok()?;
let mo: i64 = dp.next()?.parse().ok()?;
let d: i64 = dp.next()?.parse().ok()?;
let mut tp = time.split(':');
let h: i64 = tp.next()?.parse().ok()?;
let mi: i64 = tp.next()?.parse().ok()?;
let sec: f64 = tp.next()?.parse().ok()?;
let y_adj = if mo <= 2 { y - 1 } else { y };
let era = y_adj.div_euclid(400);
let yoe = y_adj - era * 400;
let m = if mo > 2 { mo - 3 } else { mo + 9 };
let doy = (153 * m + 2) / 5 + d - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
let days = era * 146097 + doe - 719468;
Some(days as f64 * 86400.0 + (h * 3600 + mi * 60) as f64 + sec)
}
// ---- background sweep ----
/// Forever-loop: pick ONE pending video, scan it, store the result, repeat.
/// Serial by design (requirement: throttle — never scan the NAS in parallel).
pub async fn sweep(state: AppState) {
loop {
let next: Option<(i32, String, String, String)> = sqlx::query_as(
r#"SELECT v.id, v.rel_path, p.source_path, p.source_kind
FROM videos v JOIN projects p ON v.project_id = p.id
WHERE v.gps_status = 'pending'
ORDER BY v.id LIMIT 1"#,
)
.fetch_optional(&state.db)
.await
.unwrap_or(None);
let Some((id, rel_path, source_path, source_kind)) = next else {
tokio::time::sleep(SWEEP_IDLE).await;
continue;
};
let ok = scan_one(&state, id, &rel_path, &source_path, &source_kind).await;
tokio::time::sleep(if ok { SWEEP_BETWEEN } else { SWEEP_AFTER_ERROR }).await;
}
}
/// Scan a single video and persist the outcome. Returns false on error (the video
/// is marked 'error' either way, so the sweep never re-picks it until a re-scan).
async fn scan_one(state: &AppState, id: i32, rel_path: &str, source_path: &str, source_kind: &str) -> bool {
let (sp, kind, host, opts) =
(source_path.to_string(), source_kind.to_string(), state.nfs_host.clone(), state.nfs_opts.clone());
let dir = match tokio::task::spawn_blocking(move || crate::nfs::resolve_dir(&sp, &kind, &host, &opts)).await {
Ok(Ok(d)) => d,
Ok(Err(e)) => return mark_error(&state.db, id, &e).await,
Err(e) => return mark_error(&state.db, id, &format!("mount task failed: {e}")).await,
};
match run_exiftool(&dir.join(rel_path)).await {
Ok(ScanResult { samples, t0_abs, duration_s, fps }) if samples.len() >= 2 => {
// Data-quality stats are measured BEFORE the storage cap: the raw fix
// count and the largest inter-fix gap (dropouts) must reflect reality.
let sample_count = samples.len() as i32;
let max_gap = samples.windows(2).map(|w| w[1][2] - w[0][2]).fold(0.0_f64, f64::max);
let end_epoch = t0_abs.map(|t0| t0 + samples.last().unwrap()[2]);
let samples = downsample(samples, MAX_TRACK_PTS);
let len = polyline_len_m(&samples);
let res = sqlx::query(
r#"UPDATE videos SET gps_status='ok', gps_track=$2, gps_len_m=$3, gps_error=NULL,
gps_scanned_at=now(),
gps_start_epoch=$4, gps_end_epoch=$5, gps_sample_count=$6, gps_max_gap_s=$7,
duration_s=$8,
-- fill fps for raw NAS videos (no sibling JSON) so frame→position
-- math works; never clobber a JSON/desktop-derived fps.
fps=COALESCE(fps, $9),
-- a fresh raw track invalidates any correction derived from the old one
gps_track_corrected=NULL, gps_corrected_by=NULL, gps_corrected_at=NULL,
gps_corrected_note=NULL
WHERE id=$1"#,
)
.bind(id)
.bind(track_value(&samples))
.bind(len)
.bind(t0_abs)
.bind(end_epoch)
.bind(sample_count)
.bind(max_gap)
.bind(duration_s)
.bind(fps)
.execute(&state.db)
.await;
match res {
Ok(_) => {
tracing::info!("gps: video {id} ok — {sample_count} samples, {len:.0} m, max gap {max_gap:.0}s");
// Auto-correct this video (spikes + start stitch), then its
// temporal successor — its start stitch may only now be possible.
auto_correct(&state.db, id).await;
if let Some(end) = end_epoch {
let succ: Option<i32> = sqlx::query_scalar(
r#"SELECT v.id FROM videos v, videos me
WHERE me.id=$1 AND v.id<>$1 AND v.project_id=me.project_id
AND v.gps_status='ok' AND v.gps_start_epoch IS NOT NULL
AND v.gps_start_epoch > $2 AND v.gps_start_epoch - $2 <= $3
AND (CASE WHEN v.rel_path LIKE '%/%' THEN regexp_replace(v.rel_path,'/[^/]*$','') ELSE '' END)
= (CASE WHEN me.rel_path LIKE '%/%' THEN regexp_replace(me.rel_path,'/[^/]*$','') ELSE '' END)
ORDER BY v.gps_start_epoch LIMIT 1"#,
)
.bind(id)
.bind(end)
.bind(STITCH_MAX_GAP_S)
.fetch_optional(&state.db)
.await
.unwrap_or(None);
if let Some(succ) = succ {
auto_correct(&state.db, succ).await;
}
}
true
}
Err(e) => mark_error(&state.db, id, &format!("db write failed: {e}")).await,
}
}
Ok(_) => {
let _ = sqlx::query(
"UPDATE videos SET gps_status='none', gps_error=NULL, gps_scanned_at=now() WHERE id=$1",
)
.bind(id)
.execute(&state.db)
.await;
tracing::info!("gps: video {id} has no GPS metadata");
true
}
Err(e) => mark_error(&state.db, id, &e).await,
}
}
async fn mark_error(db: &sqlx::PgPool, id: i32, msg: &str) -> bool {
tracing::warn!("gps: video {id} scan failed: {msg}");
let _ = sqlx::query(
"UPDATE videos SET gps_status='error', gps_error=$2, gps_scanned_at=now() WHERE id=$1",
)
.bind(id)
.bind(msg)
.execute(db)
.await;
false
}
// ---- Stage-1 "Viterbi" smoother ----
/// Speed gate: hops implying more than this are implausible (≈160 km/h).
const MAX_SPEED_MPS: f64 = 45.0;
/// Cost of dropping (re-interpolating) one sample. An outlier is dropped when
/// keeping it costs more than this.
const DROP_PENALTY: f64 = 30.0;
/// Max consecutive dropped samples + 1 (DP transition window).
const VITERBI_WINDOW: usize = 12;
/// Min-cost keep/drop path over the samples (Viterbi over a keep-lattice):
/// transition cost between consecutive KEPT samples grows quadratically with the
/// implied speed above `MAX_SPEED_MPS`; every dropped sample costs `DROP_PENALTY`.
/// Dropped samples are re-interpolated (by time) between their kept neighbors —
/// principled outlier rejection with no road network. Returns the corrected track
/// (same length + timestamps as the input) and how many samples were replaced.
pub fn viterbi_smooth(track: &[Pt]) -> (Vec<Pt>, usize) {
let n = track.len();
if n < 3 {
return (track.to_vec(), 0);
}
let trans_cost = |a: &Pt, b: &Pt| -> f64 {
let dt = (b[2] - a[2]).max(0.5);
let v = dist_m(a[0], a[1], b[0], b[1]) / dt;
let excess = (v - MAX_SPEED_MPS).max(0.0);
(excess / 5.0).powi(2)
};
// cost[i] = min cost of a path whose last kept sample is i.
let mut cost = vec![f64::INFINITY; n];
let mut prev = vec![usize::MAX; n];
for i in 0..n {
if i <= VITERBI_WINDOW {
// Start the path at i, paying for the dropped leading samples.
cost[i] = DROP_PENALTY * i as f64;
}
for j in i.saturating_sub(VITERBI_WINDOW)..i {
if cost[j].is_finite() {
let c = cost[j] + DROP_PENALTY * (i - j - 1) as f64 + trans_cost(&track[j], &track[i]);
if c < cost[i] {
cost[i] = c;
prev[i] = j;
}
}
}
}
// End the path at the sample minimizing total cost incl. dropped trailing ones.
let mut end = n - 1;
let mut best = f64::INFINITY;
for i in n.saturating_sub(VITERBI_WINDOW + 1)..n {
let c = cost[i] + DROP_PENALTY * (n - 1 - i) as f64;
if c < best {
best = c;
end = i;
}
}
let mut kept = vec![false; n];
let mut i = end;
loop {
kept[i] = true;
if prev[i] == usize::MAX {
break;
}
i = prev[i];
}
// Rebuild: kept samples stay; dropped ones are re-interpolated between the
// nearest kept neighbors (clamped to the first/last kept at the edges).
let kept_idx: Vec<usize> = (0..n).filter(|&k| kept[k]).collect();
let mut out = track.to_vec();
let mut replaced = 0usize;
for k in 0..n {
if kept[k] {
continue;
}
let after = kept_idx.partition_point(|&ki| ki < k);
let (lo, hi) = match (after.checked_sub(1).map(|a| kept_idx[a]), kept_idx.get(after)) {
(Some(lo), Some(&hi)) => (lo, hi),
(None, Some(&hi)) => (hi, hi),
(Some(lo), None) => (lo, lo),
(None, None) => continue, // unreachable: at least one sample is kept
};
let (a, b) = (&track[lo], &track[hi]);
let f = if b[2] > a[2] { ((track[k][2] - a[2]) / (b[2] - a[2])).clamp(0.0, 1.0) } else { 0.0 };
out[k] = [a[0] + (b[0] - a[0]) * f, a[1] + (b[1] - a[1]) * f, track[k][2]];
replaced += 1;
}
(out, replaced)
}
// ---- auto-correction: spikes + start-of-video stitching ----
/// Two files are one continuous recording only when the GPS gap between them is at
/// most this many seconds.
const STITCH_MAX_GAP_S: f64 = 180.0;
/// Gaps at or below this are the normal ~1 Hz cadence across a file boundary —
/// nothing is missing.
const STITCH_MIN_GAP_S: f64 = 2.0;
/// Decide the time-shift for a missing-start stitch. `gap_s` = seconds between the
/// predecessor's last fix and this video's first fix; `span_s` = this track's own
/// first→last duration; `duration_s` = the video's real length when known
/// (frame_count / fps); `jump_m` = distance from the predecessor's last fix to this
/// video's first fix. Returns the seconds this track starts late (its samples are
/// shifted by this), or None when nothing should be stitched.
fn plan_stitch(gap_s: f64, span_s: f64, duration_s: Option<f64>, jump_m: f64) -> Option<f64> {
if gap_s <= STITCH_MIN_GAP_S || gap_s > STITCH_MAX_GAP_S {
return None; // healthy rollover, or not a continuous recording
}
if jump_m / gap_s > MAX_SPEED_MPS {
return None; // the vehicle can't have covered the jump — different session
}
let mut shift = gap_s - 1.0; // one normal sample period isn't missing data
if let Some(dur) = duration_s {
// The track already covers the whole file ⇒ the gap was the PREDECESSOR's
// problem (missing end), not ours — shifting would misalign every frame.
let missing = dur - span_s;
if missing < STITCH_MIN_GAP_S {
return None;
}
shift = shift.min(missing);
}
(shift > 0.5).then_some(shift)
}
/// The unified correction result: Viterbi spike removal + (when a temporal
/// predecessor exists in the same folder) a missing-start stitch that re-bases the
/// clock so t = 0 is the actual video start. Raw is never modified.
pub struct CorrectionOutcome {
pub raw: Vec<Pt>,
pub corrected: Vec<Pt>,
pub replaced: usize,
pub stitched_from: Option<String>,
pub shift_s: f64,
pub note: String,
}
impl CorrectionOutcome {
pub fn changed(&self) -> bool {
self.replaced > 0 || self.stitched_from.is_some()
}
}
/// Compute (but do not store) the correction for a video. Errors when the video has
/// no usable raw track.
pub async fn compute_correction(db: &sqlx::PgPool, video_id: i32) -> Result<CorrectionOutcome, String> {
#[allow(clippy::type_complexity)]
let row: Option<(Option<Value>, Option<f64>, Option<f64>, Option<i32>, Option<f64>, i32, String)> = sqlx::query_as(
r#"SELECT gps_track, gps_start_epoch, fps, frame_count, duration_s, project_id, rel_path
FROM videos WHERE id=$1 AND gps_status='ok'"#,
)
.bind(video_id)
.fetch_optional(db)
.await
.map_err(|e| e.to_string())?;
let Some((track, start_epoch, fps, frame_count, duration_s, project_id, rel_path)) = row else {
return Err("video has no GPS track".into());
};
let raw = track.as_ref().map(parse_track).unwrap_or_default();
if raw.len() < 2 {
return Err("video has no GPS track".into());
}
let (mut corrected, replaced) = viterbi_smooth(&raw);
let mut notes: Vec<String> = Vec::new();
if replaced > 0 {
notes.push(format!("{replaced} spike(s) re-interpolated"));
}
// Missing-start stitch: the nearest earlier-ending video in the same folder,
// matched by absolute GPS time (UTC on every camera — no timezone games).
let mut stitched_from = None;
let mut shift_s = 0.0;
if let Some(start_epoch) = start_epoch {
let folder = match rel_path.rfind('/') {
Some(i) => rel_path[..i].to_string(),
None => String::new(),
};
let prev: Option<(String, f64, Value)> = sqlx::query_as(
r#"SELECT file_name, gps_end_epoch, gps_track FROM videos
WHERE project_id=$1 AND id<>$2 AND gps_track IS NOT NULL AND gps_end_epoch IS NOT NULL
AND gps_end_epoch < $3 AND $3 - gps_end_epoch <= $4
AND (CASE WHEN rel_path LIKE '%/%' THEN regexp_replace(rel_path,'/[^/]*$','') ELSE '' END) = $5
ORDER BY gps_end_epoch DESC LIMIT 1"#,
)
.bind(project_id)
.bind(video_id)
.bind(start_epoch)
.bind(STITCH_MAX_GAP_S)
.bind(&folder)
.fetch_optional(db)
.await
.map_err(|e| e.to_string())?;
if let Some((prev_name, prev_end, prev_track)) = prev {
// Borrow from the predecessor's RAW track (clean provenance — its own
// correction may already contain stitched points).
let prev_pts = parse_track(&prev_track);
if let Some(anchor) = prev_pts.last() {
let gap = start_epoch - prev_end;
let span = corrected.last().unwrap()[2] - corrected[0][2];
// Container duration (from the scan) first; the annotation-doc
// fps/frame_count only as a fallback for pre-upgrade rows.
let duration = duration_s.or(match (fps, frame_count) {
(Some(f), Some(n)) if f > 0.0 => Some(n as f64 / f),
_ => None,
});
let jump = dist_m(anchor[0], anchor[1], corrected[0][0], corrected[0][1]);
if let Some(shift) = plan_stitch(gap, span, duration, jump) {
for p in corrected.iter_mut() {
p[2] += shift;
}
corrected.insert(0, [anchor[0], anchor[1], 0.0]);
shift_s = shift;
notes.push(format!(
"start stitched from {prev_name} (first {shift:.1}s of GPS were missing — clock re-based)"
));
stitched_from = Some(prev_name);
}
}
}
}
Ok(CorrectionOutcome {
raw,
corrected,
replaced,
stitched_from,
shift_s,
note: notes.join(" · "),
})
}
/// Compute + store the correction as 'auto' — but never over a human decision:
/// only when the video is untouched or its current correction is itself 'auto'.
/// A no-op correction clears a stale auto one. Best-effort (sweep path).
pub async fn auto_correct(db: &sqlx::PgPool, video_id: i32) {
let eligible: Option<bool> = sqlx::query_scalar(
"SELECT (gps_corrected_by IS NULL OR gps_corrected_by='auto') FROM videos WHERE id=$1 AND gps_status='ok'",
)
.bind(video_id)
.fetch_optional(db)
.await
.ok()
.flatten();
if eligible != Some(true) {
return;
}
match compute_correction(db, video_id).await {
Ok(o) if o.changed() => {
let len = polyline_len_m(&o.corrected);
let res = sqlx::query(
r#"UPDATE videos SET gps_track_corrected=$2, gps_corrected_by='auto',
gps_corrected_at=now(), gps_corrected_note=$3, gps_len_m=$4 WHERE id=$1"#,
)
.bind(video_id)
.bind(track_value(&o.corrected))
.bind(&o.note)
.bind(len)
.execute(db)
.await;
match res {
Ok(_) => tracing::info!("gps: video {video_id} auto-corrected — {}", o.note),
Err(e) => tracing::warn!("gps: video {video_id} auto-correct store failed: {e}"),
}
}
Ok(o) => {
// Nothing to fix (anymore) — drop a stale auto correction if present.
let _ = sqlx::query(
r#"UPDATE videos SET gps_track_corrected=NULL, gps_corrected_by=NULL,
gps_corrected_at=NULL, gps_corrected_note=NULL, gps_len_m=$2
WHERE id=$1 AND gps_corrected_by='auto'"#,
)
.bind(video_id)
.bind(polyline_len_m(&o.raw))
.execute(db)
.await;
}
Err(e) => tracing::debug!("gps: video {video_id} auto-correct skipped: {e}"),
}
}
// ---- endpoints ----
/// `POST /api/videos/:id/gps/rescan` — queue a video for a fresh metadata scan
/// (admin). The sweep picks it up; any previous error is cleared.
pub async fn rescan(State(s): State<AppState>, user: AuthUser, AxPath(id): AxPath<i32>) -> ApiResult<Value> {
user.require_admin()?;
let updated: Option<i32> =
sqlx::query_scalar("UPDATE videos SET gps_status='pending', gps_error=NULL WHERE id=$1 RETURNING id")
.bind(id)
.fetch_optional(&s.db)
.await
.map_err(err)?;
if updated.is_none() {
return Err((StatusCode::NOT_FOUND, "video not found".into()));
}
Ok(Json(json!({ "queued": true })))
}
#[derive(Deserialize)]
pub struct CorrectRequest {
/// 'preview' (compute, don't save) | 'keep' (save corrected) | 'discard' (revert to raw).
pub action: String,
}
/// `POST /api/videos/:id/gps/correct {action}` — the keep-or-discard correction flow
/// (admin), running the SAME pipeline the scan sweep auto-applies (spike removal +
/// missing-start stitch). Preview returns original + corrected tracks so the map can
/// overlay them; keep persists (attributed to the admin); discard reverts to raw AND
/// tombstones the video so the sweep won't re-auto-correct it (a re-scan resets that).
pub async fn correct(
State(s): State<AppState>,
user: AuthUser,
AxPath(id): AxPath<i32>,
Json(req): Json<CorrectRequest>,
) -> ApiResult<Value> {
user.require_admin()?;
let exists: Option<String> = sqlx::query_scalar("SELECT gps_status FROM videos WHERE id=$1")
.bind(id)
.fetch_optional(&s.db)
.await
.map_err(err)?;
let Some(gps_status) = exists else {
return Err((StatusCode::NOT_FOUND, "video not found".into()));
};
if gps_status != "ok" {
return Err((StatusCode::BAD_REQUEST, "video has no GPS track".into()));
}
match req.action.as_str() {
"discard" => {
let raw_track: Option<Value> = sqlx::query_scalar("SELECT gps_track FROM videos WHERE id=$1")
.bind(id)
.fetch_one(&s.db)
.await
.map_err(err)?;
let len = raw_track.as_ref().map(|t| polyline_len_m(&parse_track(t))).unwrap_or_default();
// Tombstone: gps_corrected_by keeps the admin's name with a NULL track,
// so auto_correct (which only touches NULL/'auto') leaves this video raw.
sqlx::query(
r#"UPDATE videos SET gps_track_corrected=NULL,
gps_corrected_by=$2, gps_corrected_at=now(),
gps_corrected_note='reverted to raw — auto-correction off (re-scan to reset)',
gps_len_m=$3
WHERE id=$1"#,
)
.bind(id)
.bind(&user.username)
.bind(len)
.execute(&s.db)
.await
.map_err(err)?;
Ok(Json(json!({ "action": "discard", "len_m": len })))
}
"preview" | "keep" => {
let o = compute_correction(&s.db, id).await.map_err(|e| (StatusCode::BAD_REQUEST, e))?;
let len_before = polyline_len_m(&o.raw);
let len_after = polyline_len_m(&o.corrected);
if req.action == "keep" {
sqlx::query(
r#"UPDATE videos SET gps_track_corrected=$2, gps_corrected_by=$3,
gps_corrected_at=now(), gps_corrected_note=$4, gps_len_m=$5 WHERE id=$1"#,
)
.bind(id)
.bind(track_value(&o.corrected))
.bind(&user.username)
.bind(&o.note)
.bind(len_after)
.execute(&s.db)
.await
.map_err(err)?;
crate::worker::log_event(&s.db, id, &user.username, "gps_corrected",
json!({ "replaced": o.replaced, "stitched": o.stitched_from.is_some() })).await;
}
Ok(Json(json!({
"action": req.action,
"points": o.raw.len(),
"replaced": o.replaced,
"stitched_from": o.stitched_from,
"shift_s": o.shift_s,
"note": o.note,
"len_before_m": len_before,
"len_after_m": len_after,
"original": track_value(&o.raw),
"corrected": track_value(&o.corrected),
})))
}
_ => Err((StatusCode::BAD_REQUEST, "action must be preview, keep or discard".into())),
}
}
/// `POST /api/projects/:id/gps/rescan_all` — queue EVERY already-scanned video in the
/// project for a fresh scan (admin). Needed once after upgrading: pre-upgrade scans
/// have no absolute timestamps, so stitching/data-quality stats only exist after a
/// re-scan. The serial sweep works through them in the background.
pub async fn rescan_all(State(s): State<AppState>, user: AuthUser, AxPath(id): AxPath<i32>) -> ApiResult<Value> {
user.require_admin()?;
let n = sqlx::query(
"UPDATE videos SET gps_status='pending', gps_error=NULL WHERE project_id=$1 AND gps_status <> 'pending'",
)
.bind(id)
.execute(&s.db)
.await
.map_err(err)?
.rows_affected();
crate::admin::audit(&s.db, &user.username, "gps_rescan_all", json!({ "project_id": id, "queued": n })).await;
Ok(Json(json!({ "queued": n })))
}
#[cfg(test)]
mod tests {
use super::*;
/// A straight east-west track at the equator: 1 pt/s, ~11.1 m apart (40 km/h).
fn straight(n: usize) -> Vec<Pt> {
(0..n).map(|i| [0.0, i as f64 * 0.0001, i as f64]).collect()
}
#[test]
fn smoother_removes_a_jump() {
let mut t = straight(60);
// one wild point 0.01° (~1.1 km) off the line
t[30][0] = 0.01;
let (fixed, replaced) = viterbi_smooth(&t);
assert_eq!(replaced, 1);
assert!(fixed[30][0].abs() < 1e-6, "outlier not re-interpolated: {}", fixed[30][0]);
// everything else untouched
assert_eq!(fixed[29], t[29]);
assert_eq!(fixed[31], t[31]);
}
#[test]
fn smoother_keeps_a_clean_track() {
let t = straight(60);
let (fixed, replaced) = viterbi_smooth(&t);
assert_eq!(replaced, 0);
assert_eq!(fixed, t);
}
#[test]
fn smoother_removes_a_burst() {
let mut t = straight(80);
for k in 40..44 {
t[k][0] = 0.02; // 2.2 km off for 4 consecutive samples
}
let (fixed, replaced) = viterbi_smooth(&t);
assert_eq!(replaced, 4);
for k in 40..44 {
assert!(fixed[k][0].abs() < 1e-6);
}
}
#[test]
fn downsample_caps_and_keeps_last() {
let t = straight(12000);
let d = downsample(t.clone(), 5000);
assert!(d.len() <= 5001);
assert_eq!(d.last(), t.last());
}
#[test]
fn stitch_planning_rules() {
// Healthy 1 Hz rollover (1 s gap) → nothing missing.
assert_eq!(plan_stitch(1.0, 299.0, Some(300.0), 20.0), None);
// 9 s gap, duration unknown → shift 8 s.
assert_eq!(plan_stitch(9.0, 291.0, None, 100.0), Some(8.0));
// 9 s gap but the track already covers the whole file → the PREDECESSOR was
// missing its end; shifting us would misalign every frame.
assert_eq!(plan_stitch(9.0, 299.0, Some(300.0), 100.0), None);
// Duration caps the shift: 20 s gap but only 5 s missing from this file.
assert_eq!(plan_stitch(20.0, 295.0, Some(300.0), 100.0), Some(5.0));
// Teleport guard: 5 km jump in 9 s can't be the same vehicle.
assert_eq!(plan_stitch(9.0, 291.0, None, 5000.0), None);
// Different session: 10 minutes apart.
assert_eq!(plan_stitch(600.0, 291.0, None, 100.0), None);
}
#[test]
fn wgs84_distance_matches_reference_arcs() {
// Textbook WGS84 arc lengths: 1° latitude ≈ 110.574 km (equator) /
// 111.132 km (45°); 1° longitude at the equator ≈ 111.320 km.
assert!((dist_m(0.0, 0.0, 1.0, 0.0) - 110_574.0).abs() < 200.0);
assert!((dist_m(0.0, 0.0, 0.0, 1.0) - 111_320.0).abs() < 30.0);
assert!((dist_m(44.5, 0.0, 45.5, 0.0) - 111_132.0).abs() < 200.0);
}
#[test]
fn track_roundtrip() {
let t = straight(10);
let v = track_value(&t);
let back = parse_track(&v);
assert_eq!(back.len(), 10);
assert!((back[9][1] - t[9][1]).abs() < 1e-5);
}
}

335
server/src/main.rs
View File

@@ -1,5 +1,7 @@
mod admin;
mod auth;
mod chainage;
mod gps;
mod ingest;
mod models;
mod nfs;
@@ -41,6 +43,17 @@ const FOLDER_FILTER: &str = " AND (NOT EXISTS (SELECT 1 FROM project_folders pf
OR (CASE WHEN rel_path LIKE '%/%' THEN regexp_replace(rel_path, '/[^/]*$', '') ELSE '' END) \
IN (SELECT folder FROM project_folders WHERE project_id = $1)) ";
/// SQL fragment: the derived lifecycle label for the UI badge. References the
/// unaliased `videos` table; shared by the video list and the map endpoint.
const WORKFLOW_STATUS_SQL: &str = "CASE
WHEN videos.claimed_by IS NOT NULL AND videos.lease_expires_at > now() THEN 'in-progress'
WHEN videos.status = 'verified' THEN
CASE WHEN (SELECT count(*) FROM video_events e WHERE e.video_id = videos.id AND e.event='push') >= 2
THEN 're-verified' ELSE 'verified' END
WHEN videos.assigned_to IS NOT NULL AND videos.assigned_to <> '' THEN 'assigned'
ELSE videos.status
END";
#[derive(Clone)]
pub struct AppState {
pub db: PgPool,
@@ -117,6 +130,10 @@ async fn main() -> anyhow::Result<()> {
let state = AppState { db, ingest_dir, admin_token, lease_secs, nfs_host, nfs_opts };
// GPS pipeline: serial background sweep extracting each pending video's track
// with exiftool (one at a time — never hammers the NAS).
tokio::spawn(gps::sweep(state.clone()));
// Phase 3: background task that auto-releases expired claims back to the pool.
{
let db = state.db.clone();
@@ -180,6 +197,19 @@ async fn main() -> anyhow::Result<()> {
.route("/api/projects/:id/members/:username", delete(admin::remove_member))
.route("/api/projects/:id/timer", post(project_timer))
.route("/api/projects/:id/folders", get(list_folders).post(set_folders))
// Map view + GPS pipeline + road_type + chainage.
.route("/api/projects/:id/map", get(map_data))
.route("/api/projects/:id/tracks.kml", get(export_tracks_kml))
.route("/api/projects/:id/road_types", get(list_road_types).post(set_road_types))
.route("/api/projects/:id/road_type_assign", post(assign_road_type))
.route("/api/videos/:id/gps/rescan", post(gps::rescan))
.route("/api/projects/:id/gps/rescan_all", post(gps::rescan_all))
.route("/api/videos/:id/gps/correct", post(gps::correct))
.route("/api/projects/:id/chainage/points", get(chainage::list_points).post(chainage::add_point))
.route("/api/projects/:id/chainage/points/bulk", post(chainage::add_points_bulk))
.route("/api/chainage/points/:id", delete(chainage::delete_point))
.route("/api/projects/:id/chainage/recompute", post(chainage::recompute))
.route("/api/projects/:id/chainage/quick", post(chainage::quick_assign))
// Phase 3 admin (token-gated).
.route("/api/admin/storage", get(storage_info))
.route("/api/users", get(admin::list_users).post(admin::create_user))
@@ -387,6 +417,247 @@ async fn set_folders(
Ok(Json(json!({ "folders": req.folders.len() })))
}
/// Max polyline points per video in the map payload. Full-resolution tracks stay in
/// the DB for chainage math; the map only needs the shape.
const MAP_MAX_PTS: usize = 400;
/// `GET /api/projects/:id/map` — everything the map needs per video: GPS track
/// (corrected preferred), status/workflow labels, road_type + chainage. Videos with
/// `gps_status` none/error feed the "videos without metadata" panel. Folder-aware.
async fn map_data(State(s): State<AppState>, user: AuthUser, AxPath(id): AxPath<i32>) -> ApiResult<Vec<MapVideo>> {
require_project_access(&s, &user, id).await?;
let q = format!(
r#"SELECT id, file_name, rel_path, status, {WORKFLOW_STATUS_SQL} AS workflow_status,
ignored, hand_raised, assigned_to, claimed_by, annotation_count,
road_type, gps_status, gps_len_m, gps_error,
chainage_start_m, chainage_end_m, sr_chainage_start_m, sr_chainage_end_m, route_seq,
COALESCE((SELECT count(*) FROM video_remarks r WHERE r.video_id = videos.id), 0) AS remark_count,
COALESCE(gps_track_corrected, gps_track) AS track,
(gps_track_corrected IS NOT NULL) AS corrected,
gps_corrected_by, gps_corrected_note, gps_sample_count, gps_max_gap_s
FROM videos WHERE project_id=$1 {FOLDER_FILTER} ORDER BY file_name"#
);
let mut rows = sqlx::query_as::<_, MapVideo>(&q)
.bind(id)
.fetch_all(&s.db)
.await
.map_err(err)?;
// Downsample tracks for transport — hundreds of videos × 5k points would be tens
// of MB; ~400 points per video keeps the payload in the low single-digit MB.
for r in rows.iter_mut() {
if let Some(t) = r.track.take() {
let pts = gps::parse_track(&t);
r.track = Some(gps::track_value(&gps::downsample(pts, MAP_MAX_PTS)));
}
}
Ok(Json(rows))
}
/// `GET /api/projects/:id/road_types` — the project's road-type labels + how many
/// videos carry each (admin or member).
async fn list_road_types(State(s): State<AppState>, user: AuthUser, AxPath(id): AxPath<i32>) -> ApiResult<Vec<RoadTypeRow>> {
require_project_access(&s, &user, id).await?;
let rows = sqlx::query_as::<_, RoadTypeRow>(
r#"SELECT rt.name, COALESCE(c.cnt, 0) AS video_count
FROM project_road_types rt
LEFT JOIN (SELECT road_type, count(*)::bigint AS cnt
FROM videos WHERE project_id=$1 GROUP BY road_type) c
ON c.road_type = rt.name
WHERE rt.project_id=$1 ORDER BY rt.name"#,
)
.bind(id)
.fetch_all(&s.db)
.await
.map_err(err)?;
Ok(Json(rows))
}
/// `POST /api/projects/:id/road_types {names}` — replace the label set (admin).
/// Videos keep whatever label they already carry; removing a name never rewrites videos.
async fn set_road_types(
State(s): State<AppState>,
user: AuthUser,
AxPath(id): AxPath<i32>,
Json(req): Json<RoadTypesRequest>,
) -> ApiResult<serde_json::Value> {
user.require_admin()?;
let names: Vec<String> = req
.names
.iter()
.map(|n| n.trim().to_string())
.filter(|n| !n.is_empty())
.collect();
let mut tx = s.db.begin().await.map_err(err)?;
sqlx::query("DELETE FROM project_road_types WHERE project_id=$1")
.bind(id)
.execute(&mut *tx)
.await
.map_err(err)?;
for n in &names {
sqlx::query("INSERT INTO project_road_types (project_id, name) VALUES ($1,$2) ON CONFLICT DO NOTHING")
.bind(id)
.bind(n)
.execute(&mut *tx)
.await
.map_err(err)?;
}
tx.commit().await.map_err(err)?;
admin::audit(&s.db, &user.username, "road_types_updated", json!({ "project_id": id, "count": names.len() })).await;
Ok(Json(json!({ "road_types": names.len() })))
}
/// `POST /api/projects/:id/road_type_assign {video_ids, road_type}` — bulk-assign a
/// road type to the selected videos (admin; empty road_type clears it).
async fn assign_road_type(
State(s): State<AppState>,
user: AuthUser,
AxPath(id): AxPath<i32>,
Json(req): Json<RoadTypeAssignRequest>,
) -> ApiResult<serde_json::Value> {
user.require_admin()?;
if req.video_ids.is_empty() {
return Err((StatusCode::BAD_REQUEST, "no videos selected".into()));
}
let rt = req.road_type.trim().to_string();
if !rt.is_empty() {
let known: Option<i32> =
sqlx::query_scalar("SELECT 1 FROM project_road_types WHERE project_id=$1 AND name=$2")
.bind(id)
.bind(&rt)
.fetch_optional(&s.db)
.await
.map_err(err)?;
if known.is_none() {
return Err((StatusCode::BAD_REQUEST, format!("'{rt}' is not a road type of this project")));
}
}
let n = sqlx::query("UPDATE videos SET road_type=$3 WHERE project_id=$1 AND id = ANY($2)")
.bind(id)
.bind(&req.video_ids)
.bind(&rt)
.execute(&s.db)
.await
.map_err(err)?
.rows_affected();
admin::audit(&s.db, &user.username, "road_type_assigned",
json!({ "project_id": id, "road_type": rt, "count": n })).await;
Ok(Json(json!({ "assigned": n, "road_type": rt })))
}
fn xml_escape(s: &str) -> String {
s.replace('&', "&amp;").replace('<', "&lt;").replace('>', "&gt;").replace('"', "&quot;")
}
/// KML line color (aabbggrr) per workflow status — same buckets as the map view.
fn kml_status_color(status: &str) -> &'static str {
match status {
"pending" => "ff5252ff",
"annotated" => "ff23a6f5",
"assigned" => "ffff8cb4",
"in-progress" => "ffffa86e",
"verified" => "ff75cf5c",
"re-verified" => "ffa8bf2b",
_ => "ffae958f", // ignored / unknown → gray
}
}
const KML_STATUSES: &[&str] =
&["pending", "annotated", "assigned", "in-progress", "verified", "re-verified", "ignored"];
/// `GET /api/projects/:id/tracks.kml?token=` — every scanned video's track as a KML
/// LineString colored by status (+ the calibration points), for Google Earth /
/// client hand-off. Auth via `?token=` (file download).
async fn export_tracks_kml(
State(s): State<AppState>,
Query(q): Query<TokenQuery>,
AxPath(id): AxPath<i32>,
) -> Result<Response, (StatusCode, String)> {
if auth::validate_token(&s, q.token.as_deref().unwrap_or("")).await.is_none() {
return Err((StatusCode::UNAUTHORIZED, "missing or invalid token".into()));
}
let vq = format!(
r#"SELECT file_name, {WORKFLOW_STATUS_SQL} AS workflow_status, ignored, road_type,
chainage_start_m, chainage_end_m, COALESCE(gps_track_corrected, gps_track) AS track
FROM videos WHERE project_id=$1 AND COALESCE(gps_track_corrected, gps_track) IS NOT NULL
{FOLDER_FILTER} ORDER BY file_name"#
);
#[allow(clippy::type_complexity)]
let rows: Vec<(String, String, bool, String, Option<f64>, Option<f64>, serde_json::Value)> =
sqlx::query_as(&vq).bind(id).fetch_all(&s.db).await.map_err(err)?;
let points: Vec<(f64, f64, f64, String, String)> = sqlx::query_as(
"SELECT lat, lon, chainage_m, note, road_type FROM chainage_points WHERE project_id=$1 ORDER BY chainage_m",
)
.bind(id)
.fetch_all(&s.db)
.await
.map_err(err)?;
let mut kml = String::with_capacity(1 << 20);
kml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"><Document>\n");
kml.push_str(&format!("<name>project {id} — video tracks</name>\n"));
for st in KML_STATUSES {
kml.push_str(&format!(
"<Style id=\"st-{st}\"><LineStyle><color>{}</color><width>4</width></LineStyle></Style>\n",
kml_status_color(st)
));
}
kml.push_str("<Folder><name>Videos</name>\n");
for (file_name, workflow_status, ignored, road_type, ch_s, ch_e, track) in rows {
let status = if ignored { "ignored".to_string() } else { workflow_status };
let pts = gps::downsample(gps::parse_track(&track), 1000);
if pts.len() < 2 {
continue;
}
let mut desc = format!("status: {status}");
if !road_type.is_empty() {
desc.push_str(&format!(" · road: {road_type}"));
}
if let (Some(a), Some(b)) = (ch_s, ch_e) {
desc.push_str(&format!(" · chainage {:.3} → {:.3} km", a / 1000.0, b / 1000.0));
}
kml.push_str(&format!(
"<Placemark><name>{}</name><description>{}</description><styleUrl>#st-{}</styleUrl><LineString><tessellate>1</tessellate><coordinates>",
xml_escape(&file_name), xml_escape(&desc), status
));
for p in &pts {
kml.push_str(&format!("{},{},0 ", p[1], p[0]));
}
kml.push_str("</coordinates></LineString></Placemark>\n");
}
kml.push_str("</Folder>\n<Folder><name>Calibration points</name>\n");
for (lat, lon, chainage_m, note, road_type) in points {
let name = format!("{:.3} km{}", chainage_m / 1000.0, if road_type.is_empty() { String::new() } else { format!(" ({road_type})") });
kml.push_str(&format!(
"<Placemark><name>{}</name><description>{}</description><Point><coordinates>{lon},{lat},0</coordinates></Point></Placemark>\n",
xml_escape(&name), xml_escape(&note)
));
}
kml.push_str("</Folder>\n</Document></kml>\n");
Response::builder()
.header(header::CONTENT_TYPE, "application/vnd.google-earth.kml+xml")
.header(header::CONTENT_DISPOSITION, format!("attachment; filename=\"project_{id}_tracks.kml\""))
.body(Body::from(kml))
.map_err(err)
}
/// Server-authoritative wire-format metadata: inject/overwrite `road_type` + chainage
/// keys at the top level of an export doc. Applied on claim (pull), push (store) and
/// both export endpoints, so a worker's old client can never drop them.
pub fn stamp_road_meta(
doc: &mut serde_json::Value,
road_type: &str,
chainage: (Option<f64>, Option<f64>, Option<f64>, Option<f64>),
) {
if let Some(obj) = doc.as_object_mut() {
obj.insert("road_type".into(), json!(road_type));
obj.insert("chainage_start_m".into(), json!(chainage.0));
obj.insert("chainage_end_m".into(), json!(chainage.1));
obj.insert("sr_chainage_start_m".into(), json!(chainage.2));
obj.insert("sr_chainage_end_m".into(), json!(chainage.3));
}
}
async fn connect_with_retry(url: &str, attempts: u32) -> anyhow::Result<PgPool> {
let mut last = None;
for i in 1..=attempts {
@@ -538,14 +809,9 @@ async fn list_videos(State(s): State<AppState>, user: AuthUser, AxPath(id): AxPa
WHERE video_id = videos.id AND event='push' GROUP BY username) s), '') AS verifiers,
assigned_to,
-- derived lifecycle label for the UI badge
CASE
WHEN videos.claimed_by IS NOT NULL AND videos.lease_expires_at > now() THEN 'in-progress'
WHEN videos.status = 'verified' THEN
CASE WHEN (SELECT count(*) FROM video_events e WHERE e.video_id = videos.id AND e.event='push') >= 2
THEN 're-verified' ELSE 'verified' END
WHEN videos.assigned_to IS NOT NULL AND videos.assigned_to <> '' THEN 'assigned'
ELSE videos.status
END AS workflow_status,
{WORKFLOW_STATUS_SQL} AS workflow_status,
road_type, gps_status, gps_len_m, gps_error, gps_sample_count, gps_max_gap_s, duration_s,
chainage_start_m, chainage_end_m, sr_chainage_start_m, sr_chainage_end_m, route_seq,
hand_raised, hand_raised_by, hand_raised_at, ignored, ignored_by,
COALESCE((SELECT json_agg(json_build_object('username', r.username, 'body', r.body, 'created_at', r.created_at)
ORDER BY r.created_at)
@@ -598,19 +864,29 @@ async fn export_video(State(s): State<AppState>, Query(q): Query<TokenQuery>, Ax
if auth::validate_token(&s, q.token.as_deref().unwrap_or("")).await.is_none() {
return Err((StatusCode::UNAUTHORIZED, "missing or invalid token".into()));
}
let row: Option<(String, String, Option<serde_json::Value>)> =
sqlx::query_as("SELECT file_name, rel_path, raw_json FROM videos WHERE id=$1")
#[allow(clippy::type_complexity)]
let row: Option<(String, String, Option<serde_json::Value>, String, Option<f64>, Option<f64>, Option<f64>, Option<f64>)> =
sqlx::query_as(
r#"SELECT file_name, rel_path, raw_json, road_type,
chainage_start_m, chainage_end_m, sr_chainage_start_m, sr_chainage_end_m
FROM videos WHERE id=$1"#,
)
.bind(id)
.fetch_optional(&s.db)
.await
.map_err(err)?;
let (file_name, rel_path, raw_json) = row.ok_or((StatusCode::NOT_FOUND, "video not found".to_string()))?;
let (file_name, rel_path, raw_json, road_type, ch_s, ch_e, sr_s, sr_e) =
row.ok_or((StatusCode::NOT_FOUND, "video not found".to_string()))?;
// Embed the video identity inside the doc so the file's name/path travel with it.
let doc = ensure_video_block(
let mut doc = ensure_video_block(
raw_json.unwrap_or_else(|| json!({ "fixed_annotations": [], "range_annotations": [] })),
&file_name,
&rel_path,
);
stamp_road_meta(&mut doc, &road_type, (ch_s, ch_e, sr_s, sr_e));
if let Some(geo) = chainage::video_geo(&s.db, id).await {
chainage::stamp_annotation_geo(&mut doc, &geo);
}
let body = serde_json::to_vec_pretty(&doc).map_err(err)?;
let stem = file_name.rsplit_once('.').map(|(a, _)| a).unwrap_or(&file_name);
let fname = format!("{stem}_annotations.json");
@@ -627,26 +903,31 @@ async fn export_project(State(s): State<AppState>, Query(q): Query<TokenQuery>,
if auth::validate_token(&s, q.token.as_deref().unwrap_or("")).await.is_none() {
return Err((StatusCode::UNAUTHORIZED, "missing or invalid token".into()));
}
let rows: Vec<(i32, String, String, Option<serde_json::Value>)> = sqlx::query_as(
"SELECT id, file_name, rel_path, raw_json FROM videos WHERE project_id=$1 AND raw_json IS NOT NULL ORDER BY rel_path",
#[allow(clippy::type_complexity)]
let rows: Vec<(i32, String, String, Option<serde_json::Value>, String, Option<f64>, Option<f64>, Option<f64>, Option<f64>)> = sqlx::query_as(
r#"SELECT id, file_name, rel_path, raw_json, road_type,
chainage_start_m, chainage_end_m, sr_chainage_start_m, sr_chainage_end_m
FROM videos WHERE project_id=$1 AND raw_json IS NOT NULL ORDER BY rel_path"#,
)
.bind(id)
.fetch_all(&s.db)
.await
.map_err(err)?;
let videos: Vec<ExportedVideo> = rows
.into_iter()
.map(|(vid, fname, rel, raw)| {
// file_name first (struct order) AND embedded in the annotations doc, so the
// name is both immediately visible per entry and self-contained with the data.
let annotations = ensure_video_block(
raw.unwrap_or_else(|| json!({ "fixed_annotations": [], "range_annotations": [] })),
&fname,
&rel,
);
ExportedVideo { file_name: fname, rel_path: rel, video_id: vid, annotations }
})
.collect();
let mut videos: Vec<ExportedVideo> = Vec::with_capacity(rows.len());
for (vid, fname, rel, raw, road_type, ch_s, ch_e, sr_s, sr_e) in rows {
// file_name first (struct order) AND embedded in the annotations doc, so the
// name is both immediately visible per entry and self-contained with the data.
let mut annotations = ensure_video_block(
raw.unwrap_or_else(|| json!({ "fixed_annotations": [], "range_annotations": [] })),
&fname,
&rel,
);
stamp_road_meta(&mut annotations, &road_type, (ch_s, ch_e, sr_s, sr_e));
if let Some(geo) = chainage::video_geo(&s.db, vid).await {
chainage::stamp_annotation_geo(&mut annotations, &geo);
}
videos.push(ExportedVideo { file_name: fname, rel_path: rel, video_id: vid, annotations });
}
let doc = json!({ "project_id": id, "video_count": videos.len(), "videos": videos });
let body = serde_json::to_vec_pretty(&doc).map_err(err)?;
Response::builder()

73
server/src/models.rs
View File

@@ -148,6 +148,24 @@ pub struct VideoRow {
// pending | annotated | assigned | in-progress | verified | re-verified.
pub assigned_to: Option<String>,
pub workflow_status: String,
// Road type label ('' = unassigned) — injected into the export/claim JSON.
pub road_type: String,
// GPS pipeline: pending | ok | none | error (+ measured track length / last error).
pub gps_status: String,
pub gps_len_m: Option<f64>,
pub gps_error: Option<String>,
// Data quality: raw fix count + biggest dropout — the table compares the count
// against the video duration (≈1 fix/s) and flags mismatches.
pub gps_sample_count: Option<i32>,
pub gps_max_gap_s: Option<f64>,
// Real container duration from the scan (raw NAS videos have no fps/frame_count).
pub duration_s: Option<f64>,
// Client-calibrated chainage + the immutable GPS-measured (sr) chainage, meters.
pub chainage_start_m: Option<f64>,
pub chainage_end_m: Option<f64>,
pub sr_chainage_start_m: Option<f64>,
pub sr_chainage_end_m: Option<f64>,
pub route_seq: Option<i32>,
// "Raise hand" signal — anyone may raise (to start a conversation) or lower it.
pub hand_raised: bool,
pub hand_raised_by: Option<String>,
@@ -163,6 +181,61 @@ pub struct VideoRow {
pub passes: serde_json::Value,
}
/// One row of `GET /api/projects/:id/map` — a video as the map sees it. `track` is
/// the (transport-downsampled) effective polyline: corrected when kept, else raw;
/// None until the GPS sweep has scanned the video (or when it has no metadata).
#[derive(Serialize, sqlx::FromRow)]
pub struct MapVideo {
pub id: i32,
pub file_name: String,
pub rel_path: String,
pub status: String,
pub workflow_status: String,
pub ignored: bool,
pub hand_raised: bool,
pub assigned_to: Option<String>,
pub claimed_by: Option<String>,
pub annotation_count: i32,
pub road_type: String,
pub gps_status: String,
pub gps_len_m: Option<f64>,
pub gps_error: Option<String>,
pub chainage_start_m: Option<f64>,
pub chainage_end_m: Option<f64>,
pub sr_chainage_start_m: Option<f64>,
pub sr_chainage_end_m: Option<f64>,
pub route_seq: Option<i32>,
pub remark_count: i64,
pub track: Option<Value>,
pub corrected: bool,
// Who/what produced the correction ('auto' or a username) + its summary note.
pub gps_corrected_by: Option<String>,
pub gps_corrected_note: Option<String>,
// Data quality from the raw scan: fix count + biggest dropout between fixes.
pub gps_sample_count: Option<i32>,
pub gps_max_gap_s: Option<f64>,
}
/// `GET /api/projects/:id/road_types` — a label + how many videos carry it.
#[derive(Serialize, sqlx::FromRow)]
pub struct RoadTypeRow {
pub name: String,
pub video_count: i64,
}
/// `POST /api/projects/:id/road_types {names}` — replace the label set (admin).
#[derive(Deserialize)]
pub struct RoadTypesRequest {
pub names: Vec<String>,
}
/// `POST /api/projects/:id/road_type_assign {video_ids, road_type}` — bulk-assign.
#[derive(Deserialize)]
pub struct RoadTypeAssignRequest {
pub video_ids: Vec<i32>,
pub road_type: String, // '' = clear
}
/// `POST /api/videos/:id/hand {raised}` — raise/lower the hand on a video.
#[derive(Deserialize)]
pub struct HandRequest {

65
server/src/worker.rs
View File

@@ -52,14 +52,16 @@ pub async fn claim(
user: AuthUser,
AxPath(id): AxPath<i32>,
) -> ApiResult<ClaimResponse> {
let row: Option<(i32, String, String, Option<i32>, Option<i32>, Option<f64>, Option<i32>, chrono::DateTime<chrono::Utc>, Option<Value>)> =
#[allow(clippy::type_complexity)]
let row: Option<(i32, String, String, Option<i32>, Option<i32>, Option<f64>, Option<i32>, chrono::DateTime<chrono::Utc>, Option<Value>, String, Option<f64>, Option<f64>, Option<f64>, Option<f64>)> =
sqlx::query_as(
r#"
UPDATE videos
SET claimed_by=$2, claimed_at=now(), lease_expires_at=now() + ($3::int * interval '1 second')
WHERE id=$1
AND (claimed_by IS NULL OR claimed_by=$2 OR lease_expires_at < now())
RETURNING id, file_name, rel_path, width, height, fps, frame_count, lease_expires_at, raw_json
RETURNING id, file_name, rel_path, width, height, fps, frame_count, lease_expires_at, raw_json,
road_type, chainage_start_m, chainage_end_m, sr_chainage_start_m, sr_chainage_end_m
"#,
)
.bind(id)
@@ -69,7 +71,8 @@ pub async fn claim(
.await
.map_err(err)?;
let Some((video_id, file_name, rel_path, width, height, fps, frame_count, lease_expires_at, raw_json)) = row
let Some((video_id, file_name, rel_path, width, height, fps, frame_count, lease_expires_at, raw_json,
road_type, ch_s, ch_e, sr_s, sr_e)) = row
else {
// Distinguish "doesn't exist" from "not claimable".
let exists: Option<i32> = sqlx::query_scalar("SELECT id FROM videos WHERE id=$1")
@@ -88,6 +91,21 @@ pub async fn claim(
log_event(&s.db, video_id, &user.username, "claim", json!({})).await;
// Inject the server-authoritative road_type + chainage into the doc the worker
// pulls (only when a doc exists — a pending video legitimately has none; push
// re-stamps regardless, so the metadata always persists). Each annotation also
// gets its frame-exact lat/lon + chainage.
let annotations = match raw_json {
Some(mut doc) => {
crate::stamp_road_meta(&mut doc, &road_type, (ch_s, ch_e, sr_s, sr_e));
if let Some(geo) = crate::chainage::video_geo(&s.db, video_id).await {
crate::chainage::stamp_annotation_geo(&mut doc, &geo);
}
doc
}
None => Value::Null,
};
Ok(Json(ClaimResponse {
video_id,
file_name,
@@ -98,7 +116,7 @@ pub async fn claim(
frame_count,
lease_expires_at,
download_url: format!("/api/videos/{video_id}/download"),
annotations: raw_json.unwrap_or(Value::Null),
annotations,
}))
}
@@ -223,13 +241,18 @@ pub async fn push(
// lease lapsed (auto-released, or re-claimed by someone else) the worker must
// re-claim before pushing — their local work is never lost, it just can't clobber a
// claim they no longer hold. Admins may always push.
let row: Option<(Option<String>, Option<chrono::DateTime<chrono::Utc>>)> =
sqlx::query_as("SELECT claimed_by, lease_expires_at FROM videos WHERE id=$1")
#[allow(clippy::type_complexity)]
let row: Option<(Option<String>, Option<chrono::DateTime<chrono::Utc>>, String, Option<f64>, Option<f64>, Option<f64>, Option<f64>)> =
sqlx::query_as(
r#"SELECT claimed_by, lease_expires_at, road_type,
chainage_start_m, chainage_end_m, sr_chainage_start_m, sr_chainage_end_m
FROM videos WHERE id=$1"#,
)
.bind(id)
.fetch_optional(&s.db)
.await
.map_err(err)?;
let (claimed_by, lease_expires_at) =
let (claimed_by, lease_expires_at, road_type, ch_s, ch_e, sr_s, sr_e) =
row.ok_or((StatusCode::NOT_FOUND, "video not found".to_string()))?;
if !user.is_admin() {
let holds_active_claim = claimed_by.as_deref() == Some(user.username.as_str())
@@ -250,6 +273,34 @@ pub async fn push(
let doc: ExportDoc = serde_json::from_value(body.clone())
.map_err(|e| (StatusCode::BAD_REQUEST, format!("invalid export doc: {e}")))?;
// Adopt the doc's fps/frame_count/resolution BEFORE stamping. A raw (from-scratch)
// NAS video has no fps in our DB — but the desktop decoded the real file, so its
// fps is authoritative. Without this, per-annotation lat/lon/chainage can't be
// computed (frame→time needs fps) and the geo silently wouldn't be written.
// COALESCE(doc, existing) so a doc that omits a field never wipes it.
if let Some(v) = doc.video.as_ref() {
let _ = sqlx::query(
r#"UPDATE videos SET fps=COALESCE($2, fps), frame_count=COALESCE($3, frame_count),
width=COALESCE($4, width), height=COALESCE($5, height) WHERE id=$1"#,
)
.bind(id)
.bind(v.fps)
.bind(v.frame_count.map(|x| x as i32))
.bind(v.width.map(|x| x as i32))
.bind(v.height.map(|x| x as i32))
.execute(&s.db)
.await;
}
// Re-stamp the server-authoritative road_type + chainage onto the stored doc —
// whatever the worker's client sent (or dropped), the server's values win. Every
// annotation gets its frame-exact lat/lon + chainage too (from the fps just set).
let mut body = body;
crate::stamp_road_meta(&mut body, &road_type, (ch_s, ch_e, sr_s, sr_e));
if let Some(geo) = crate::chainage::video_geo(&s.db, id).await {
crate::chainage::stamp_annotation_geo(&mut body, &geo);
}
let ann_count = (doc.fixed_annotations.len() + doc.range_annotations.len()) as i32;
// Recompute the video-level annotator fields from the pushed doc (same idea as