Add project files

pipeline/bg_removal.py

@@ -0,0 +1,147 @@
+"""
+Background removal via rembg (BiRefNet lite), disk-cached by content hash
+so a re-upload of the same image never re-runs the model.
+"""
+
+import gc
+import hashlib
+import io
+import logging
+import os
+
+import cv2
+import numpy as np
+from PIL import Image
+from rembg import remove, new_session
+
+import config
+
+logger = logging.getLogger(__name__)
+
+_session = None
+
+
+def get_session():
+    """Lazily create the rembg session once and reuse it for every request."""
+    global _session
+    if _session is None:
+        try:
+            _session = new_session(config.REMBG_MODEL_NAME,
+                                    providers=["CUDAExecutionProvider", "CPUExecutionProvider"])
+        except Exception:
+            logger.exception(
+                "Failed to create rembg session with CUDAExecutionProvider, "
+                "falling back to CPU-only"
+            )
+            _session = new_session(config.REMBG_MODEL_NAME, providers=["CPUExecutionProvider"])
+
+        try:
+            providers = _session.inner_session.get_providers()
+            logger.info("rembg session ready, using providers: %s", providers)
+            if "CUDAExecutionProvider" not in providers:
+                logger.warning(
+                    "rembg is running on CPU (no CUDAExecutionProvider) -- "
+                    "background removal will be much slower. Check that "
+                    "onnxruntime-gpu is installed and the CUDA driver is visible."
+                )
+        except AttributeError:
+            pass
+    return _session
+
+
+def unload_session():
+    """Drops the rembg/onnxruntime session so its (possibly CUDA-backed)
+    memory is freed. It's a lazy singleton (see get_session above), so the
+    next call to remove_background_bytes/_file simply recreates it. Used to
+    make room for SAM-based flower counting on an 8GB card."""
+    global _session
+    freed = _session is not None
+    _session = None
+    if freed:
+        gc.collect()
+    return freed
+
+
+def resize_max_dim(pil_img: Image.Image, max_dim: int) -> Image.Image:
+    """Downscale in place-equivalent fashion so neither side exceeds max_dim.
+    No-op (returns the same image) if already within bounds -- this is the
+    single choke point that keeps memory/time bounded for every model in the
+    pipeline (bg removal, SIFT/ORB, SuperPoint, LoFTR), regardless of how
+    large the original upload was."""
+    w, h = pil_img.size
+    scale = max_dim / max(w, h)
+    if scale >= 1.0:
+        return pil_img
+    new_size = (max(1, int(w * scale)), max(1, int(h * scale)))
+    return pil_img.resize(new_size, Image.LANCZOS)
+
+
+def hash_bytes(data: bytes) -> str:
+    return hashlib.sha1(data).hexdigest()[:20]
+
+
+def remove_background_bytes(image_bytes: bytes, cache_dir: str):
+    """
+    Runs background removal on raw image bytes (an upload), caching the
+    result by content hash. Returns (rgba_bgra_ndarray, cache_key).
+    """
+    os.makedirs(cache_dir, exist_ok=True)
+    key = hash_bytes(image_bytes)
+    cache_path = os.path.join(cache_dir, key + ".png")
+
+    if os.path.exists(cache_path):
+        img = cv2.imread(cache_path, cv2.IMREAD_UNCHANGED)
+        if img is not None and img.ndim == 3 and img.shape[2] == 4:
+            return img, key
+
+    pil_img = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+    orig_size = pil_img.size
+    pil_img = resize_max_dim(pil_img, config.MAX_IMAGE_DIM)
+    if pil_img.size != orig_size:
+        logger.info("Resized upload %s -> %s before processing", orig_size, pil_img.size)
+
+    result = remove(pil_img, session=get_session())
+    result.save(cache_path)
+
+    rgba = cv2.cvtColor(np.array(result), cv2.COLOR_RGBA2BGRA)
+    return rgba, key
+
+
+def remove_background_file(src_path: str, cache_dir: str):
+    """Same as remove_background_bytes, but caches by original filename --
+    used for the fixed template set, which doesn't change between requests."""
+    os.makedirs(cache_dir, exist_ok=True)
+    base_name = os.path.splitext(os.path.basename(src_path))[0]
+    cache_path = os.path.join(cache_dir, base_name + ".png")
+
+    if os.path.exists(cache_path):
+        img = cv2.imread(cache_path, cv2.IMREAD_UNCHANGED)
+        if img is not None and img.ndim == 3 and img.shape[2] == 4:
+            return img
+
+    pil_img = Image.open(src_path).convert("RGB")
+    pil_img = resize_max_dim(pil_img, config.MAX_IMAGE_DIM)
+    result = remove(pil_img, session=get_session())
+    result.save(cache_path)
+
+    return cv2.cvtColor(np.array(result), cv2.COLOR_RGBA2BGRA)
+
+
+def create_mask(alpha):
+    _, mask = cv2.threshold(alpha, config.ALPHA_THRESHOLD, 255, cv2.THRESH_BINARY)
+    kernel = np.ones((3, 3), np.uint8)
+    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
+    mask = cv2.erode(mask, kernel, iterations=config.ERODE_ITER)
+    return mask
+
+
+def split_rgba(rgba):
+    """rgba: BGRA ndarray -> (bgr, alpha, mask)"""
+    if rgba.shape[2] == 4:
+        bgr = rgba[:, :, :3]
+        alpha = rgba[:, :, 3]
+    else:
+        bgr = rgba
+        alpha = np.ones(bgr.shape[:2], dtype=np.uint8) * 255
+    mask = create_mask(alpha)
+    return bgr, alpha, mask

pipeline/classical.py

@@ -0,0 +1,110 @@
+"""
+SIFT / ORB matching: one query image vs the fixed template set.
+Template keypoints/descriptors are precomputed once at startup and reused
+for every request -- only the query image is processed per request.
+"""
+
+import cv2
+import numpy as np
+
+import config
+
+_sift = None
+_orb = None
+_template_features = {"SIFT": None, "ORB": None}  # name -> {template_name: {"kp","des"}}
+
+
+def _get_sift():
+    global _sift
+    if _sift is None:
+        _sift = cv2.SIFT_create()
+    return _sift
+
+
+def _get_orb():
+    global _orb
+    if _orb is None:
+        _orb = cv2.ORB_create(nfeatures=config.ORB_N_FEATURES)
+    return _orb
+
+
+def _detector_for(method):
+    return _get_sift() if method == "SIFT" else _get_orb()
+
+
+def _matcher_for(method):
+    if method == "SIFT":
+        index_params = dict(algorithm=1, trees=5)  # FLANN_INDEX_KDTREE
+    else:
+        index_params = dict(algorithm=6, table_number=6, key_size=12,
+                             multi_probe_level=1)  # FLANN_INDEX_LSH
+    search_params = dict(checks=50)
+    return cv2.FlannBasedMatcher(index_params, search_params)
+
+
+def extract_features(method, bgr, mask):
+    gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
+    detector = _detector_for(method)
+    kp, des = detector.detectAndCompute(gray, mask)
+    return kp, des
+
+
+def set_template_features(method, template_name, kp, des):
+    if _template_features[method] is None:
+        _template_features[method] = {}
+    _template_features[method][template_name] = {"kp": kp, "des": des}
+
+
+def get_template_features(method):
+    return _template_features[method] or {}
+
+
+def match_pair(method, kp_q, des_q, kp_t, des_t):
+    """Returns (inlier_count, confidence_pct)."""
+    if des_q is None or des_t is None or len(kp_q) == 0 or len(kp_t) == 0:
+        return 0, 0.0
+
+    matcher = _matcher_for(method)
+    try:
+        raw_matches = matcher.knnMatch(des_q, des_t, k=2)
+    except cv2.error:
+        return 0, 0.0
+
+    good = []
+    for pair in raw_matches:
+        if len(pair) != 2:
+            continue
+        m, n = pair
+        if m.distance < config.LOWE_RATIO * n.distance:
+            good.append(m)
+
+    if len(good) < config.MIN_RAW_MATCHES:
+        return 0, 0.0
+
+    src = np.float32([kp_q[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
+    dst = np.float32([kp_t[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
+
+    _, ransac_mask = cv2.findHomography(src, dst, cv2.RANSAC, 5.0)
+    if ransac_mask is None:
+        return 0, 0.0
+
+    inlier_count = int(ransac_mask.sum())
+    confidence_pct = (inlier_count / len(good)) * 100 if good else 0.0
+    return inlier_count, confidence_pct
+
+
+def match_against_templates(method, kp_q, des_q):
+    """Returns a list of {"template": name, "score": int, "confidence": float},
+    sorted by score descending, for every precomputed template."""
+    results = []
+    for template_name, data in get_template_features(method).items():
+        inlier_count, confidence_pct = match_pair(
+            method, kp_q, des_q, data["kp"], data["des"]
+        )
+        results.append({
+            "template": template_name,
+            "score": inlier_count,
+            "confidence": round(confidence_pct, 1),
+        })
+    results.sort(key=lambda r: r["score"], reverse=True)
+    return results

pipeline/color.py

@@ -0,0 +1,148 @@
+"""
+Independent color-space comparison: how much of the uploaded photo's color
+distribution is also present in each template, plus which specific colors
+matched. Entirely separate from the SIFT/ORB/SuperGlue/LoFTR scores and the
+weighted final match -- nothing here feeds into those, it's its own section.
+
+Template-side color data (histogram + dominant colors) is precomputed once
+at startup from the same masked bgr/mask already produced during bootstrap,
+same principle as the other methods: a request only ever analyzes the one
+uploaded image.
+"""
+
+import cv2
+import numpy as np
+
+N_DOMINANT_COLORS = 5
+HIST_HUE_BINS = 30
+HIST_SAT_BINS = 32
+
+# k-means on every foreground pixel is wasteful and, given this app's
+# history of memory blowups, worth bounding explicitly -- a random sample
+# is statistically equivalent for "what are the dominant colors" purposes.
+MAX_KMEANS_SAMPLE = 20000
+
+# Empirical ceiling for CIE76 Lab distance beyond which two colors are
+# considered maximally dissimilar (similarity floors at 0%). ~100 comfortably
+# covers the largest perceptual differences (e.g. black vs. white is ~100).
+MAX_LAB_DISTANCE = 100.0
+
+_template_color_data = {}  # name -> {"hist": ndarray, "colors": [...]}
+
+
+def _masked_pixels_bgr(bgr, mask):
+    ys, xs = np.where(mask > 0)
+    if len(ys) == 0:
+        return np.zeros((0, 3), dtype=np.uint8)
+    return bgr[ys, xs]
+
+
+def compute_hs_histogram(bgr, mask):
+    """Hue+Saturation 2D histogram (Value/brightness deliberately excluded
+    so lighting differences between photos don't masquerade as color
+    differences), normalized so it sums to 1."""
+    hsv = cv2.cvtColor(bgr, cv2.COLOR_BGR2HSV)
+    hist = cv2.calcHist([hsv], [0, 1], mask, [HIST_HUE_BINS, HIST_SAT_BINS],
+                        [0, 180, 0, 256])
+    total = hist.sum()
+    if total > 0:
+        hist = hist / total
+    return hist
+
+
+def histogram_match_pct(hist_a, hist_b):
+    """% of one distribution's mass that the other also covers -- both
+    histograms are pre-normalized, so HISTCMP_INTERSECT directly sums
+    min(a[i], b[i]) into a 0-1 overlap fraction."""
+    intersection = cv2.compareHist(hist_a.astype(np.float32), hist_b.astype(np.float32),
+                                    cv2.HISTCMP_INTERSECT)
+    return round(float(intersection) * 100, 1)
+
+
+def dominant_colors(bgr, mask, k=N_DOMINANT_COLORS):
+    pixels = _masked_pixels_bgr(bgr, mask)
+    if len(pixels) == 0:
+        return []
+
+    if len(pixels) > MAX_KMEANS_SAMPLE:
+        idx = np.random.choice(len(pixels), MAX_KMEANS_SAMPLE, replace=False)
+        pixels = pixels[idx]
+
+    unique_count = len(np.unique(pixels, axis=0))
+    k_eff = min(k, unique_count)
+    if k_eff < 1:
+        return []
+
+    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 20, 0.5)
+    _, labels, centers = cv2.kmeans(pixels.astype(np.float32), k_eff, None,
+                                     criteria, 3, cv2.KMEANS_PP_CENTERS)
+
+    labels = labels.flatten()
+    counts = np.bincount(labels, minlength=k_eff)
+    total = counts.sum()
+
+    colors = []
+    for i in range(k_eff):
+        b, g, r = centers[i]
+        colors.append({
+            "rgb": [int(round(r)), int(round(g)), int(round(b))],
+            "pct": round(float(counts[i]) / total * 100, 1),
+        })
+    colors.sort(key=lambda c: c["pct"], reverse=True)
+    return colors
+
+
+def _lab_distance(rgb_a, rgb_b):
+    a = np.uint8([[[rgb_a[2], rgb_a[1], rgb_a[0]]]])  # rgb -> bgr for cv2
+    b = np.uint8([[[rgb_b[2], rgb_b[1], rgb_b[0]]]])
+    lab_a = cv2.cvtColor(a, cv2.COLOR_BGR2LAB)[0][0].astype(np.float32)
+    lab_b = cv2.cvtColor(b, cv2.COLOR_BGR2LAB)[0][0].astype(np.float32)
+    return float(np.linalg.norm(lab_a - lab_b))
+
+
+def match_color_pairs(input_colors, template_colors):
+    """For each of the input's dominant colors, find the closest template
+    dominant color by perceptual (Lab) distance and report a similarity %."""
+    pairs = []
+    for ic in input_colors:
+        if not template_colors:
+            break
+        best = min(template_colors, key=lambda tc: _lab_distance(ic["rgb"], tc["rgb"]))
+        dist = _lab_distance(ic["rgb"], best["rgb"])
+        similarity = max(0.0, 100.0 * (1 - dist / MAX_LAB_DISTANCE))
+        pairs.append({
+            "input_rgb": ic["rgb"],
+            "input_pct": ic["pct"],
+            "template_rgb": best["rgb"],
+            "template_pct": best["pct"],
+            "similarity": round(similarity, 1),
+        })
+    return pairs
+
+
+def set_template_color_data(name, bgr, mask):
+    _template_color_data[name] = {
+        "hist": compute_hs_histogram(bgr, mask),
+        "colors": dominant_colors(bgr, mask),
+    }
+
+
+def compare_input_to_templates(input_bgr, input_mask):
+    input_hist = compute_hs_histogram(input_bgr, input_mask)
+    input_colors = dominant_colors(input_bgr, input_mask)
+
+    results = []
+    for name, data in _template_color_data.items():
+        match_pct = histogram_match_pct(input_hist, data["hist"])
+        pairs = match_color_pairs(input_colors, data["colors"])
+        results.append({
+            "template": name,
+            "match_pct": match_pct,
+            "color_pairs": pairs,
+        })
+    results.sort(key=lambda r: r["match_pct"], reverse=True)
+
+    return {
+        "input_dominant_colors": input_colors,
+        "templates": results,
+    }

pipeline/color_grid.py

@@ -0,0 +1,194 @@
+"""
+Color family grid: dynamically discovers color regions within an image via
+K-means clustering in LAB space (adapted from a standalone dynamic-color-
+family script the user provided as a reference), then renders each region
+as its own tile in a grid -- keeping only that region's original pixels,
+everything else left dark -- and, given a second image, matches each region
+to its closest counterpart by average color and reports how much area they
+each cover.
+
+Purely visual/informational, like the AI verification section: nothing here
+feeds into any score.
+"""
+
+import logging
+import math
+
+import cv2
+import numpy as np
+from PIL import Image, ImageDraw
+
+import config
+
+logger = logging.getLogger(__name__)
+
+# Bounded the same way pipeline/color.py bounds its own k-means call --
+# clustering every foreground pixel is wasteful, a random sample is
+# statistically equivalent for finding cluster centers.
+MAX_KMEANS_SAMPLE = 20000
+
+
+def cluster_families(bgr, mask, k=None):
+    """K-means-clusters the masked (foreground) pixels into k color
+    families in LAB space. Returns a list sorted by area descending:
+    {"mask": HxW bool ndarray, "rgb": [r,g,b], "pct": float, "pixel_count": int}
+    """
+    k = k or config.FAMILY_GRID_K
+    ys, xs = np.where(mask > 0)
+    if len(ys) == 0:
+        return []
+
+    lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB).astype(np.float32)
+    fg_lab = lab[ys, xs]
+
+    # Down-weight lightness so light/dark shadows of the same hue land in
+    # the same family instead of splitting across families by brightness.
+    features = fg_lab.copy()
+    features[:, 0] *= config.FAMILY_GRID_LIGHTNESS_WEIGHT
+
+    if len(features) > MAX_KMEANS_SAMPLE:
+        sample_idx = np.random.choice(len(features), MAX_KMEANS_SAMPLE, replace=False)
+        sample = features[sample_idx]
+    else:
+        sample = features
+
+    k_eff = min(k, len(np.unique(sample, axis=0)))
+    if k_eff < 1:
+        return []
+
+    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 20, 0.5)
+    _, _, centers = cv2.kmeans(sample, k_eff, None, criteria, 3, cv2.KMEANS_PP_CENTERS)
+
+    # Assign EVERY foreground pixel (not just the sample) to its nearest
+    # center -- needed to build full-resolution family masks. Cheap: a
+    # handful of vectorized (N,) distance passes, one per cluster.
+    dists = np.stack([np.linalg.norm(features - c, axis=1) for c in centers], axis=1)
+    labels = np.argmin(dists, axis=1)
+
+    h, w = mask.shape
+    total = len(labels)
+    families = []
+    for i in range(k_eff):
+        member = labels == i
+        count = int(member.sum())
+        if count == 0:
+            continue
+        full_mask = np.zeros((h, w), dtype=bool)
+        full_mask[ys[member], xs[member]] = True
+        mean_bgr = bgr[ys[member], xs[member]].mean(axis=0)
+        families.append({
+            "mask": full_mask,
+            "rgb": [int(round(mean_bgr[2])), int(round(mean_bgr[1])), int(round(mean_bgr[0]))],
+            "pct": round(count / total * 100, 1),
+            "pixel_count": count,
+        })
+
+    families.sort(key=lambda f: f["pct"], reverse=True)
+    return families
+
+
+def _lab_distance_rgb(rgb_a, rgb_b):
+    a = np.uint8([[[rgb_a[2], rgb_a[1], rgb_a[0]]]])
+    b = np.uint8([[[rgb_b[2], rgb_b[1], rgb_b[0]]]])
+    lab_a = cv2.cvtColor(a, cv2.COLOR_BGR2LAB)[0][0].astype(np.float32)
+    lab_b = cv2.cvtColor(b, cv2.COLOR_BGR2LAB)[0][0].astype(np.float32)
+    return float(np.linalg.norm(lab_a - lab_b))
+
+
+def match_families(input_families, template_families):
+    """For each input family (largest first), finds its closest template
+    family by average color and reports an area-match %: the smaller area
+    share divided by the larger, so it reads 100% when both cover the same
+    proportion of their own (differently-sized) images, regardless of which
+    one is physically bigger."""
+    pairs = []
+    for rank, ifam in enumerate(input_families):
+        if not template_families:
+            break
+        tfam = min(template_families,
+                   key=lambda f: _lab_distance_rgb(ifam["rgb"], f["rgb"]))
+        color_dist = _lab_distance_rgb(ifam["rgb"], tfam["rgb"])
+        color_similarity = max(0.0, 100.0 * (1 - color_dist / 100.0))
+        bigger = max(ifam["pct"], tfam["pct"])
+        area_match = (min(ifam["pct"], tfam["pct"]) / bigger * 100) if bigger > 0 else 0.0
+        pairs.append({
+            "rank": rank + 1,
+            "input_rgb": ifam["rgb"],
+            "input_pct": ifam["pct"],
+            "template_rgb": tfam["rgb"],
+            "template_pct": tfam["pct"],
+            "area_match_pct": round(area_match, 1),
+            "color_similarity_pct": round(color_similarity, 1),
+        })
+    return pairs
+
+
+def overall_area_match(pairs):
+    """Single headline %: each pair's area-match weighted by how much of
+    the input image that family actually covers (so a good match on the
+    dominant color counts for more than a good match on a sliver)."""
+    if not pairs:
+        return None
+    total_weight = sum(p["input_pct"] for p in pairs)
+    if total_weight <= 0:
+        return None
+    weighted = sum(p["area_match_pct"] * p["input_pct"] for p in pairs)
+    return round(weighted / total_weight, 1)
+
+
+def render_family_grid(bgr, families):
+    """Renders each family as its own tile (original colors preserved,
+    everything else left dark) arranged in a grid with separator lines and
+    a rank/percentage label -- returns a BGR ndarray ready for cv2.imwrite.
+    Each tile is downscaled to FAMILY_GRID_TILE_MAX_DIM before being placed,
+    so the assembled canvas stays small by construction."""
+    if not families:
+        return bgr.copy()
+
+    h, w = bgr.shape[:2]
+    tile_max = config.FAMILY_GRID_TILE_MAX_DIM
+    scale = min(1.0, tile_max / max(h, w))
+    tw, th = max(1, int(w * scale)), max(1, int(h * scale))
+
+    rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
+    if scale < 1.0:
+        rgb_small = cv2.resize(rgb, (tw, th), interpolation=cv2.INTER_AREA)
+    else:
+        rgb_small = rgb
+
+    line_width = config.FAMILY_GRID_LINE_WIDTH
+    k = len(families)
+    cols = math.ceil(math.sqrt(k))
+    rows = math.ceil(k / cols)
+
+    grid_w = cols * tw + (cols + 1) * line_width
+    grid_h = rows * th + (rows + 1) * line_width
+
+    canvas = Image.new("RGB", (grid_w, grid_h), (24, 22, 19))
+    draw = ImageDraw.Draw(canvas)
+
+    for rank, fam in enumerate(families):
+        r_idx, c_idx = divmod(rank, cols)
+        x0 = line_width + c_idx * (tw + line_width)
+        y0 = line_width + r_idx * (th + line_width)
+
+        mask_small = fam["mask"]
+        if scale < 1.0:
+            mask_small = cv2.resize(mask_small.astype(np.uint8), (tw, th),
+                                     interpolation=cv2.INTER_NEAREST).astype(bool)
+
+        tile = np.zeros_like(rgb_small)
+        tile[mask_small] = rgb_small[mask_small]
+        canvas.paste(Image.fromarray(tile, mode="RGB"), (x0, y0))
+
+        draw.rectangle([x0, y0, x0 + tw - 1, y0 + th - 1],
+                        outline=(210, 200, 180), width=max(1, line_width // 2))
+
+        label = f"#{rank + 1}  {fam['pct']}%"
+        label_w = 9 * len(label) + 16
+        draw.rectangle([x0 + 6, y0 + 6, x0 + 6 + label_w, y0 + 26], fill=(20, 18, 16))
+        draw.rectangle([x0 + 10, y0 + 10, x0 + 22, y0 + 22],
+                        fill=tuple(fam["rgb"]), outline=(255, 255, 255))
+        draw.text((x0 + 28, y0 + 10), label, fill=(240, 235, 225))
+
+    return cv2.cvtColor(np.array(canvas), cv2.COLOR_RGB2BGR)

pipeline/deep.py

@@ -0,0 +1,235 @@
+"""
+Deep-learning matchers: SuperPoint+LightGlue ("SuperGlue" in the UI/output
+naming, per the existing convention -- LightGlue is the permissively
+licensed, actively maintained successor actually running under the hood)
+and LoFTR (dense, pairwise).
+
+Template-side features/tensors are precomputed once at startup and reused
+for every request; only the query image is processed per request. Since
+there are only a handful of templates, even LoFTR (which has no reusable
+per-image descriptor and must re-run a full forward pass per pair) is cheap
+here -- it does NOT scale the way it would against hundreds of images.
+"""
+
+import gc
+import logging
+
+import cv2
+import numpy as np
+import torch
+
+import config
+
+logger = logging.getLogger(__name__)
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+_superpoint = None
+_lightglue = None
+_loftr = None
+
+_template_superpoint_feats = {}   # template_name -> {"feats", "mask"}
+_template_loftr_tensors = {}       # template_name -> {"tensor", "mask"}
+
+
+def get_superpoint():
+    global _superpoint
+    if _superpoint is None:
+        logger.info("Loading SuperPoint on %s...", DEVICE)
+        from lightglue import SuperPoint
+        _superpoint = SuperPoint(
+            max_num_keypoints=config.SUPERPOINT_MAX_KEYPOINTS
+        ).eval().to(DEVICE)
+    return _superpoint
+
+
+def get_lightglue():
+    global _lightglue
+    if _lightglue is None:
+        logger.info("Loading LightGlue on %s...", DEVICE)
+        from lightglue import LightGlue
+        _lightglue = LightGlue(features="superpoint").eval().to(DEVICE)
+    return _lightglue
+
+
+def get_loftr():
+    global _loftr
+    if _loftr is None:
+        logger.info("Loading LoFTR on %s...", DEVICE)
+        import kornia.feature as KF
+        _loftr = KF.LoFTR(pretrained="outdoor").eval().to(DEVICE)
+    return _loftr
+
+
+def unload_models():
+    """Drops the SuperPoint/LightGlue/LoFTR model objects (not the small
+    per-template feature/tensor caches, which stay put) and frees their CUDA
+    memory. They're plain lazy singletons (see get_superpoint/get_lightglue/
+    get_loftr above), so the next call to any of those simply reloads --
+    exactly like a fresh process start. Used to make room for SAM-based
+    flower counting on an 8GB card that can't hold everything at once."""
+    global _superpoint, _lightglue, _loftr
+    freed = _superpoint is not None or _lightglue is not None or _loftr is not None
+    _superpoint = None
+    _lightglue = None
+    _loftr = None
+    if freed:
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    return freed
+
+
+def masked_gray_tensor(bgr, mask, max_dim=None, fill_value=0.5):
+    gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY).astype(np.float32) / 255.0
+
+    if max_dim is not None:
+        h, w = gray.shape
+        scale = max_dim / max(h, w)
+        if scale < 1.0:
+            new_w, new_h = int(w * scale), int(h * scale)
+            gray = cv2.resize(gray, (new_w, new_h), interpolation=cv2.INTER_AREA)
+            mask = cv2.resize(mask, (new_w, new_h), interpolation=cv2.INTER_NEAREST)
+
+    gray[mask == 0] = fill_value
+    tensor = torch.from_numpy(gray)[None, None].to(DEVICE)
+    return tensor, mask
+
+
+def keep_points_inside_mask(pts, mask):
+    if len(pts) == 0:
+        return np.zeros(0, dtype=bool)
+    h, w = mask.shape
+    xs = np.clip(pts[:, 0].round().astype(int), 0, w - 1)
+    ys = np.clip(pts[:, 1].round().astype(int), 0, h - 1)
+    return mask[ys, xs] > 0
+
+
+# ---------------------------------------------------------------
+# SuperPoint + LightGlue
+# ---------------------------------------------------------------
+
+def superpoint_extract(bgr, mask):
+    # ASSUMPTION: cap at MAX_IMAGE_DIM even though uploads are already
+    # resized upstream -- this is what used to run at full (sometimes
+    # multi-thousand-pixel) upload resolution and was the main driver of
+    # the multi-GB memory spikes that got the process OOM-killed.
+    tensor, resized_mask = masked_gray_tensor(bgr, mask, max_dim=config.MAX_IMAGE_DIM)
+    with torch.no_grad():
+        feats = get_superpoint().extract(tensor)
+    return feats, resized_mask
+
+
+def set_template_superpoint(template_name, bgr, mask):
+    feats, resized_mask = superpoint_extract(bgr, mask)
+    _template_superpoint_feats[template_name] = {"feats": feats, "mask": resized_mask}
+
+
+def _lightglue_pair(feats_q, feats_t, mask_q, mask_t):
+    from lightglue.utils import rbd
+
+    with torch.no_grad():
+        matches01 = get_lightglue()({"image0": feats_q, "image1": feats_t})
+
+    feats_q_, feats_t_, matches01_ = [rbd(x) for x in [feats_q, feats_t, matches01]]
+    matches = matches01_["matches"]
+    if matches.shape[0] == 0:
+        return 0, 0.0
+
+    kpts_q = feats_q_["keypoints"][matches[..., 0]].cpu().numpy()
+    kpts_t = feats_t_["keypoints"][matches[..., 1]].cpu().numpy()
+
+    keep = keep_points_inside_mask(kpts_q, mask_q) & keep_points_inside_mask(kpts_t, mask_t)
+    kpts_q, kpts_t = kpts_q[keep], kpts_t[keep]
+
+    if len(kpts_q) < 4:
+        return 0, 0.0
+
+    src = kpts_q.reshape(-1, 1, 2).astype(np.float32)
+    dst = kpts_t.reshape(-1, 1, 2).astype(np.float32)
+
+    _, ransac_mask = cv2.findHomography(src, dst, cv2.RANSAC, 5.0)
+    if ransac_mask is None:
+        return 0, 0.0
+
+    inlier_count = int(ransac_mask.sum())
+    confidence_pct = (inlier_count / len(src)) * 100 if len(src) > 0 else 0.0
+    return inlier_count, confidence_pct
+
+
+def superglue_match_against_templates(bgr, mask):
+    feats_q, mask_q = superpoint_extract(bgr, mask)
+    results = []
+    for template_name, data in _template_superpoint_feats.items():
+        inlier_count, confidence_pct = _lightglue_pair(
+            feats_q, data["feats"], mask_q, data["mask"]
+        )
+        results.append({
+            "template": template_name,
+            "score": inlier_count,
+            "confidence": round(confidence_pct, 1),
+        })
+    results.sort(key=lambda r: r["score"], reverse=True)
+    return results
+
+
+# ---------------------------------------------------------------
+# LoFTR
+# ---------------------------------------------------------------
+
+def set_template_loftr(template_name, bgr, mask):
+    tensor, resized_mask = masked_gray_tensor(bgr, mask, max_dim=config.LOFTR_MAX_DIM)
+    _template_loftr_tensors[template_name] = {"tensor": tensor, "mask": resized_mask}
+
+
+def _loftr_pair(tensor_q, tensor_t, mask_q, mask_t):
+    with torch.no_grad():
+        out = get_loftr()({"image0": tensor_q, "image1": tensor_t})
+
+    conf = out["confidence"].cpu().numpy()
+    keep_conf = conf >= config.LOFTR_CONFIDENCE_THRESHOLD
+
+    kpts_q = out["keypoints0"].cpu().numpy()[keep_conf]
+    kpts_t = out["keypoints1"].cpu().numpy()[keep_conf]
+
+    keep = keep_points_inside_mask(kpts_q, mask_q) & keep_points_inside_mask(kpts_t, mask_t)
+    kpts_q, kpts_t = kpts_q[keep], kpts_t[keep]
+
+    if len(kpts_q) < 4:
+        return 0, 0.0
+
+    src = kpts_q.reshape(-1, 1, 2).astype(np.float32)
+    dst = kpts_t.reshape(-1, 1, 2).astype(np.float32)
+
+    _, ransac_mask = cv2.findHomography(src, dst, cv2.RANSAC, 5.0)
+    if ransac_mask is None:
+        return 0, 0.0
+
+    inlier_count = int(ransac_mask.sum())
+    confidence_pct = (inlier_count / len(src)) * 100 if len(src) > 0 else 0.0
+    return inlier_count, confidence_pct
+
+
+def loftr_match_against_templates(bgr, mask):
+    tensor_q, mask_q = masked_gray_tensor(bgr, mask, max_dim=config.LOFTR_MAX_DIM)
+    results = []
+    for template_name, data in _template_loftr_tensors.items():
+        inlier_count, confidence_pct = _loftr_pair(
+            tensor_q, data["tensor"], mask_q, data["mask"]
+        )
+        results.append({
+            "template": template_name,
+            "score": inlier_count,
+            "confidence": round(confidence_pct, 1),
+        })
+        # LoFTR has no reusable per-image descriptor -- this is a full dense
+        # CNN+transformer forward pass PER TEMPLATE (6 of them per query,
+        # back to back). Left uncleared, each pass's intermediate
+        # activations fragment the allocator further than SuperPoint/
+        # LightGlue's much lighter per-template cost ever does, and on an
+        # 8GB card that's enough on its own to tip into CUDA OOM a few
+        # templates in.
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    results.sort(key=lambda r: r["score"], reverse=True)
+    return results

pipeline/engine.py

@@ -0,0 +1,858 @@
+"""
+Orchestrates the whole match pipeline:
+
+  startup (once):  bg-remove every template + precompute features for all
+                    four methods, so a request only ever has to process the
+                    single uploaded image.
+
+  per request:      bg-remove the upload, then run SIFT / ORB / SuperGlue /
+                     LoFTR one after another. They used to run concurrently
+                     (real wall-clock ~max() instead of sum()), but on an
+                     8GB card that let SuperGlue and LoFTR hold overlapping
+                     GPU allocations at once, which was enough on its own to
+                     tip into CUDA OOM. Running them sequentially caps peak
+                     GPU memory to whichever single method needs the most,
+                     at the cost of some latency.
+"""
+
+import gc
+import logging
+import os
+import resource
+import shutil
+import threading
+import time
+import uuid
+
+import cv2
+import torch
+
+import config
+from pipeline import (bg_removal, classical, color, color_grid, deep,
+                       flower_count, sam3_client, shape_match, texture_match,
+                       utils, vase_compare, yolo_world)
+
+logger = logging.getLogger(__name__)
+
+METHODS = ("SIFT", "ORB", "SuperGlue", "LoFTR")
+
+_templates_meta = {}   # name -> {"original_filename", "nobg_path"}
+_bootstrapped = False
+
+# Serializes each request's pipeline end-to-end. Combined with running the
+# four methods sequentially (see below), only one method for one request is
+# ever doing heavy CPU/GPU work at any instant across the whole process.
+_pipeline_lock = threading.Lock()
+
+
+def bootstrap():
+    """Precompute everything template-side. Safe to call more than once;
+    only does real work the first time."""
+    global _bootstrapped
+    if _bootstrapped:
+        return
+
+    logger.info("Preparing templates (bg removal + feature precompute for "
+                "SIFT / ORB / SuperGlue / LoFTR)...")
+
+    for fname in utils.list_template_files():
+        name = utils.template_name(fname)
+        src_path = os.path.join(config.TEMPLATE_IMAGES_DIR, fname)
+
+        rgba = bg_removal.remove_background_file(src_path, config.TEMPLATES_NOBG_CACHE)
+        bgr, _, mask = bg_removal.split_rgba(rgba)
+
+        kp, des = classical.extract_features("SIFT", bgr, mask)
+        classical.set_template_features("SIFT", name, kp, des)
+
+        kp, des = classical.extract_features("ORB", bgr, mask)
+        classical.set_template_features("ORB", name, kp, des)
+
+        deep.set_template_superpoint(name, bgr, mask)
+        deep.set_template_loftr(name, bgr, mask)
+        color.set_template_color_data(name, bgr, mask)
+        shape_match.set_template_shape_data(name, mask)
+        texture_match.set_template_texture_data(name, bgr, mask)
+
+        _templates_meta[name] = {
+            "original_filename": fname,
+            "nobg_path": os.path.join(config.TEMPLATES_NOBG_CACHE, name + ".png"),
+        }
+        logger.info("  template ready: %s", name)
+
+    _bootstrapped = True
+    logger.info("%d templates ready (device: %s).", len(_templates_meta), deep.DEVICE)
+
+
+def templates_meta():
+    return _templates_meta
+
+
+def _run_sift(bgr, mask):
+    kp, des = classical.extract_features("SIFT", bgr, mask)
+    return classical.match_against_templates("SIFT", kp, des)
+
+
+def _run_orb(bgr, mask):
+    kp, des = classical.extract_features("ORB", bgr, mask)
+    return classical.match_against_templates("ORB", kp, des)
+
+
+def _run_superglue(bgr, mask):
+    return deep.superglue_match_against_templates(bgr, mask)
+
+
+def _run_loftr(bgr, mask):
+    return deep.loftr_match_against_templates(bgr, mask)
+
+
+_METHOD_RUNNERS = {
+    "SIFT": _run_sift,
+    "ORB": _run_orb,
+    "SuperGlue": _run_superglue,
+    "LoFTR": _run_loftr,
+}
+
+
+def _timed(method, fn, *args):
+    """Runs one method's matcher, catching any exception so a single
+    failing method (e.g. a CUDA OOM on an unusual image) is logged and
+    reported back as a failed card instead of taking the whole request
+    down with it."""
+    start = time.perf_counter()
+    try:
+        result = fn(*args)
+        error = None
+    except Exception as e:
+        logger.exception("Method %s failed", method)
+        result = []
+        error = str(e)
+    elapsed = time.perf_counter() - start
+    return result, elapsed, error
+
+
+def _overall_best(method_results):
+    """Borda-count-style aggregate across methods: each method's ranking of
+    templates contributes points, so one noisy method can't dominate the
+    call the way a raw-score sum could."""
+    points = {}
+    for method, payload in method_results.items():
+        ranked = payload["results"]
+        n = len(ranked)
+        for idx, row in enumerate(ranked):
+            points[row["template"]] = points.get(row["template"], 0) + (n - idx)
+
+    if not points:
+        return None
+    return max(points, key=points.get)
+
+
+def _weighted_scores(method_results):
+    """Literal weighted sum per template: weighted_score(t) = sum_m
+    WEIGHT[m] * raw_score(m, t). Returns (winning_template_or_None, ranked
+    list of {"template", "weighted_score", "breakdown"} sorted descending).
+    A method that errored contributes 0 everywhere (its results list is
+    empty), it doesn't skew the total."""
+    totals = {}
+    breakdown = {}  # template -> {method: contribution}
+
+    for method, payload in method_results.items():
+        weight = config.METHOD_WEIGHTS.get(method, 0)
+        for row in payload["results"]:
+            template = row["template"]
+            contribution = weight * row["score"]
+            totals[template] = totals.get(template, 0.0) + contribution
+            breakdown.setdefault(template, {})[method] = round(contribution, 2)
+
+    if not totals:
+        return None, []
+
+    ranked = sorted(
+        (
+            {
+                "template": template,
+                "weighted_score": round(total, 2),
+                "breakdown": breakdown.get(template, {}),
+            }
+            for template, total in totals.items()
+        ),
+        key=lambda r: r["weighted_score"],
+        reverse=True,
+    )
+    return ranked[0]["template"], ranked
+
+
+def _color_as_method_result(color_analysis, elapsed, error):
+    """Reshapes color.compare_input_to_templates()'s output into the same
+    {"template", "score", "confidence"} shape the other methods use, so it
+    can be handed to _weighted_scores() generically -- that's the only
+    place this participates; it's deliberately kept out of the sequential
+    METHODS loop (color analysis isn't a GPU/CPU-heavy per-template model
+    call, no need for that machinery) and out of _overall_best (Borda stays
+    exactly the original four methods, unchanged)."""
+    results = [
+        {"template": t["template"], "score": t["match_pct"], "confidence": t["match_pct"]}
+        for t in color_analysis.get("templates", [])
+    ]
+    best = results[0] if results else None
+    return {
+        "results": results,
+        "time_sec": elapsed,
+        "error": error,
+        "best": best,
+        "is_confident": bool(best and best["score"] >= config.SCORE_THRESHOLD["Color"]),
+        "best_image_file": None,
+    }
+
+
+def _build_family_grid(request_dir, input_bgr, input_mask, weighted_best):
+    """Purely visual, tied to the weighted-best template only: clusters
+    both the upload and the winning template into color-region tiles,
+    renders each as its own side-by-side grid image, and matches up
+    corresponding regions by color to report an area-match % per region."""
+    input_families = color_grid.cluster_families(input_bgr, input_mask)
+
+    template_rgba = cv2.imread(
+        _templates_meta[weighted_best]["nobg_path"], cv2.IMREAD_UNCHANGED
+    )
+    template_bgr, _, template_mask = bg_removal.split_rgba(template_rgba)
+    template_families = color_grid.cluster_families(template_bgr, template_mask)
+
+    matches = color_grid.match_families(input_families, template_families)
+    overall_pct = color_grid.overall_area_match(matches)
+
+    input_grid_file = "input_family_grid.png"
+    template_grid_file = "template_family_grid.png"
+    cv2.imwrite(os.path.join(request_dir, input_grid_file),
+                color_grid.render_family_grid(input_bgr, input_families))
+    cv2.imwrite(os.path.join(request_dir, template_grid_file),
+                color_grid.render_family_grid(template_bgr, template_families))
+
+    return {
+        "template": weighted_best,
+        "input_grid_file": input_grid_file,
+        "template_grid_file": template_grid_file,
+        "overall_area_match_pct": overall_pct,
+        "matches": matches,
+        "error": None,
+    }
+
+
+def _build_shape_visuals(request_dir, input_shape_data, weighted_best):
+    """Purely visual, tied to the weighted-best template: renders both
+    silhouettes (normalized into the same canvas) side by side plus an
+    overlay highlighting exactly where they agree/diverge."""
+    template_shape_data = shape_match._template_shape_data.get(weighted_best)
+    if template_shape_data is None:
+        return None
+
+    input_file = "input_silhouette.png"
+    template_file = "template_silhouette.png"
+    overlay_file = "shape_overlay.png"
+    cv2.imwrite(os.path.join(request_dir, input_file),
+                shape_match.render_silhouette(input_shape_data["canonical_mask"],
+                                               shape_match._INPUT_COLOR))
+    cv2.imwrite(os.path.join(request_dir, template_file),
+                shape_match.render_silhouette(template_shape_data["canonical_mask"],
+                                               shape_match._TEMPLATE_COLOR))
+    cv2.imwrite(os.path.join(request_dir, overlay_file),
+                shape_match.render_overlay(input_shape_data["canonical_mask"],
+                                            template_shape_data["canonical_mask"]))
+
+    return {
+        "input_file": input_file,
+        "template_file": template_file,
+        "overlay_file": overlay_file,
+    }
+
+
+def _build_texture_visuals(request_dir, input_texture_data, weighted_best):
+    """Purely visual, tied to the weighted-best template: renders both
+    images' LBP (local micro-pattern) maps side by side."""
+    template_texture_data = texture_match._template_texture_data.get(weighted_best)
+    if template_texture_data is None:
+        return None
+
+    input_file = "input_lbp.png"
+    template_file = "template_lbp.png"
+    cv2.imwrite(os.path.join(request_dir, input_file),
+                texture_match.render_lbp_visual(input_texture_data["lbp_image"],
+                                                 input_texture_data["mask_bool"]))
+    cv2.imwrite(os.path.join(request_dir, template_file),
+                texture_match.render_lbp_visual(template_texture_data["lbp_image"],
+                                                 template_texture_data["mask_bool"]))
+
+    return {
+        "input_file": input_file,
+        "template_file": template_file,
+    }
+
+
+def _cleanup_old_uploads():
+    if not os.path.isdir(config.UPLOADS_DIR):
+        return
+    cutoff = time.time() - config.MAX_UPLOAD_AGE_SECONDS
+    for entry in os.listdir(config.UPLOADS_DIR):
+        path = os.path.join(config.UPLOADS_DIR, entry)
+        try:
+            if os.path.isdir(path) and os.path.getmtime(path) < cutoff:
+                shutil.rmtree(path, ignore_errors=True)
+        except OSError:
+            pass
+
+
+def _peak_rss_mb():
+    # ru_maxrss is KB on Linux, bytes on macOS -- this app only targets Linux.
+    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
+
+
+def _release_resources():
+    """Best-effort cleanup between requests. gc.collect() drops any
+    lingering numpy/cv2/torch objects promptly instead of waiting for the
+    next allocation to trigger a cycle; empty_cache() hands unused *cached*
+    CUDA blocks back to the driver so nvidia-smi/other processes see them
+    freed (it does not, and cannot, free host RAM)."""
+    gc.collect()
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+
+def _build_flower_count_comparison(input_count, template_count, template):
+    diff = input_count - template_count
+    if diff == 0:
+        message = (
+            f"Your upload and {template} both show {input_count} flowers -- "
+            f"flower count doesn't look like a factor in the match score here."
+        )
+    else:
+        more_fewer = "more" if diff > 0 else "fewer"
+        message = (
+            f"Your upload has {input_count} flowers detected vs {template_count} in "
+            f"{template} -- {abs(diff)} {more_fewer}. No method above scores flower "
+            f"count directly, but a mismatch like this usually also lowers the "
+            f"keypoint (SIFT/ORB/SuperGlue/LoFTR) and shape/texture scores, since "
+            f"there's simply more or less bouquet for them to match against -- so "
+            f"it's often part of why the overall match isn't higher."
+        )
+    return {
+        "input_count": input_count,
+        "template_count": template_count,
+        "diff": diff,
+        "message": message,
+        "error": None,
+    }
+
+
+def count_flowers(request_id: str, template: str = None) -> dict:
+    """On-demand flower counting for an already-processed upload, via two
+    independent methods shown side by side, plus (if a matched template
+    name is given) a vase-identity comparison:
+
+      - SAM3 (instance segmentation): finds individual flower-shaped
+        regions by prompting SAM3 with the plain-English concept "flower".
+        Unlike SAM1's old "segment everything, then guess" approach, this
+        never proposes the vase or any ribbon/bow in the first place -- no
+        exclude-box filtering needed. Runs as a subprocess into a separate
+        Python 3.10 env (see pipeline/sam3_client.py) since transformers'
+        Sam3Model needs Python>=3.10, incompatible with this app's own env.
+        The same subprocess call also fetches SAM3's "vase" mask for the
+        upload (and the template's, if a match is known) for the vase
+        comparison below -- one model load covers every job.
+      - YOLO-World (open-vocabulary detection): independent second opinion,
+        unchanged from before -- its own "flower"/"vase"/"ribbon"/"bow"
+        counts and box visualization, shown as its own coarser corroborating
+        signal (one box per contiguous flower region, not per bloom).
+      - Vase comparison (DINOv2 + CLIP): crops the vase out of the upload
+        and the matched template using SAM3's precise masks (background
+        pixels inside the crop are blacked out, so the embeddings only see
+        the vase itself), then embeds both crops with DINOv2 (fine detail)
+        and CLIP (holistic/semantic) and reports a same/uncertain/different
+        verdict. Skipped (not an error) if no vase was detected on either
+        side.
+
+    Heavyweight and strictly opt-in -- triggered only by its own UI button,
+    never run as part of process_upload. Unloads the matching pipeline's own
+    GPU-resident models first (SuperPoint/LightGlue/LoFTR, rembg's
+    onnxruntime session) so YOLO-World/DINOv2/CLIP never have to share the
+    8GB card with them (SAM3 runs in its own process/env regardless), then
+    unloads YOLO-World/DINOv2/CLIP again afterward -- everything lazily
+    reloads itself on whichever request needs it next, exactly like a fresh
+    process start would."""
+    request_dir = os.path.join(config.UPLOADS_DIR, request_id)
+    nobg_path = os.path.join(request_dir, "nobg.png")
+    if not os.path.isfile(nobg_path):
+        raise FileNotFoundError(f"no processed upload found for request {request_id}")
+
+    with _pipeline_lock:
+        start = time.perf_counter()
+        rgba = cv2.imread(nobg_path, cv2.IMREAD_UNCHANGED)
+        bgr, _, mask = bg_removal.split_rgba(rgba)
+
+        freed_deep = deep.unload_models()
+        freed_rembg = bg_removal.unload_session()
+        if freed_deep or freed_rembg:
+            logger.info("[%s] unloaded matching-pipeline models before "
+                        "SAM3/YOLO-World/DINO/CLIP run", request_id)
+
+        try:
+            by_class = yolo_world.detect(bgr)
+            yolo_error = None
+        except Exception as e:
+            logger.exception("[%s] YOLO-World detection failed", request_id)
+            by_class = {}
+            yolo_error = str(e)
+
+        template_bgr = None
+        template_mask = None
+        if template is not None and template in _templates_meta:
+            template_rgba = cv2.imread(
+                _templates_meta[template]["nobg_path"], cv2.IMREAD_UNCHANGED
+            )
+            template_bgr, _, template_mask = bg_removal.split_rgba(template_rgba)
+
+        sam3_images = {"input": bgr}
+        sam3_jobs = [
+            {"image": "input", "prompt": config.SAM3_FLOWER_PROMPT,
+             "threshold": config.SAM3_FLOWER_THRESHOLD},
+            {"image": "input", "prompt": config.SAM3_VASE_PROMPT,
+             "threshold": config.SAM3_VASE_THRESHOLD},
+        ]
+        if template_bgr is not None:
+            sam3_images["template"] = template_bgr
+            sam3_jobs.append({"image": "template", "prompt": config.SAM3_VASE_PROMPT,
+                               "threshold": config.SAM3_VASE_THRESHOLD})
+            # Also count flowers in the template photo itself (same "flower"
+            # concept prompt) -- lets the UI tell the user when a lower match
+            # score might partly be explained by a different flower count,
+            # rather than leaving that as an unexplained low number.
+            sam3_jobs.append({"image": "template", "prompt": config.SAM3_FLOWER_PROMPT,
+                               "threshold": config.SAM3_FLOWER_THRESHOLD})
+
+        try:
+            sam3_result = sam3_client.run_jobs(sam3_images, sam3_jobs, workdir=request_dir)
+            sam3_error = None
+        except Exception as e:
+            logger.exception("[%s] SAM3 failed", request_id)
+            sam3_result = {}
+            sam3_error = str(e)
+
+        try:
+            flower_instances = sam3_result.get(("input", config.SAM3_FLOWER_PROMPT), [])
+            count_data = flower_count.count_flowers(bgr, mask, request_dir,
+                                                      instances_raw=flower_instances)
+            sam_error = sam3_error
+        except Exception as e:
+            logger.exception("[%s] flower counting (SAM3) failed", request_id)
+            count_data = {"total_count": 0, "clusters": []}
+            sam_error = sam3_error or str(e)
+
+        sam_visual_file = None
+        if sam_error is None:
+            try:
+                visual = flower_count.render_instances(bgr, count_data)
+                sam_visual_file = "flower_count_sam.png"
+                cv2.imwrite(os.path.join(request_dir, sam_visual_file), visual)
+            except Exception:
+                logger.exception("[%s] SAM3 flower count visualization failed", request_id)
+
+        # Same flower-instance segmentation, run on the matched template's
+        # own photo -- lets callers (the web UI's mismatch note, tester.py's
+        # report image) show/compare "your photo's flowers" side by side
+        # with "the template's flowers", not just two bare numbers.
+        template_count_data = {"total_count": 0, "clusters": []}
+        template_sam_visual_file = None
+        template_sam_error = None
+        if template_bgr is not None and template_mask is not None:
+            if sam3_error is not None:
+                template_sam_error = sam3_error
+            else:
+                try:
+                    template_flower_instances = sam3_result.get(
+                        ("template", config.SAM3_FLOWER_PROMPT), []
+                    )
+                    template_count_data = flower_count.count_flowers(
+                        template_bgr, template_mask, request_dir,
+                        instances_raw=template_flower_instances,
+                    )
+                    visual = flower_count.render_instances(template_bgr, template_count_data)
+                    template_sam_visual_file = "flower_count_sam_template.png"
+                    cv2.imwrite(os.path.join(request_dir, template_sam_visual_file), visual)
+                except Exception as e:
+                    logger.exception("[%s] template flower count failed", request_id)
+                    template_sam_error = str(e)
+
+        yolo_visual_file = None
+        if yolo_error is None:
+            try:
+                yolo_overlay = yolo_world.render_boxes(bgr, by_class)
+                yolo_visual_file = "flower_count_yolo.png"
+                cv2.imwrite(os.path.join(request_dir, yolo_visual_file), yolo_overlay)
+            except Exception:
+                logger.exception("[%s] YOLO-World visualization failed", request_id)
+
+        # Vase-identity comparison: needs a matched template name and a vase
+        # mask on both sides (from the batched SAM3 call above). Soft-
+        # skipped (not an error) if either is missing -- e.g. a weak/no
+        # match, or a photo where SAM3 simply didn't find the vase.
+        input_vase_instances = sam3_result.get(("input", config.SAM3_VASE_PROMPT), [])
+        vase_comparison = None
+        if template is not None and sam3_error is not None:
+            vase_comparison = {"error": f"SAM3 unavailable: {sam3_error}"}
+        elif template is not None and not input_vase_instances:
+            vase_comparison = {"error": "No vase detected in your upload."}
+        elif template is not None and template_bgr is not None:
+            try:
+                template_vase_instances = sam3_result.get(
+                    ("template", config.SAM3_VASE_PROMPT), []
+                )
+                if not template_vase_instances:
+                    vase_comparison = {"error": "No vase detected in the matched template photo."}
+                else:
+                    input_vase_mask = max(input_vase_instances, key=lambda i: i["score"])["mask"]
+                    template_vase_mask = max(template_vase_instances, key=lambda i: i["score"])["mask"]
+                    input_vase_crop = vase_compare.crop_mask(bgr, input_vase_mask)
+                    template_vase_crop = vase_compare.crop_mask(template_bgr, template_vase_mask)
+
+                    result = vase_compare.compare_vases(input_vase_crop, template_vase_crop)
+                    input_crop_file = "vase_crop_input.png"
+                    template_crop_file = "vase_crop_template.png"
+                    cv2.imwrite(os.path.join(request_dir, input_crop_file), input_vase_crop)
+                    cv2.imwrite(os.path.join(request_dir, template_crop_file), template_vase_crop)
+                    vase_comparison = {
+                        **result,
+                        "template": template,
+                        "input_crop_file": input_crop_file,
+                        "template_crop_file": template_crop_file,
+                        "error": None,
+                    }
+            except Exception as e:
+                logger.exception("[%s] vase comparison failed", request_id)
+                vase_comparison = {"error": str(e)}
+
+        # Flower-count comparison: surfaces a plain-English explanation when
+        # the upload and the matched template simply have different numbers
+        # of flowers -- none of SIFT/ORB/SuperGlue/LoFTR/shape/texture score
+        # "flower count" directly, so a lower match score caused mostly by a
+        # count mismatch would otherwise look unexplained to the user.
+        flower_count_comparison = None
+        if template is not None and template_sam_error is None and template_bgr is not None:
+            flower_count_comparison = _build_flower_count_comparison(
+                count_data.get("total_count", 0), template_count_data.get("total_count", 0), template
+            )
+        elif template is not None and template_sam_error is not None:
+            flower_count_comparison = {"error": f"SAM3 unavailable: {template_sam_error}"}
+
+        yolo_world.unload_model()
+        vase_compare.unload_models()
+        _release_resources()
+        elapsed = time.perf_counter() - start
+        logger.info(
+            "[%s] flower count: %.2fs, SAM3 total=%s, YOLO flower/vase/ribbon=%s/%s/%s, "
+            "vase comparison=%s",
+            request_id, elapsed, count_data.get("total_count"),
+            len(by_class.get("flower", [])), len(by_class.get("vase", [])),
+            len(by_class.get("ribbon", [])) + len(by_class.get("bow", [])),
+            vase_comparison.get("verdict") if vase_comparison and not vase_comparison.get("error") else None,
+        )
+
+    return {
+        "sam": {
+            "total_count": count_data.get("total_count", 0),
+            "clusters": count_data.get("clusters", []),
+            "visual_file": sam_visual_file,
+            "error": sam_error,
+        },
+        "sam_template": {
+            "total_count": template_count_data.get("total_count", 0),
+            "clusters": template_count_data.get("clusters", []),
+            "visual_file": template_sam_visual_file,
+            "error": template_sam_error,
+        },
+        "yolo": {
+            "flower_count": len(by_class.get("flower", [])),
+            "vase_count": len(by_class.get("vase", [])),
+            "ribbon_count": len(by_class.get("ribbon", [])) + len(by_class.get("bow", [])),
+            "visual_file": yolo_visual_file,
+            "error": yolo_error,
+        },
+        "vase_comparison": vase_comparison,
+        "flower_count_comparison": flower_count_comparison,
+        "time_sec": round(elapsed, 3),
+    }
+
+
+def flower_summary(request_id: str, template: str) -> dict:
+    """Lightweight companion to the weighted verdict, shown automatically
+    right beside it (not behind the opt-in "Count flowers" button) as soon
+    as a confident match is found: just the flower-count comparison (same
+    message as count_flowers' version) plus a CLIP-only similarity of the
+    flower material itself (SAM3's "flower" masks on both sides, unioned
+    and background-blacked-out, then embedded with CLIP alone).
+
+    Deliberately skips YOLO-World, DINOv2, and the vase comparison -- those
+    stay behind the button since this one runs unconditionally on every
+    confident match and should stay as fast as a SAM3 round trip allows.
+    Still needs the matching pipeline's GPU-resident models unloaded first,
+    same as count_flowers."""
+    request_dir = os.path.join(config.UPLOADS_DIR, request_id)
+    nobg_path = os.path.join(request_dir, "nobg.png")
+    if not os.path.isfile(nobg_path):
+        raise FileNotFoundError(f"no processed upload found for request {request_id}")
+    if template not in _templates_meta:
+        raise ValueError(f"unknown template {template!r}")
+
+    with _pipeline_lock:
+        rgba = cv2.imread(nobg_path, cv2.IMREAD_UNCHANGED)
+        bgr, _, mask = bg_removal.split_rgba(rgba)
+        template_rgba = cv2.imread(
+            _templates_meta[template]["nobg_path"], cv2.IMREAD_UNCHANGED
+        )
+        template_bgr, _, template_mask = bg_removal.split_rgba(template_rgba)
+
+        freed_deep = deep.unload_models()
+        freed_rembg = bg_removal.unload_session()
+        if freed_deep or freed_rembg:
+            logger.info("[%s] unloaded matching-pipeline models before flower summary",
+                        request_id)
+
+        count_comparison = None
+        clip_pct = None
+        error = None
+        try:
+            sam3_result = sam3_client.run_jobs(
+                images={"input": bgr, "template": template_bgr},
+                jobs=[
+                    {"image": "input", "prompt": config.SAM3_FLOWER_PROMPT,
+                     "threshold": config.SAM3_FLOWER_THRESHOLD},
+                    {"image": "template", "prompt": config.SAM3_FLOWER_PROMPT,
+                     "threshold": config.SAM3_FLOWER_THRESHOLD},
+                ],
+                workdir=request_dir,
+            )
+            input_instances = sam3_result.get(("input", config.SAM3_FLOWER_PROMPT), [])
+            template_instances = sam3_result.get(("template", config.SAM3_FLOWER_PROMPT), [])
+
+            input_data = flower_count.count_flowers(bgr, mask, request_dir,
+                                                      instances_raw=input_instances)
+            template_data = flower_count.count_flowers(template_bgr, template_mask, request_dir,
+                                                         instances_raw=template_instances)
+            count_comparison = _build_flower_count_comparison(
+                input_data["total_count"], template_data["total_count"], template
+            )
+
+            input_union = flower_count.union_mask(input_data["_instance_masks"])
+            template_union = flower_count.union_mask(template_data["_instance_masks"])
+            if input_union is not None and template_union is not None:
+                input_crop = vase_compare.crop_mask(bgr, input_union)
+                template_crop = vase_compare.crop_mask(template_bgr, template_union)
+                if input_crop is not None and template_crop is not None:
+                    clip_pct = vase_compare.clip_similarity_pct(input_crop, template_crop)
+        except Exception as e:
+            logger.exception("[%s] flower summary failed", request_id)
+            error = str(e)
+
+        vase_compare.unload_models()
+        _release_resources()
+
+    return {
+        "flower_count_comparison": count_comparison,
+        "flower_clip_similarity_pct": clip_pct,
+        "error": error,
+    }
+
+
+def process_upload(image_bytes: bytes, orig_filename: str) -> dict:
+    bootstrap()
+    _cleanup_old_uploads()
+
+    request_id = uuid.uuid4().hex[:12]
+    logger.info("[%s] new upload: %r (%.1f KB)", request_id, orig_filename,
+                len(image_bytes) / 1024)
+
+    with _pipeline_lock:
+        try:
+            result = _process_upload_locked(request_id, image_bytes, orig_filename)
+        except Exception:
+            logger.exception("[%s] pipeline failed", request_id)
+            raise
+        finally:
+            _release_resources()
+            logger.info("[%s] done, peak RSS so far: %.0f MB", request_id, _peak_rss_mb())
+
+    return result
+
+
+def _process_upload_locked(request_id: str, image_bytes: bytes, orig_filename: str) -> dict:
+    request_dir = os.path.join(config.UPLOADS_DIR, request_id)
+    os.makedirs(request_dir, exist_ok=True)
+
+    total_start = time.perf_counter()
+
+    ext = os.path.splitext(orig_filename)[1].lower() or ".png"
+    original_path = os.path.join(request_dir, "original" + ext)
+    with open(original_path, "wb") as f:
+        f.write(image_bytes)
+
+    bg_start = time.perf_counter()
+    rgba, _ = bg_removal.remove_background_bytes(image_bytes, config.UPLOADS_NOBG_CACHE)
+    bg_elapsed = time.perf_counter() - bg_start
+    logger.info("[%s] background removal: %.2fs", request_id, bg_elapsed)
+    bgr, _, mask = bg_removal.split_rgba(rgba)
+
+    nobg_path = os.path.join(request_dir, "nobg.png")
+    cv2.imwrite(nobg_path, rgba)
+
+    method_results = {}
+    for method in METHODS:
+        results, elapsed, error = _timed(method, _METHOD_RUNNERS[method], bgr, mask)
+        method_results[method] = {"results": results, "time_sec": elapsed, "error": error}
+        logger.info("[%s] %s: %.2fs%s", request_id, method, elapsed,
+                    f" (FAILED: {error})" if error else "")
+        # Free this method's GPU allocations before the next one runs rather
+        # than waiting until the whole request finishes -- SuperGlue and
+        # LoFTR are the two that actually use the GPU, and freeing between
+        # them keeps their peak allocations from ever coexisting.
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+    for method, payload in method_results.items():
+        results = payload["results"]
+        best = results[0] if results else None
+        payload["best"] = best
+        payload["is_confident"] = bool(
+            best and best["score"] >= config.SCORE_THRESHOLD[method]
+        )
+        if best is not None:
+            template_bgr = cv2.imread(
+                _templates_meta[best["template"]]["nobg_path"], cv2.IMREAD_UNCHANGED
+            )
+            annotated = utils.annotate_score(
+                template_bgr, best["score"], best["confidence"],
+                label=config.METHOD_LABELS[method],
+            )
+            out_name = f"{method}_best.png"
+            cv2.imwrite(os.path.join(request_dir, out_name), annotated)
+            payload["best_image_file"] = out_name
+
+    # Color-space comparison: its own section in the UI (independent of the
+    # method-grid cards above), but also folded into the weighted verdict
+    # below via config.METHOD_WEIGHTS["Color"]. A failure here shouldn't
+    # take down a request that otherwise succeeded.
+    color_start = time.perf_counter()
+    try:
+        color_analysis = color.compare_input_to_templates(bgr, mask)
+        color_error = None
+    except Exception as e:
+        logger.exception("[%s] color analysis failed", request_id)
+        color_analysis = {"input_dominant_colors": [], "templates": []}
+        color_error = str(e)
+    color_elapsed = time.perf_counter() - color_start
+    logger.info("[%s] color analysis: %.2fs%s", request_id, color_elapsed,
+                f" (FAILED: {color_error})" if color_error else "")
+
+    # Shape matching (silhouette/contour, via Hu moments + canonical-aligned
+    # IoU) and texture matching (LBP + GLCM/Haralick) -- both independent
+    # sections, like color space: never fed into weighting or any score.
+    shape_start = time.perf_counter()
+    try:
+        input_shape_data, shape_results = shape_match.compare_to_templates(mask)
+        shape_error = None
+    except Exception as e:
+        logger.exception("[%s] shape analysis failed", request_id)
+        input_shape_data, shape_results = None, []
+        shape_error = str(e)
+    shape_elapsed = time.perf_counter() - shape_start
+    logger.info("[%s] shape analysis: %.2fs%s", request_id, shape_elapsed,
+                f" (FAILED: {shape_error})" if shape_error else "")
+
+    texture_start = time.perf_counter()
+    try:
+        input_texture_data, texture_results = texture_match.compare_to_templates(bgr, mask)
+        texture_error = None
+    except Exception as e:
+        logger.exception("[%s] texture analysis failed", request_id)
+        input_texture_data, texture_results = None, []
+        texture_error = str(e)
+    texture_elapsed = time.perf_counter() - texture_start
+    logger.info("[%s] texture analysis: %.2fs%s", request_id, texture_elapsed,
+                f" (FAILED: {texture_error})" if texture_error else "")
+
+    # Only used for weighting/the returned "methods" dict (so the frontend
+    # can look up Color's raw score for the weighted-breakdown display) --
+    # NOT rendered as a 5th method-grid card, and NOT part of _overall_best.
+    methods_with_color = dict(method_results)
+    methods_with_color["Color"] = _color_as_method_result(color_analysis, color_elapsed, color_error)
+
+    weighted_best, weighted_scores = _weighted_scores(methods_with_color)
+
+    # Color family grid: purely visual, tied to whichever template the
+    # weighted verdict landed on. Skipped if there's no confident match at
+    # all (nothing to compare against). A failure here is likewise soft.
+    family_grid = None
+    if weighted_best is not None:
+        try:
+            family_grid = _build_family_grid(request_dir, bgr, mask, weighted_best)
+        except Exception as e:
+            logger.exception("[%s] family grid failed", request_id)
+            family_grid = {"error": str(e)}
+
+    # Shape/texture visuals: same "tied to whichever template the weighted
+    # verdict landed on" convention as the color family grid.
+    shape_visuals = None
+    if weighted_best is not None and input_shape_data is not None:
+        try:
+            shape_visuals = _build_shape_visuals(request_dir, input_shape_data, weighted_best)
+        except Exception as e:
+            logger.exception("[%s] shape visuals failed", request_id)
+            shape_visuals = None
+
+    texture_visuals = None
+    if weighted_best is not None and input_texture_data is not None:
+        try:
+            texture_visuals = _build_texture_visuals(request_dir, input_texture_data, weighted_best)
+        except Exception as e:
+            logger.exception("[%s] texture visuals failed", request_id)
+            texture_visuals = None
+
+    total_elapsed = time.perf_counter() - total_start
+    logger.info("[%s] total: %.2fs, weighted best: %s", request_id, total_elapsed, weighted_best)
+
+    return {
+        "request_id": request_id,
+        "upload_nobg_file": "nobg.png",
+        "upload_original_file": os.path.basename(original_path),
+        "bg_removal_time_sec": round(bg_elapsed, 3),
+        "total_time_sec": round(total_elapsed, 3),
+        "overall_best": _overall_best(method_results),
+        "method_weights": config.METHOD_WEIGHTS,
+        "weighted_best": weighted_best,
+        "weighted_scores": weighted_scores,
+        "color_analysis": color_analysis,
+        "color_analysis_time_sec": round(color_elapsed, 3),
+        "color_analysis_error": color_error,
+        "family_grid": family_grid,
+        "shape_analysis": {
+            "results": shape_results,
+            "time_sec": round(shape_elapsed, 3),
+            "error": shape_error,
+            "visuals": shape_visuals,
+        },
+        "texture_analysis": {
+            "results": texture_results,
+            "time_sec": round(texture_elapsed, 3),
+            "error": texture_error,
+            "visuals": texture_visuals,
+        },
+        "methods": {
+            method: {
+                "label": config.METHOD_LABELS[method],
+                "time_sec": round(payload["time_sec"], 3),
+                "best": payload["best"],
+                "is_confident": payload["is_confident"],
+                "best_image_file": payload.get("best_image_file"),
+                "results": payload["results"],
+                "error": payload["error"],
+            }
+            for method, payload in methods_with_color.items()
+        },
+    }

pipeline/flower_count.py

@@ -0,0 +1,138 @@
+"""
+Flower-instance counting via SAM3 (facebook/sam3), prompted with the plain-
+English concept "flower". See pipeline/sam3_client.py (and config.py's SAM3
+section) for why this runs as a one-shot subprocess into a separate Python
+3.10 environment rather than an in-process model call.
+
+Unlike the earlier approach (SAM1 in automatic "segment everything" mode,
+then guessing which proposals were flowers from size/position heuristics
+and excluding boxes YOLO-World identified as vase/ribbon), SAM3's concept
+prompting does the semantic part itself: prompted with "flower", it simply
+never proposes the vase or any ribbon/bow in the first place. What's left
+here is only: a light sanity filter (the instance must actually overlap the
+already-known foreground), then color-clustering the survivors as a rough
+proxy for distinct flower "kinds" -- there's still no trained species
+classifier, just an assumption that different flower types usually differ
+in color.
+"""
+
+import logging
+
+import cv2
+import numpy as np
+
+import config
+from pipeline import sam3_client
+
+logger = logging.getLogger(__name__)
+
+
+def union_mask(instance_masks):
+    """OR-combines every per-flower instance mask into one -- "all the
+    flower material, regardless of which bloom it belongs to". Used to crop
+    just the flowers (excluding vase/ribbon/background) out of a photo for
+    the CLIP flower-similarity check. Returns None if there are no
+    instances to combine."""
+    if not instance_masks:
+        return None
+    union = instance_masks[0].copy()
+    for m in instance_masks[1:]:
+        union |= m
+    return union
+
+
+def count_flowers(bgr, mask, workdir, image_key="input", instances_raw=None):
+    """instances_raw, if given, reuses SAM3 results already fetched by the
+    caller (engine.py batches the "flower" job for the upload together with
+    any "vase" jobs into a single sam3_client.run_jobs() call so the model
+    only loads once per request); otherwise fetches them here standalone."""
+    fg_area = int((mask > 0).sum())
+    if fg_area == 0:
+        return {"total_count": 0, "clusters": [], "_instance_masks": [], "_cluster_of_instance": []}
+
+    if instances_raw is None:
+        result = sam3_client.run_jobs(
+            images={image_key: bgr},
+            jobs=[{"image": image_key, "prompt": config.SAM3_FLOWER_PROMPT,
+                   "threshold": config.SAM3_FLOWER_THRESHOLD}],
+            workdir=workdir,
+        )
+        instances_raw = result.get((image_key, config.SAM3_FLOWER_PROMPT), [])
+
+    # Light sanity filter only -- SAM3 already did the semantic work of
+    # "is this a flower", this just guards against a stray instance
+    # entirely outside the known foreground (shouldn't happen against an
+    # already background-removed image, but costs nothing to check).
+    instance_masks = [
+        inst["mask"] for inst in instances_raw
+        if np.logical_and(inst["mask"], mask > 0).any()
+    ]
+
+    # Color-cluster the surviving instances as a proxy for distinct flower
+    # "kinds" -- same idea as the color-family grid elsewhere in this app
+    # (pipeline/color_grid.py), just applied per-instance instead of
+    # per-pixel-region.
+    avg_colors_lab = []
+    for m in instance_masks:
+        pixels = bgr[m].reshape(-1, 1, 3).astype(np.uint8)
+        lab = cv2.cvtColor(pixels, cv2.COLOR_BGR2LAB).reshape(-1, 3)
+        avg_colors_lab.append(lab.mean(axis=0))
+
+    clusters = []
+    cluster_of_instance = []
+    if avg_colors_lab:
+        pts = np.array(avg_colors_lab, dtype=np.float32)
+        k = min(config.SAM_MAX_KIND_CLUSTERS, len(pts))
+        if k <= 1:
+            labels = np.zeros(len(pts), dtype=int)
+            centers = pts
+        else:
+            criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.5)
+            _, labels, centers = cv2.kmeans(pts, k, None, criteria, 5, cv2.KMEANS_PP_CENTERS)
+            labels = labels.flatten()
+        cluster_of_instance = labels.tolist()
+        for c in range(len(centers)):
+            count = int((labels == c).sum())
+            if count == 0:
+                continue
+            lab_center = centers[c].reshape(1, 1, 3).astype(np.uint8)
+            bgr_center = cv2.cvtColor(lab_center, cv2.COLOR_LAB2BGR)[0, 0]
+            clusters.append({
+                "count": count,
+                "color_rgb": [int(bgr_center[2]), int(bgr_center[1]), int(bgr_center[0])],
+            })
+        clusters.sort(key=lambda c: c["count"], reverse=True)
+
+    return {
+        "total_count": len(instance_masks),
+        "clusters": clusters,
+        "_instance_masks": instance_masks,
+        "_cluster_of_instance": cluster_of_instance,
+    }
+
+
+_PALETTE = [
+    (66, 133, 244), (219, 68, 55), (244, 180, 0), (15, 157, 88),
+    (171, 71, 188), (255, 112, 67), (0, 172, 193), (158, 157, 36),
+]
+
+
+def render_instances(bgr, count_data):
+    """Overlays a translucent, distinctly-colored fill per detected flower
+    instance (color = its cluster/"kind") plus its index, so the count is
+    visually verifiable rather than just a bare number."""
+    overlay = bgr.copy()
+    instance_masks = count_data.get("_instance_masks", [])
+    cluster_of_instance = count_data.get("_cluster_of_instance", [])
+    for idx, m in enumerate(instance_masks):
+        cluster_id = cluster_of_instance[idx] if idx < len(cluster_of_instance) else idx
+        color = np.array(_PALETTE[cluster_id % len(_PALETTE)], dtype=np.float32)
+        overlay[m] = (color * 0.55 + overlay[m].astype(np.float32) * 0.45).astype(np.uint8)
+        ys, xs = np.nonzero(m)
+        if len(xs):
+            cx, cy = int(xs.mean()), int(ys.mean())
+            cv2.putText(overlay, str(idx + 1), (cx - 8, cy + 6),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2, cv2.LINE_AA)
+            cv2.putText(overlay, str(idx + 1), (cx - 8, cy + 6),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.6, (30, 30, 30), 1, cv2.LINE_AA)
+    return overlay

pipeline/sam3_client.py

@@ -0,0 +1,103 @@
+"""
+Bridge from the main app (torch17_new, Python 3.8) to SAM3 (sam3_worker.py,
+run under sam2_env's Python 3.10 -- see config.py's SAM3 section for why
+this has to be a subprocess rather than an in-process import).
+
+One subprocess invocation can batch multiple (image, prompt) jobs -- e.g.
+"flower" on the upload and "vase" on both the upload and the matched
+template -- so the ~6s model load only happens once per request, not once
+per job.
+"""
+
+import json
+import logging
+import os
+import shutil
+import subprocess
+import uuid
+
+import cv2
+import numpy as np
+
+import config
+
+logger = logging.getLogger(__name__)
+
+
+class Sam3Error(RuntimeError):
+    pass
+
+
+def run_jobs(images: dict, jobs: list, workdir: str) -> dict:
+    """images: {image_key: bgr_ndarray}. jobs: [{"image": key, "prompt": str,
+    "threshold": float}, ...]. Returns {(image_key, prompt): [{"mask": bool
+    ndarray, "score": float, "box": [x1,y1,x2,y2] or None}, ...]}.
+
+    Raises Sam3Error on any failure (missing token, subprocess crash,
+    timeout, malformed response) -- callers treat this the same as any
+    other soft/optional-feature failure (caught, logged, reported back as
+    an error string, never taking the whole request down)."""
+    if not config.HF_TOKEN:
+        raise Sam3Error(
+            "No HF_TOKEN found (checked environment and .env) -- SAM3's "
+            "weights are gated on Hugging Face and can't be downloaded "
+            "without an access-granted token."
+        )
+
+    request_dir = os.path.join(workdir, f"sam3_{uuid.uuid4().hex[:8]}")
+    os.makedirs(request_dir, exist_ok=True)
+    try:
+        image_paths = {}
+        for key, bgr in images.items():
+            path = os.path.join(request_dir, f"{key}.png")
+            cv2.imwrite(path, bgr)
+            image_paths[key] = path
+
+        output_dir = os.path.join(request_dir, "out")
+        request = {"images": image_paths, "jobs": jobs, "output_dir": output_dir}
+        request_path = os.path.join(request_dir, "request.json")
+        with open(request_path, "w") as f:
+            json.dump(request, f)
+
+        env = dict(os.environ)
+        env["HF_TOKEN"] = config.HF_TOKEN
+
+        proc = subprocess.run(
+            [config.SAM3_PYTHON_BIN, config.SAM3_WORKER_SCRIPT, "--request", request_path],
+            capture_output=True, text=True, timeout=config.SAM3_TIMEOUT_SECONDS, env=env,
+        )
+
+        response_path = os.path.join(output_dir, "response.json")
+        if not os.path.isfile(response_path):
+            raise Sam3Error(
+                f"SAM3 worker produced no response (exit {proc.returncode}): "
+                f"{proc.stderr[-2000:] if proc.stderr else '(no stderr)'}"
+            )
+
+        with open(response_path) as f:
+            response = json.load(f)
+
+        if response.get("error"):
+            raise Sam3Error(f"SAM3 worker failed: {response['error'][:2000]}")
+
+        out = {}
+        for entry in response["results"]:
+            key = (entry["image"], entry["prompt"])
+            instances = []
+            for inst in entry["instances"]:
+                mask_path = os.path.join(output_dir, inst["mask_file"])
+                mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE)
+                if mask is None:
+                    continue
+                instances.append({
+                    "mask": mask > 127,
+                    "score": inst["score"],
+                    "box": inst.get("box"),
+                })
+            out[key] = instances
+        return out
+
+    except subprocess.TimeoutExpired as e:
+        raise Sam3Error(f"SAM3 worker timed out after {config.SAM3_TIMEOUT_SECONDS}s") from e
+    finally:
+        shutil.rmtree(request_dir, ignore_errors=True)

pipeline/shape_match.py

@@ -0,0 +1,135 @@
+"""
+Shape matching: compares the overall silhouette/contour of the uploaded
+arrangement against each template -- independent of color and of the local
+keypoint/texture matching SIFT/ORB/SuperGlue/LoFTR do. Two complementary
+signals, both standard, well-established methods (no deep learning needed
+for this):
+
+  - cv2.matchShapes (built on Hu moments): translation/rotation/scale
+    invariant shape-distance between the two contours' raw geometry.
+  - Silhouette IoU after canonical alignment: crop each mask to its own
+    bounding box, resize+center into a fixed canvas, then measure direct
+    pixel overlap -- catches proportion/aspect differences Hu moments can
+    miss, and doubles as the visual side-by-side/overlay image.
+
+Purely informational, its own section: never feeds into any score.
+"""
+
+import logging
+
+import cv2
+import numpy as np
+
+import config
+
+logger = logging.getLogger(__name__)
+
+_template_shape_data = {}  # name -> {"contour": ndarray|None, "canonical_mask": HxW bool}
+
+
+def _largest_contour(mask):
+    contours, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL,
+                                    cv2.CHAIN_APPROX_SIMPLE)
+    if not contours:
+        return None
+    return max(contours, key=cv2.contourArea)
+
+
+def _canonical_silhouette(mask):
+    """Crops to the mask's bounding box, then resizes+centers it into a
+    fixed square canvas preserving aspect ratio -- so silhouettes are
+    directly visually/IoU comparable regardless of the original photo's
+    scale, crop, or resolution."""
+    size = config.SHAPE_CANONICAL_SIZE
+    ys, xs = np.where(mask > 0)
+    if len(ys) == 0:
+        return np.zeros((size, size), dtype=bool)
+
+    y0, y1, x0, x1 = ys.min(), ys.max(), xs.min(), xs.max()
+    cropped = (mask[y0:y1 + 1, x0:x1 + 1] > 0).astype(np.uint8) * 255
+
+    h, w = cropped.shape
+    scale = (size * 0.9) / max(h, w)
+    new_w, new_h = max(1, int(w * scale)), max(1, int(h * scale))
+    resized = cv2.resize(cropped, (new_w, new_h), interpolation=cv2.INTER_NEAREST)
+
+    canvas = np.zeros((size, size), dtype=np.uint8)
+    y_off = (size - new_h) // 2
+    x_off = (size - new_w) // 2
+    canvas[y_off:y_off + new_h, x_off:x_off + new_w] = resized
+    return canvas > 0
+
+
+def compute_shape_data(mask):
+    return {
+        "contour": _largest_contour(mask),
+        "canonical_mask": _canonical_silhouette(mask),
+    }
+
+
+def set_template_shape_data(name, mask):
+    _template_shape_data[name] = compute_shape_data(mask)
+
+
+def _hu_similarity_pct(contour_a, contour_b):
+    if contour_a is None or contour_b is None:
+        return 0.0
+    dist = cv2.matchShapes(contour_a, contour_b, cv2.CONTOURS_MATCH_I1, 0.0)
+    return max(0.0, 100.0 * (1 - dist / config.SHAPE_HU_DISTANCE_SCALE))
+
+
+def _iou_pct(mask_a, mask_b):
+    inter = np.logical_and(mask_a, mask_b).sum()
+    union = np.logical_or(mask_a, mask_b).sum()
+    return (float(inter) / float(union) * 100.0) if union > 0 else 0.0
+
+
+def compare_to_templates(mask):
+    """Returns (input_shape_data, ranked_results) -- results sorted by
+    match_pct descending, one entry per template with the Hu-based and
+    IoU-based sub-scores broken out too."""
+    input_data = compute_shape_data(mask)
+
+    results = []
+    for name, tdata in _template_shape_data.items():
+        hu_sim = _hu_similarity_pct(input_data["contour"], tdata["contour"])
+        iou = _iou_pct(input_data["canonical_mask"], tdata["canonical_mask"])
+        match_pct = round((hu_sim + iou) / 2, 1)
+        results.append({
+            "template": name,
+            "match_pct": match_pct,
+            "hu_similarity_pct": round(hu_sim, 1),
+            "iou_pct": round(iou, 1),
+        })
+    results.sort(key=lambda r: r["match_pct"], reverse=True)
+    return input_data, results
+
+
+# ---------------------------------------------------------------
+# Visuals: two normalized silhouettes side by side + an overlay showing
+# exactly where they agree/diverge.
+# ---------------------------------------------------------------
+
+_BG = (24, 22, 19)
+_INPUT_COLOR = (118, 143, 124)     # BGR -- matches the site's --sage
+_TEMPLATE_COLOR = (90, 122, 185)   # BGR -- matches the site's --terracotta
+_OVERLAP_COLOR = (150, 205, 200)
+
+
+def render_silhouette(canonical_mask, color=_INPUT_COLOR):
+    size = config.SHAPE_CANONICAL_SIZE
+    img = np.full((size, size, 3), _BG, dtype=np.uint8)
+    img[canonical_mask] = color
+    return img
+
+
+def render_overlay(input_mask, template_mask):
+    size = config.SHAPE_CANONICAL_SIZE
+    img = np.full((size, size, 3), _BG, dtype=np.uint8)
+    only_input = input_mask & ~template_mask
+    only_template = template_mask & ~input_mask
+    both = input_mask & template_mask
+    img[only_input] = _INPUT_COLOR
+    img[only_template] = _TEMPLATE_COLOR
+    img[both] = _OVERLAP_COLOR
+    return img

pipeline/texture_match.py

@@ -0,0 +1,134 @@
+"""
+Texture matching: compares surface/material texture -- independent of
+color and of overall silhouette shape. Two standard, complementary
+classical texture descriptors (no deep learning needed):
+
+  - Local Binary Patterns (LBP): encodes each pixel's local micro-pattern
+    relative to its neighbors, compared as a histogram (same convention as
+    pipeline/color.py's hue/saturation histogram) -- good at catching fine,
+    repetitive patterns like fabric weave or petal grain.
+  - GLCM (gray-level co-occurrence matrix) / Haralick features (contrast,
+    homogeneity, energy, correlation) -- coarser statistical texture
+    properties, good at catching smooth-vs-rough, uniform-vs-busy material
+    differences that a local pattern histogram alone can miss.
+
+Purely informational, its own section: never feeds into any score.
+"""
+
+import logging
+
+import cv2
+import numpy as np
+from skimage.feature import graycomatrix, graycoprops, local_binary_pattern
+
+import config
+
+logger = logging.getLogger(__name__)
+
+_LBP_BINS = config.TEXTURE_LBP_POINTS + 2  # "uniform" LBP yields P+2 distinct codes
+
+_template_texture_data = {}  # name -> {"lbp_hist", "glcm_features", "lbp_image", "mask_bool"}
+
+
+def _masked_gray(bgr, mask):
+    gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
+    return gray, mask > 0
+
+
+def _compute_lbp(gray, mask_bool):
+    lbp_image = local_binary_pattern(gray, config.TEXTURE_LBP_POINTS,
+                                      config.TEXTURE_LBP_RADIUS, method="uniform")
+    values = lbp_image[mask_bool]
+    if values.size == 0:
+        return lbp_image, np.zeros(_LBP_BINS, dtype=np.float64)
+    hist, _ = np.histogram(values, bins=_LBP_BINS, range=(0, _LBP_BINS), density=True)
+    return lbp_image, hist
+
+
+def _compute_glcm_features(gray, mask_bool):
+    ys, xs = np.where(mask_bool)
+    if len(ys) == 0:
+        return {p: 0.0 for p in config.TEXTURE_GLCM_PROPS}
+
+    y0, y1, x0, x1 = ys.min(), ys.max() + 1, xs.min(), xs.max() + 1
+    crop_gray = gray[y0:y1, x0:x1].copy()
+    crop_mask = mask_bool[y0:y1, x0:x1]
+
+    levels = config.TEXTURE_GLCM_LEVELS
+    quantized = (crop_gray.astype(np.float32) / 256 * levels).astype(np.uint8)
+    quantized[~crop_mask] = 0  # background -> level 0, excluded from GLCM below
+
+    angles = (0, np.pi / 4, np.pi / 2, 3 * np.pi / 4)
+    glcm = graycomatrix(quantized, distances=list(config.TEXTURE_GLCM_DISTANCES),
+                         angles=list(angles), levels=levels, symmetric=True, normed=True)
+
+    # Exclude any co-occurrence touching the masked-out background level.
+    glcm[0, :, :, :] = 0
+    glcm[:, 0, :, :] = 0
+    total = glcm.sum()
+    if total > 0:
+        glcm = glcm / total
+
+    return {p: float(np.mean(graycoprops(glcm, p))) for p in config.TEXTURE_GLCM_PROPS}
+
+
+def compute_texture_data(bgr, mask):
+    gray, mask_bool = _masked_gray(bgr, mask)
+    lbp_image, lbp_hist = _compute_lbp(gray, mask_bool)
+    glcm_features = _compute_glcm_features(gray, mask_bool)
+    return {
+        "lbp_hist": lbp_hist,
+        "glcm_features": glcm_features,
+        "lbp_image": lbp_image,
+        "mask_bool": mask_bool,
+    }
+
+
+def set_template_texture_data(name, bgr, mask):
+    _template_texture_data[name] = compute_texture_data(bgr, mask)
+
+
+def _hist_similarity_pct(hist_a, hist_b):
+    # Histogram intersection, same convention as pipeline/color.py.
+    return round(float(np.minimum(hist_a, hist_b).sum()) * 100, 1)
+
+
+def _glcm_similarity_pct(features_a, features_b):
+    sims = []
+    for prop in config.TEXTURE_GLCM_PROPS:
+        a, b = features_a[prop], features_b[prop]
+        scale = max(abs(a), abs(b), 1e-9)
+        sims.append(max(0.0, 1 - abs(a - b) / scale))
+    return round(float(np.mean(sims)) * 100, 1)
+
+
+def compare_to_templates(bgr, mask):
+    """Returns (input_texture_data, ranked_results) -- results sorted by
+    match_pct descending, one entry per template with the LBP-based and
+    GLCM-based sub-scores (plus the raw GLCM features) broken out too."""
+    input_data = compute_texture_data(bgr, mask)
+
+    results = []
+    for name, tdata in _template_texture_data.items():
+        lbp_sim = _hist_similarity_pct(input_data["lbp_hist"], tdata["lbp_hist"])
+        glcm_sim = _glcm_similarity_pct(input_data["glcm_features"], tdata["glcm_features"])
+        match_pct = round((lbp_sim + glcm_sim) / 2, 1)
+        results.append({
+            "template": name,
+            "match_pct": match_pct,
+            "lbp_similarity_pct": lbp_sim,
+            "glcm_similarity_pct": glcm_sim,
+            "input_glcm_features": {p: round(v, 3) for p, v in input_data["glcm_features"].items()},
+            "template_glcm_features": {p: round(v, 3) for p, v in tdata["glcm_features"].items()},
+        })
+    results.sort(key=lambda r: r["match_pct"], reverse=True)
+    return input_data, results
+
+
+def render_lbp_visual(lbp_image, mask_bool):
+    """Normalizes the LBP code map to a viewable grayscale image, masked to
+    the foreground only, for the side-by-side visual comparison."""
+    norm = cv2.normalize(lbp_image, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8)
+    out = np.zeros_like(norm)
+    out[mask_bool] = norm[mask_bool]
+    return cv2.cvtColor(out, cv2.COLOR_GRAY2BGR)

pipeline/utils.py

@@ -0,0 +1,95 @@
+import io
+import logging
+import os
+
+import cv2
+from PIL import Image, ImageOps
+
+import config
+
+logger = logging.getLogger(__name__)
+
+
+def list_template_files():
+    if not os.path.isdir(config.TEMPLATE_IMAGES_DIR):
+        raise FileNotFoundError(f"Template folder not found: {config.TEMPLATE_IMAGES_DIR}")
+    return sorted(
+        f for f in os.listdir(config.TEMPLATE_IMAGES_DIR)
+        if f.lower().endswith(config.VALID_EXTS)
+    )
+
+
+def template_name(fname):
+    return os.path.splitext(fname)[0]
+
+
+def annotate_score(bgr_img, score, confidence_pct, label=None):
+    img = bgr_img.copy()
+    text = f"Score: {score}  Conf: {confidence_pct:.1f}%"
+    if label:
+        text = f"{label} | {text}"
+
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    scale = max(0.55, img.shape[1] / 900)
+    thickness = max(1, int(scale * 2))
+
+    (tw, th), baseline = cv2.getTextSize(text, font, scale, thickness)
+    cv2.rectangle(img, (5, 5), (15 + tw, 20 + th + baseline), (20, 20, 20), -1)
+    cv2.putText(img, text, (10, 15 + th), font, scale, (110, 231, 183), thickness,
+                cv2.LINE_AA)
+    return img
+
+
+def encode_png_bytes(bgr_or_bgra_img):
+    ok, buf = cv2.imencode(".png", bgr_or_bgra_img)
+    if not ok:
+        raise RuntimeError("Failed to encode image to PNG")
+    return buf.tobytes()
+
+
+def compress_image_bytes(image_bytes: bytes, max_bytes: int, max_dim: int) -> tuple:
+    """
+    Downscale + re-encode as JPEG only if image_bytes exceeds max_bytes;
+    otherwise returns it untouched -- images already under the limit are
+    never re-compressed, so nothing is lost for the common case.
+
+    Downscaling to max_dim costs no *usable* detail here: the matching
+    pipeline (bg removal, SIFT/ORB/SuperPoint/LoFTR) already caps every
+    image to this same size before processing it, and the external AI
+    verification endpoint's vision model downsamples internally to its own
+    fixed input resolution regardless. This just stops storing/transmitting
+    pixels nothing in the system ever actually looks at.
+
+    Returns (bytes, was_compressed).
+    """
+    if len(image_bytes) <= max_bytes:
+        return image_bytes, False
+
+    original_size = len(image_bytes)
+
+    pil_img = Image.open(io.BytesIO(image_bytes))
+    pil_img = ImageOps.exif_transpose(pil_img)  # bake in camera rotation before resizing
+    pil_img = pil_img.convert("RGB")
+
+    w, h = pil_img.size
+    scale = max_dim / max(w, h)
+    if scale < 1.0:
+        pil_img = pil_img.resize((max(1, int(w * scale)), max(1, int(h * scale))),
+                                  Image.LANCZOS)
+
+    quality = config.COMPRESS_JPEG_QUALITY_START
+    data = None
+    while True:
+        buf = io.BytesIO()
+        pil_img.save(buf, format="JPEG", quality=quality, optimize=True)
+        data = buf.getvalue()
+        if len(data) <= max_bytes or quality <= config.COMPRESS_JPEG_QUALITY_MIN:
+            break
+        quality -= config.COMPRESS_JPEG_QUALITY_STEP
+
+    logger.info(
+        "Compressed upload: %.1f MB -> %.1f MB (%dx%d, JPEG q%d)",
+        original_size / (1024 * 1024), len(data) / (1024 * 1024),
+        pil_img.width, pil_img.height, quality,
+    )
+    return data, True

pipeline/vase_compare.py

@@ -0,0 +1,158 @@
+"""
+Vase-identity comparison: crops the vase region (via SAM3's "vase" concept
+mask -- see pipeline/sam3_client.py) from both the upload and the matched
+template, then compares the two crops with two complementary embedding
+models:
+
+  - DINOv2 (facebook/dinov2-base): self-supervised, patch-level visual
+    features -- good at fine-grained shape/texture/material detail.
+  - CLIP (openai/clip-vit-base-patch32): contrastive image embedding -- a
+    coarser, more holistic notion of visual similarity, used as a second
+    opinion that isn't fooled by the same quirks DINO might be.
+
+Purely informational, opt-in (run alongside the SAM/YOLO-World flower count,
+triggered by the same button) -- never feeds into matching/scoring.
+"""
+
+import gc
+import logging
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+
+import config
+
+logger = logging.getLogger(__name__)
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+_dino_model = None
+_dino_processor = None
+_clip_model = None
+_clip_processor = None
+
+
+def _get_dino():
+    global _dino_model, _dino_processor
+    if _dino_model is None:
+        logger.info("Loading DINOv2 (%s) on %s...", config.DINO_MODEL_NAME, DEVICE)
+        from transformers import AutoImageProcessor, AutoModel
+        _dino_processor = AutoImageProcessor.from_pretrained(config.DINO_MODEL_NAME)
+        _dino_model = AutoModel.from_pretrained(config.DINO_MODEL_NAME).eval().to(DEVICE)
+    return _dino_model, _dino_processor
+
+
+def _get_clip():
+    global _clip_model, _clip_processor
+    if _clip_model is None:
+        logger.info("Loading CLIP (%s) on %s...", config.CLIP_MODEL_NAME, DEVICE)
+        from transformers import CLIPModel, CLIPProcessor
+        _clip_model = CLIPModel.from_pretrained(config.CLIP_MODEL_NAME).eval().to(DEVICE)
+        _clip_processor = CLIPProcessor.from_pretrained(config.CLIP_MODEL_NAME)
+    return _clip_model, _clip_processor
+
+
+def unload_models():
+    global _dino_model, _dino_processor, _clip_model, _clip_processor
+    freed = _dino_model is not None or _clip_model is not None
+    _dino_model = None
+    _dino_processor = None
+    _clip_model = None
+    _clip_processor = None
+    if freed:
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    return freed
+
+
+def _to_pil(bgr):
+    return Image.fromarray(cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB))
+
+
+def _dino_embed(bgr):
+    model, processor = _get_dino()
+    with torch.no_grad():
+        inputs = processor(images=_to_pil(bgr), return_tensors="pt").to(DEVICE)
+        out = model(**inputs)
+        feat = out.last_hidden_state[:, 0]  # CLS token
+        feat = torch.nn.functional.normalize(feat, dim=-1)
+    return feat.cpu().numpy()[0]
+
+
+def _clip_embed(bgr):
+    model, processor = _get_clip()
+    with torch.no_grad():
+        inputs = processor(images=_to_pil(bgr), return_tensors="pt").to(DEVICE)
+        feat = model.get_image_features(**inputs)
+        feat = torch.nn.functional.normalize(feat, dim=-1)
+    return feat.cpu().numpy()[0]
+
+
+def _cosine_pct(a, b):
+    """Cosine similarity of two L2-normalized vectors, clamped to [0, 1] and
+    reported as a percentage -- negative similarity (near-opposite vectors)
+    is clamped to 0% rather than reported as a negative number, since
+    "how similar" isn't meaningful past that point for this use case."""
+    sim = float(np.dot(a, b))
+    return round(max(0.0, min(1.0, sim)) * 100, 1)
+
+
+def crop_mask(bgr, mask_bool):
+    """Crops the bounding box of mask_bool out of bgr, with a small pad
+    (config.VASE_CROP_PAD_FRAC) so the mask's own (occasionally imprecise)
+    edge doesn't cut off the vase's rim or base -- then blacks out any
+    pixel inside that padded box that the mask doesn't cover, so DINOv2/CLIP
+    only ever see vase pixels, not whatever flower stems or ribbon happen to
+    share the box's corners. Returns None if the mask is empty."""
+    ys, xs = np.nonzero(mask_bool)
+    if len(xs) == 0:
+        return None
+    h, w = bgr.shape[:2]
+    x1, y1, x2, y2 = xs.min(), ys.min(), xs.max(), ys.max()
+    bw, bh = x2 - x1, y2 - y1
+    pad = config.VASE_CROP_PAD_FRAC
+    px1, px2 = x1 - bw * pad, x2 + bw * pad
+    py1, py2 = y1 - bh * pad, y2 + bh * pad
+    px1, py1 = max(0, int(round(px1))), max(0, int(round(py1)))
+    px2, py2 = min(w, int(round(px2))), min(h, int(round(py2)))
+    if px2 <= px1 or py2 <= py1:
+        return None
+
+    crop = bgr[py1:py2, px1:px2].copy()
+    mask_crop = mask_bool[py1:py2, px1:px2]
+    crop[~mask_crop] = 0
+    return crop
+
+
+def clip_similarity_pct(crop_a, crop_b):
+    """CLIP-only similarity between two crops -- no DINOv2. Used for the
+    auto-triggered "flower check" beside the weighted verdict (see
+    engine.flower_summary): unlike the opt-in vase comparison, that one runs
+    on every confident match, so it deliberately loads only CLIP (small,
+    fast) rather than both embedding models."""
+    return _cosine_pct(_clip_embed(crop_a), _clip_embed(crop_b))
+
+
+def compare_vases(input_crop_bgr, template_crop_bgr):
+    dino_pct = _cosine_pct(_dino_embed(input_crop_bgr), _dino_embed(template_crop_bgr))
+    clip_pct = _cosine_pct(_clip_embed(input_crop_bgr), _clip_embed(template_crop_bgr))
+    combined_pct = round(
+        config.VASE_DINO_WEIGHT * dino_pct + config.VASE_CLIP_WEIGHT * clip_pct, 1
+    )
+
+    if combined_pct >= config.VASE_SAME_THRESHOLD:
+        verdict = "same"
+    elif combined_pct >= config.VASE_UNCERTAIN_THRESHOLD:
+        verdict = "uncertain"
+    else:
+        verdict = "different"
+
+    return {
+        "dino_similarity_pct": dino_pct,
+        "clip_similarity_pct": clip_pct,
+        "combined_pct": combined_pct,
+        "verdict": verdict,
+    }

pipeline/verify.py

@@ -0,0 +1,76 @@
+"""
+Optional third-party AI verification, layered on top of the core
+feature-matching pipeline: sends the winning template's original photo and
+the user's original upload to an external vision-LLM endpoint that runs a
+detailed QC-style comparison (flowers, vase, ribbon, composition) and
+returns a MATCH/DISCREPANCIES/CONFIDENCE verdict plus free-text description.
+
+This never affects the core match result -- if the endpoint is slow, down,
+or unreachable (it's a Cloudflare tunnel, which can go stale), the caller
+is expected to treat any exception here as a soft failure.
+"""
+
+import logging
+import os
+import re
+
+import requests
+
+import config
+
+logger = logging.getLogger(__name__)
+
+_VERDICT_RE = re.compile(
+    # DISCREP\w* rather than a literal "DISCREPANCIES" -- the LLM behind
+    # the endpoint doesn't reliably spell it the same way every time
+    # ("DISCREPANCIES" vs "DISCREPENCIES" have both been observed), and a
+    # missed match here silently dumps the whole raw block into the UI.
+    r"MATCH:\s*\[?\s*(?P<match>YES|NO|PARTIAL)\s*\]?\s*"
+    r"DISCREP\w*:\s*\[?\s*(?P<discrepancies>.*?)\s*\]?\s*"
+    r"CONFIDENCE:\s*\[?\s*(?P<confidence>High|Medium|Low)\s*\]?",
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+def _parse_result_text(text: str) -> dict:
+    """The endpoint's `result` field is free text with an embedded
+    MATCH/DISCREPANCIES/CONFIDENCE block, sometimes followed by a prose
+    description, sometimes not. Pull out the structured bits; whatever's
+    left over (if anything) is the description."""
+    text = text or ""
+    m = _VERDICT_RE.search(text)
+
+    if not m:
+        return {"match": None, "confidence": None, "discrepancies": None,
+                "description": text.strip()}
+
+    return {
+        "match": m.group("match").upper(),
+        "confidence": m.group("confidence").capitalize(),
+        "discrepancies": m.group("discrepancies").strip() or "None",
+        "description": text[m.end():].strip(),
+    }
+
+
+def verify_images(reference_path: str, actual_path: str) -> dict:
+    """reference_path = the matched template's original photo (IMAGE 1),
+    actual_path = the user's original upload (IMAGE 2)."""
+    with open(reference_path, "rb") as f1, open(actual_path, "rb") as f2:
+        files = {
+            "image1": (os.path.basename(reference_path), f1, "image/png"),
+            "image2": (os.path.basename(actual_path), f2, "image/jpeg"),
+        }
+        response = requests.post(config.VERIFY_ENDPOINT_URL, files=files,
+                                  timeout=config.VERIFY_TIMEOUT_SECONDS)
+    response.raise_for_status()
+    data = response.json()
+
+    parsed = _parse_result_text(data.get("result", ""))
+    return {
+        "match": parsed["match"],
+        "confidence": parsed["confidence"],
+        "discrepancies": parsed["discrepancies"],
+        "description": parsed["description"],
+        "raw_result": data.get("result"),
+        "pixel_precheck": data.get("pixel_precheck"),
+    }

pipeline/yolo_world.py

@@ -0,0 +1,94 @@
+"""
+Open-vocabulary object detection via YOLO-World (YOLOv8 family, ultralytics),
+run alongside SAM (see pipeline/flower_count.py) for the flower-count feature.
+
+SAM has no notion of "flower" -- it blindly proposes every objectlike region
+it can find, so the vase and any ribbon/bow tied around it get segmented and
+counted as if they were flowers. YOLO-World, prompted with plain-language
+classes, actually knows what those things look like: its "vase"/"ribbon"/
+"bow" detections are used (in engine.count_flowers) to exclude those regions
+from SAM's flower-instance count.
+
+Its own "flower" boxes are shown to the user as a second, independent count
+-- in practice it tends to draw one box per contiguous flower region rather
+than per individual bloom, so it's a coarser, corroborating signal, not a
+replacement for SAM's per-instance count.
+"""
+
+import gc
+import logging
+
+import cv2
+import torch
+
+import config
+
+logger = logging.getLogger(__name__)
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+_model = None
+
+
+def get_model():
+    global _model
+    if _model is None:
+        logger.info("Loading YOLO-World (%s) on %s...", config.YOLO_WORLD_CHECKPOINT, DEVICE)
+        from ultralytics import YOLO
+        _model = YOLO(config.YOLO_WORLD_CHECKPOINT)
+        _model.set_classes(config.YOLO_WORLD_CLASSES)
+    return _model
+
+
+def unload_model():
+    global _model
+    freed = _model is not None
+    _model = None
+    if freed:
+        gc.collect()
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    return freed
+
+
+def detect(bgr):
+    """Returns {class_name: [{"xyxy": [x1,y1,x2,y2], "confidence": float}, ...]}
+    for every class in config.YOLO_WORLD_CLASSES (empty list if none found)."""
+    model = get_model()
+    with torch.no_grad():
+        results = model.predict(bgr, conf=config.YOLO_WORLD_CONF,
+                                 iou=config.YOLO_WORLD_IOU, verbose=False)
+    r = results[0]
+    by_class = {name: [] for name in config.YOLO_WORLD_CLASSES}
+    for b in r.boxes:
+        name = r.names[int(b.cls[0])]
+        by_class.setdefault(name, []).append({
+            "xyxy": [float(v) for v in b.xyxy[0].tolist()],
+            "confidence": round(float(b.conf[0]), 3),
+        })
+    return by_class
+
+
+_BOX_COLORS = {
+    "flower": (66, 133, 244), "vase": (219, 68, 55),
+    "ribbon": (244, 180, 0), "bow": (15, 157, 88),
+}
+
+
+def render_boxes(bgr, by_class):
+    """Draws a labeled box per detection, colored by class, so the user can
+    see exactly which regions YOLO-World identified as flower vs. vase vs.
+    ribbon/bow."""
+    overlay = bgr.copy()
+    for name, boxes in by_class.items():
+        color = _BOX_COLORS.get(name, (170, 170, 170))
+        for box in boxes:
+            x1, y1, x2, y2 = [int(v) for v in box["xyxy"]]
+            cv2.rectangle(overlay, (x1, y1), (x2, y2), color, 3)
+            label = f"{name} {box['confidence']:.2f}"
+            (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2)
+            label_y = max(th + 6, y1)
+            cv2.rectangle(overlay, (x1, label_y - th - 8), (x1 + tw + 6, label_y), color, -1)
+            cv2.putText(overlay, label, (x1 + 3, label_y - 5),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2, cv2.LINE_AA)
+    return overlay