diff --git a/central_api_prototype.py b/central_api_prototype.py index d515ceb..135ba63 100644 --- a/central_api_prototype.py +++ b/central_api_prototype.py @@ -354,8 +354,33 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload): if storage_modified: clients[client_id]["active_storage_alerts"] = storage_alerts + + # GPU Failure Monitor + gpu_alert_sent = clients[client_id].get("gpu_alert_sent", False) + gpu_modified = False + + if len(payload.gpus) == 0: + if not gpu_alert_sent: + gpu_alert_sent = True + gpu_modified = True + send_rocketchat_notification( + text=f"🚨 **GPU Failure:** Client `{client_id}` is not reporting any GPU data! (nvidia-smi has failed or is missing)", + color="#e74c3c" + ) + else: + if gpu_alert_sent: + gpu_alert_sent = False + gpu_modified = True + send_rocketchat_notification( + text=f"✅ **GPU Recovered:** Client `{client_id}` is reporting GPU data successfully again.", + color="#2ecc71" + ) + + if gpu_modified: + clients[client_id]["gpu_alert_sent"] = gpu_alert_sent - save_clients(clients) + if storage_modified or gpu_modified: + save_clients(clients) data = payload.dict() client_telemetry[client_id] = data