feat: add GPU failure monitoring and Rocket.Chat alerts to client telemetry processing

This commit is contained in:
2026-06-09 18:08:24 +05:30
parent 6e05180e7c
commit 9baf93bfef

View File

@@ -354,8 +354,33 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload):
if storage_modified: if storage_modified:
clients[client_id]["active_storage_alerts"] = storage_alerts clients[client_id]["active_storage_alerts"] = storage_alerts
# GPU Failure Monitor
gpu_alert_sent = clients[client_id].get("gpu_alert_sent", False)
gpu_modified = False
if len(payload.gpus) == 0:
if not gpu_alert_sent:
gpu_alert_sent = True
gpu_modified = True
send_rocketchat_notification(
text=f"🚨 **GPU Failure:** Client `{client_id}` is not reporting any GPU data! (nvidia-smi has failed or is missing)",
color="#e74c3c"
)
else:
if gpu_alert_sent:
gpu_alert_sent = False
gpu_modified = True
send_rocketchat_notification(
text=f"✅ **GPU Recovered:** Client `{client_id}` is reporting GPU data successfully again.",
color="#2ecc71"
)
if gpu_modified:
clients[client_id]["gpu_alert_sent"] = gpu_alert_sent
save_clients(clients) if storage_modified or gpu_modified:
save_clients(clients)
data = payload.dict() data = payload.dict()
client_telemetry[client_id] = data client_telemetry[client_id] = data