feat: add GPU failure monitoring and Rocket.Chat alerts to client telemetry processing
This commit is contained in:
@@ -354,8 +354,33 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload):
|
||||
|
||||
if storage_modified:
|
||||
clients[client_id]["active_storage_alerts"] = storage_alerts
|
||||
|
||||
# GPU Failure Monitor
|
||||
gpu_alert_sent = clients[client_id].get("gpu_alert_sent", False)
|
||||
gpu_modified = False
|
||||
|
||||
if len(payload.gpus) == 0:
|
||||
if not gpu_alert_sent:
|
||||
gpu_alert_sent = True
|
||||
gpu_modified = True
|
||||
send_rocketchat_notification(
|
||||
text=f"🚨 **GPU Failure:** Client `{client_id}` is not reporting any GPU data! (nvidia-smi has failed or is missing)",
|
||||
color="#e74c3c"
|
||||
)
|
||||
else:
|
||||
if gpu_alert_sent:
|
||||
gpu_alert_sent = False
|
||||
gpu_modified = True
|
||||
send_rocketchat_notification(
|
||||
text=f"✅ **GPU Recovered:** Client `{client_id}` is reporting GPU data successfully again.",
|
||||
color="#2ecc71"
|
||||
)
|
||||
|
||||
if gpu_modified:
|
||||
clients[client_id]["gpu_alert_sent"] = gpu_alert_sent
|
||||
|
||||
save_clients(clients)
|
||||
if storage_modified or gpu_modified:
|
||||
save_clients(clients)
|
||||
|
||||
data = payload.dict()
|
||||
client_telemetry[client_id] = data
|
||||
|
||||
Reference in New Issue
Block a user