feat: add GPU failure monitoring and Rocket.Chat alerts to client telemetry processing
This commit is contained in:
@@ -354,8 +354,33 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload):
|
|||||||
|
|
||||||
if storage_modified:
|
if storage_modified:
|
||||||
clients[client_id]["active_storage_alerts"] = storage_alerts
|
clients[client_id]["active_storage_alerts"] = storage_alerts
|
||||||
|
|
||||||
|
# GPU Failure Monitor
|
||||||
|
gpu_alert_sent = clients[client_id].get("gpu_alert_sent", False)
|
||||||
|
gpu_modified = False
|
||||||
|
|
||||||
|
if len(payload.gpus) == 0:
|
||||||
|
if not gpu_alert_sent:
|
||||||
|
gpu_alert_sent = True
|
||||||
|
gpu_modified = True
|
||||||
|
send_rocketchat_notification(
|
||||||
|
text=f"🚨 **GPU Failure:** Client `{client_id}` is not reporting any GPU data! (nvidia-smi has failed or is missing)",
|
||||||
|
color="#e74c3c"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if gpu_alert_sent:
|
||||||
|
gpu_alert_sent = False
|
||||||
|
gpu_modified = True
|
||||||
|
send_rocketchat_notification(
|
||||||
|
text=f"✅ **GPU Recovered:** Client `{client_id}` is reporting GPU data successfully again.",
|
||||||
|
color="#2ecc71"
|
||||||
|
)
|
||||||
|
|
||||||
|
if gpu_modified:
|
||||||
|
clients[client_id]["gpu_alert_sent"] = gpu_alert_sent
|
||||||
|
|
||||||
save_clients(clients)
|
if storage_modified or gpu_modified:
|
||||||
|
save_clients(clients)
|
||||||
|
|
||||||
data = payload.dict()
|
data = payload.dict()
|
||||||
client_telemetry[client_id] = data
|
client_telemetry[client_id] = data
|
||||||
|
|||||||
Reference in New Issue
Block a user