Fix duplicate alerts, telemetry UI state, and deploy_agent local URL
This commit is contained in:
@@ -1,3 +1,3 @@
|
||||
ROCKETCHAT_WEBHOOK_URL=""
|
||||
ROCKETCHAT_WEBHOOK_URL="https://text.seekright.com/hooks/6a22ad73a88367bc6e1095a2/xcHkbDSce5rCwofdBZF3ri2rb6ntxT2mahqT6Gg7iKdectYk"
|
||||
MONGODB_URI="mongodb://localhost:27017"
|
||||
MONGODB_DB="rmm_db_dev"
|
||||
|
||||
@@ -52,12 +52,8 @@ MAX_ENTRIES_PER_CLIENT = 100 # Keep the latest 100 historical logs per client n
|
||||
|
||||
ROCKETCHAT_WEBHOOK_URL = os.getenv("ROCKETCHAT_WEBHOOK_URL", "")
|
||||
|
||||
# In-memory storage to prevent duplicate/spam storage notifications
|
||||
active_storage_alerts = set()
|
||||
|
||||
# Tracks the timestamp when a System DOWN notification was last sent for a client ID
|
||||
# Key: client_id (str), Value: datetime object of the last notification time
|
||||
last_down_notification_times = {}
|
||||
# Alert states are now stored directly in the MongoDB client documents
|
||||
# to support multi-worker environments safely.
|
||||
|
||||
def send_rocketchat_notification(text: str, color: str = "#808080"):
|
||||
"""
|
||||
@@ -127,12 +123,12 @@ def mark_client_active(client_id: str, clients: Dict[str, Any]):
|
||||
if client_id in clients:
|
||||
was_active = clients[client_id].get("active", False)
|
||||
if not was_active:
|
||||
if client_id in last_down_notification_times:
|
||||
if clients[client_id].get("last_down_alert_time"):
|
||||
send_rocketchat_notification(
|
||||
text=f"✅ **System UP:** Client `{client_id}` has recovered and is back online.",
|
||||
color="#2ecc71"
|
||||
)
|
||||
last_down_notification_times.pop(client_id, None)
|
||||
clients[client_id]["last_down_alert_time"] = None
|
||||
clients[client_id]["active"] = True
|
||||
clients[client_id]["last_seen"] = datetime.now().isoformat()
|
||||
|
||||
@@ -142,15 +138,7 @@ def load_clients() -> Dict[str, Any]:
|
||||
clients_collection.delete_many({"_id": {"$in": ["", None]}})
|
||||
clients_collection.delete_many({"_id": {"$regex": "^\\s*$"}})
|
||||
|
||||
# If collection is empty, seed it with default registry
|
||||
if clients_collection.count_documents({}) == 0:
|
||||
default_data = {
|
||||
"client_1": {"pending_command": "none", "last_seen": None, "active": False},
|
||||
"client_2": {"pending_command": "none", "last_seen": None, "active": False},
|
||||
"client_3": {"pending_command": "none", "last_seen": None, "active": False}
|
||||
}
|
||||
save_clients(default_data)
|
||||
return default_data
|
||||
|
||||
|
||||
# Load all documents
|
||||
cursor = clients_collection.find()
|
||||
@@ -181,16 +169,16 @@ def load_clients() -> Dict[str, Any]:
|
||||
# Active status transitions alerts
|
||||
if is_active:
|
||||
# Transitions from offline (DOWN) to online (UP)
|
||||
if cid in last_down_notification_times:
|
||||
if info.get("last_down_alert_time"):
|
||||
send_rocketchat_notification(
|
||||
text=f"✅ **System UP:** Client `{cid}` has recovered and is back online.",
|
||||
color="#2ecc71"
|
||||
)
|
||||
last_down_notification_times.pop(cid, None)
|
||||
info["last_down_alert_time"] = None
|
||||
else:
|
||||
# Transitions from online (UP) to offline (DOWN) for the first time
|
||||
now = datetime.now()
|
||||
last_down_notification_times[cid] = now
|
||||
info["last_down_alert_time"] = now.isoformat()
|
||||
send_rocketchat_notification(
|
||||
text=f"🚨 **System DOWN:** Client `{cid}` has missed heartbeats for over 30 seconds.",
|
||||
color="#e74c3c"
|
||||
@@ -199,17 +187,23 @@ def load_clients() -> Dict[str, Any]:
|
||||
# No transition (state remains the same)
|
||||
if not is_active:
|
||||
# Client remains offline. Check if we should send a repeating reminder.
|
||||
# We ONLY send a reminder if it is already tracked in last_down_notification_times.
|
||||
# We ONLY send a reminder if it is already tracked in last_down_alert_time.
|
||||
# This prevents sending alerts for historically offline systems at server boot!
|
||||
if cid in last_down_notification_times:
|
||||
last_down_str = info.get("last_down_alert_time")
|
||||
if last_down_str:
|
||||
now = datetime.now()
|
||||
elapsed_seconds = (now - last_down_notification_times[cid]).total_seconds()
|
||||
try:
|
||||
last_down_dt = datetime.fromisoformat(last_down_str)
|
||||
elapsed_seconds = (now - last_down_dt).total_seconds()
|
||||
if elapsed_seconds >= 3600: # 1 hour (3600 seconds)
|
||||
last_down_notification_times[cid] = now
|
||||
info["last_down_alert_time"] = now.isoformat()
|
||||
send_rocketchat_notification(
|
||||
text=f"🚨 **System STILL DOWN:** Client `{cid}` remains offline (reminder sent every hour).",
|
||||
color="#e74c3c"
|
||||
)
|
||||
modified = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if modified:
|
||||
save_clients(data)
|
||||
@@ -329,32 +323,40 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload):
|
||||
# Update last_seen timestamp, active status, and store the current telemetry details
|
||||
mark_client_active(client_id, clients)
|
||||
clients[client_id]["telemetry"] = payload.dict()
|
||||
save_clients(clients)
|
||||
|
||||
# Log telemetry history event
|
||||
append_to_logs("telemetry", client_id, payload.dict())
|
||||
|
||||
# Storage limits warning checker
|
||||
storage_alerts = clients[client_id].get("active_storage_alerts", [])
|
||||
storage_modified = False
|
||||
|
||||
for disk in payload.disks:
|
||||
mount = disk.get("mount", "/")
|
||||
percent = disk.get("percent", 0.0)
|
||||
alert_key = f"{client_id}:{mount}"
|
||||
|
||||
if percent >= 80.0:
|
||||
if alert_key not in active_storage_alerts:
|
||||
active_storage_alerts.add(alert_key)
|
||||
if mount not in storage_alerts:
|
||||
storage_alerts.append(mount)
|
||||
storage_modified = True
|
||||
send_rocketchat_notification(
|
||||
text=f"⚠️ **Storage Warning:** Client `{client_id}` disk `{mount}` is at **{percent}%** capacity.",
|
||||
color="#f39c12"
|
||||
)
|
||||
else:
|
||||
if alert_key in active_storage_alerts:
|
||||
active_storage_alerts.remove(alert_key)
|
||||
if mount in storage_alerts:
|
||||
storage_alerts.remove(mount)
|
||||
storage_modified = True
|
||||
send_rocketchat_notification(
|
||||
text=f"✅ **Storage Recovered:** Client `{client_id}` disk `{mount}` has cleared warning state and is at **{percent}%**.",
|
||||
color="#2ecc71"
|
||||
)
|
||||
|
||||
if storage_modified:
|
||||
clients[client_id]["active_storage_alerts"] = storage_alerts
|
||||
|
||||
save_clients(clients)
|
||||
|
||||
data = payload.dict()
|
||||
client_telemetry[client_id] = data
|
||||
|
||||
@@ -378,7 +380,8 @@ async def get_all_telemetry():
|
||||
"""
|
||||
Endpoint to view the live dashboard data of all clients.
|
||||
"""
|
||||
return client_telemetry
|
||||
clients = load_clients()
|
||||
return {cid: info.get("telemetry", {}) for cid, info in clients.items() if info.get("telemetry")}
|
||||
|
||||
@app.get("/api/clients")
|
||||
async def get_clients_api():
|
||||
|
||||
@@ -16,13 +16,13 @@ CLIENT_ID=""
|
||||
|
||||
# For Production deployment, use: "http://rmm-backend.seekright.com"
|
||||
# For Local WSL testing on your laptop, use: "http://172.18.16.1:8000"
|
||||
CENTRAL_URL="http://rmm-backend.seekright.com"
|
||||
|
||||
# Check if Client ID is set; if not, check command line arguments, otherwise default to hostname
|
||||
if [ -z "$CLIENT_ID" ]; then
|
||||
CLIENT_ID="${1:-$(hostname)}"
|
||||
fi
|
||||
|
||||
CENTRAL_URL="${2:-http://rmm-backend.seekright.com}"
|
||||
|
||||
echo "🚀 Preparing SeekRight RMM Agent installation..."
|
||||
echo "📍 Target Client ID: $CLIENT_ID"
|
||||
echo "🌐 Central Server URL: $CENTRAL_URL"
|
||||
|
||||
Reference in New Issue
Block a user