Fix duplicate alerts, telemetry UI state, and deploy_agent local URL

This commit is contained in:
2026-06-08 15:36:59 +05:30
parent 647c6ae7f0
commit 6e05180e7c
3 changed files with 42 additions and 39 deletions

View File

@@ -1,3 +1,3 @@
ROCKETCHAT_WEBHOOK_URL="" ROCKETCHAT_WEBHOOK_URL="https://text.seekright.com/hooks/6a22ad73a88367bc6e1095a2/xcHkbDSce5rCwofdBZF3ri2rb6ntxT2mahqT6Gg7iKdectYk"
MONGODB_URI="mongodb://localhost:27017" MONGODB_URI="mongodb://localhost:27017"
MONGODB_DB="rmm_db_dev" MONGODB_DB="rmm_db_dev"

View File

@@ -52,12 +52,8 @@ MAX_ENTRIES_PER_CLIENT = 100 # Keep the latest 100 historical logs per client n
ROCKETCHAT_WEBHOOK_URL = os.getenv("ROCKETCHAT_WEBHOOK_URL", "") ROCKETCHAT_WEBHOOK_URL = os.getenv("ROCKETCHAT_WEBHOOK_URL", "")
# In-memory storage to prevent duplicate/spam storage notifications # Alert states are now stored directly in the MongoDB client documents
active_storage_alerts = set() # to support multi-worker environments safely.
# Tracks the timestamp when a System DOWN notification was last sent for a client ID
# Key: client_id (str), Value: datetime object of the last notification time
last_down_notification_times = {}
def send_rocketchat_notification(text: str, color: str = "#808080"): def send_rocketchat_notification(text: str, color: str = "#808080"):
""" """
@@ -127,12 +123,12 @@ def mark_client_active(client_id: str, clients: Dict[str, Any]):
if client_id in clients: if client_id in clients:
was_active = clients[client_id].get("active", False) was_active = clients[client_id].get("active", False)
if not was_active: if not was_active:
if client_id in last_down_notification_times: if clients[client_id].get("last_down_alert_time"):
send_rocketchat_notification( send_rocketchat_notification(
text=f"✅ **System UP:** Client `{client_id}` has recovered and is back online.", text=f"✅ **System UP:** Client `{client_id}` has recovered and is back online.",
color="#2ecc71" color="#2ecc71"
) )
last_down_notification_times.pop(client_id, None) clients[client_id]["last_down_alert_time"] = None
clients[client_id]["active"] = True clients[client_id]["active"] = True
clients[client_id]["last_seen"] = datetime.now().isoformat() clients[client_id]["last_seen"] = datetime.now().isoformat()
@@ -142,15 +138,7 @@ def load_clients() -> Dict[str, Any]:
clients_collection.delete_many({"_id": {"$in": ["", None]}}) clients_collection.delete_many({"_id": {"$in": ["", None]}})
clients_collection.delete_many({"_id": {"$regex": "^\\s*$"}}) clients_collection.delete_many({"_id": {"$regex": "^\\s*$"}})
# If collection is empty, seed it with default registry
if clients_collection.count_documents({}) == 0:
default_data = {
"client_1": {"pending_command": "none", "last_seen": None, "active": False},
"client_2": {"pending_command": "none", "last_seen": None, "active": False},
"client_3": {"pending_command": "none", "last_seen": None, "active": False}
}
save_clients(default_data)
return default_data
# Load all documents # Load all documents
cursor = clients_collection.find() cursor = clients_collection.find()
@@ -181,16 +169,16 @@ def load_clients() -> Dict[str, Any]:
# Active status transitions alerts # Active status transitions alerts
if is_active: if is_active:
# Transitions from offline (DOWN) to online (UP) # Transitions from offline (DOWN) to online (UP)
if cid in last_down_notification_times: if info.get("last_down_alert_time"):
send_rocketchat_notification( send_rocketchat_notification(
text=f"✅ **System UP:** Client `{cid}` has recovered and is back online.", text=f"✅ **System UP:** Client `{cid}` has recovered and is back online.",
color="#2ecc71" color="#2ecc71"
) )
last_down_notification_times.pop(cid, None) info["last_down_alert_time"] = None
else: else:
# Transitions from online (UP) to offline (DOWN) for the first time # Transitions from online (UP) to offline (DOWN) for the first time
now = datetime.now() now = datetime.now()
last_down_notification_times[cid] = now info["last_down_alert_time"] = now.isoformat()
send_rocketchat_notification( send_rocketchat_notification(
text=f"🚨 **System DOWN:** Client `{cid}` has missed heartbeats for over 30 seconds.", text=f"🚨 **System DOWN:** Client `{cid}` has missed heartbeats for over 30 seconds.",
color="#e74c3c" color="#e74c3c"
@@ -199,17 +187,23 @@ def load_clients() -> Dict[str, Any]:
# No transition (state remains the same) # No transition (state remains the same)
if not is_active: if not is_active:
# Client remains offline. Check if we should send a repeating reminder. # Client remains offline. Check if we should send a repeating reminder.
# We ONLY send a reminder if it is already tracked in last_down_notification_times. # We ONLY send a reminder if it is already tracked in last_down_alert_time.
# This prevents sending alerts for historically offline systems at server boot! # This prevents sending alerts for historically offline systems at server boot!
if cid in last_down_notification_times: last_down_str = info.get("last_down_alert_time")
if last_down_str:
now = datetime.now() now = datetime.now()
elapsed_seconds = (now - last_down_notification_times[cid]).total_seconds() try:
last_down_dt = datetime.fromisoformat(last_down_str)
elapsed_seconds = (now - last_down_dt).total_seconds()
if elapsed_seconds >= 3600: # 1 hour (3600 seconds) if elapsed_seconds >= 3600: # 1 hour (3600 seconds)
last_down_notification_times[cid] = now info["last_down_alert_time"] = now.isoformat()
send_rocketchat_notification( send_rocketchat_notification(
text=f"🚨 **System STILL DOWN:** Client `{cid}` remains offline (reminder sent every hour).", text=f"🚨 **System STILL DOWN:** Client `{cid}` remains offline (reminder sent every hour).",
color="#e74c3c" color="#e74c3c"
) )
modified = True
except Exception:
pass
if modified: if modified:
save_clients(data) save_clients(data)
@@ -329,32 +323,40 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload):
# Update last_seen timestamp, active status, and store the current telemetry details # Update last_seen timestamp, active status, and store the current telemetry details
mark_client_active(client_id, clients) mark_client_active(client_id, clients)
clients[client_id]["telemetry"] = payload.dict() clients[client_id]["telemetry"] = payload.dict()
save_clients(clients)
# Log telemetry history event # Log telemetry history event
append_to_logs("telemetry", client_id, payload.dict()) append_to_logs("telemetry", client_id, payload.dict())
# Storage limits warning checker # Storage limits warning checker
storage_alerts = clients[client_id].get("active_storage_alerts", [])
storage_modified = False
for disk in payload.disks: for disk in payload.disks:
mount = disk.get("mount", "/") mount = disk.get("mount", "/")
percent = disk.get("percent", 0.0) percent = disk.get("percent", 0.0)
alert_key = f"{client_id}:{mount}"
if percent >= 80.0: if percent >= 80.0:
if alert_key not in active_storage_alerts: if mount not in storage_alerts:
active_storage_alerts.add(alert_key) storage_alerts.append(mount)
storage_modified = True
send_rocketchat_notification( send_rocketchat_notification(
text=f"⚠️ **Storage Warning:** Client `{client_id}` disk `{mount}` is at **{percent}%** capacity.", text=f"⚠️ **Storage Warning:** Client `{client_id}` disk `{mount}` is at **{percent}%** capacity.",
color="#f39c12" color="#f39c12"
) )
else: else:
if alert_key in active_storage_alerts: if mount in storage_alerts:
active_storage_alerts.remove(alert_key) storage_alerts.remove(mount)
storage_modified = True
send_rocketchat_notification( send_rocketchat_notification(
text=f"✅ **Storage Recovered:** Client `{client_id}` disk `{mount}` has cleared warning state and is at **{percent}%**.", text=f"✅ **Storage Recovered:** Client `{client_id}` disk `{mount}` has cleared warning state and is at **{percent}%**.",
color="#2ecc71" color="#2ecc71"
) )
if storage_modified:
clients[client_id]["active_storage_alerts"] = storage_alerts
save_clients(clients)
data = payload.dict() data = payload.dict()
client_telemetry[client_id] = data client_telemetry[client_id] = data
@@ -378,7 +380,8 @@ async def get_all_telemetry():
""" """
Endpoint to view the live dashboard data of all clients. Endpoint to view the live dashboard data of all clients.
""" """
return client_telemetry clients = load_clients()
return {cid: info.get("telemetry", {}) for cid, info in clients.items() if info.get("telemetry")}
@app.get("/api/clients") @app.get("/api/clients")
async def get_clients_api(): async def get_clients_api():

View File

@@ -16,13 +16,13 @@ CLIENT_ID=""
# For Production deployment, use: "http://rmm-backend.seekright.com" # For Production deployment, use: "http://rmm-backend.seekright.com"
# For Local WSL testing on your laptop, use: "http://172.18.16.1:8000" # For Local WSL testing on your laptop, use: "http://172.18.16.1:8000"
CENTRAL_URL="http://rmm-backend.seekright.com"
# Check if Client ID is set; if not, check command line arguments, otherwise default to hostname # Check if Client ID is set; if not, check command line arguments, otherwise default to hostname
if [ -z "$CLIENT_ID" ]; then if [ -z "$CLIENT_ID" ]; then
CLIENT_ID="${1:-$(hostname)}" CLIENT_ID="${1:-$(hostname)}"
fi fi
CENTRAL_URL="${2:-http://rmm-backend.seekright.com}"
echo "🚀 Preparing SeekRight RMM Agent installation..." echo "🚀 Preparing SeekRight RMM Agent installation..."
echo "📍 Target Client ID: $CLIENT_ID" echo "📍 Target Client ID: $CLIENT_ID"
echo "🌐 Central Server URL: $CENTRAL_URL" echo "🌐 Central Server URL: $CENTRAL_URL"