diff --git a/.env.development b/.env.development index e16f5fa..7aab897 100644 --- a/.env.development +++ b/.env.development @@ -1,3 +1,3 @@ -ROCKETCHAT_WEBHOOK_URL="" +ROCKETCHAT_WEBHOOK_URL="https://text.seekright.com/hooks/6a22ad73a88367bc6e1095a2/xcHkbDSce5rCwofdBZF3ri2rb6ntxT2mahqT6Gg7iKdectYk" MONGODB_URI="mongodb://localhost:27017" MONGODB_DB="rmm_db_dev" diff --git a/central_api_prototype.py b/central_api_prototype.py index fb5c4f9..d515ceb 100644 --- a/central_api_prototype.py +++ b/central_api_prototype.py @@ -52,12 +52,8 @@ MAX_ENTRIES_PER_CLIENT = 100 # Keep the latest 100 historical logs per client n ROCKETCHAT_WEBHOOK_URL = os.getenv("ROCKETCHAT_WEBHOOK_URL", "") -# In-memory storage to prevent duplicate/spam storage notifications -active_storage_alerts = set() - -# Tracks the timestamp when a System DOWN notification was last sent for a client ID -# Key: client_id (str), Value: datetime object of the last notification time -last_down_notification_times = {} +# Alert states are now stored directly in the MongoDB client documents +# to support multi-worker environments safely. def send_rocketchat_notification(text: str, color: str = "#808080"): """ @@ -127,12 +123,12 @@ def mark_client_active(client_id: str, clients: Dict[str, Any]): if client_id in clients: was_active = clients[client_id].get("active", False) if not was_active: - if client_id in last_down_notification_times: + if clients[client_id].get("last_down_alert_time"): send_rocketchat_notification( text=f"✅ **System UP:** Client `{client_id}` has recovered and is back online.", color="#2ecc71" ) - last_down_notification_times.pop(client_id, None) + clients[client_id]["last_down_alert_time"] = None clients[client_id]["active"] = True clients[client_id]["last_seen"] = datetime.now().isoformat() @@ -142,15 +138,7 @@ def load_clients() -> Dict[str, Any]: clients_collection.delete_many({"_id": {"$in": ["", None]}}) clients_collection.delete_many({"_id": {"$regex": "^\\s*$"}}) - # If collection is empty, seed it with default registry - if clients_collection.count_documents({}) == 0: - default_data = { - "client_1": {"pending_command": "none", "last_seen": None, "active": False}, - "client_2": {"pending_command": "none", "last_seen": None, "active": False}, - "client_3": {"pending_command": "none", "last_seen": None, "active": False} - } - save_clients(default_data) - return default_data + # Load all documents cursor = clients_collection.find() @@ -181,16 +169,16 @@ def load_clients() -> Dict[str, Any]: # Active status transitions alerts if is_active: # Transitions from offline (DOWN) to online (UP) - if cid in last_down_notification_times: + if info.get("last_down_alert_time"): send_rocketchat_notification( text=f"✅ **System UP:** Client `{cid}` has recovered and is back online.", color="#2ecc71" ) - last_down_notification_times.pop(cid, None) + info["last_down_alert_time"] = None else: # Transitions from online (UP) to offline (DOWN) for the first time now = datetime.now() - last_down_notification_times[cid] = now + info["last_down_alert_time"] = now.isoformat() send_rocketchat_notification( text=f"🚨 **System DOWN:** Client `{cid}` has missed heartbeats for over 30 seconds.", color="#e74c3c" @@ -199,17 +187,23 @@ def load_clients() -> Dict[str, Any]: # No transition (state remains the same) if not is_active: # Client remains offline. Check if we should send a repeating reminder. - # We ONLY send a reminder if it is already tracked in last_down_notification_times. + # We ONLY send a reminder if it is already tracked in last_down_alert_time. # This prevents sending alerts for historically offline systems at server boot! - if cid in last_down_notification_times: + last_down_str = info.get("last_down_alert_time") + if last_down_str: now = datetime.now() - elapsed_seconds = (now - last_down_notification_times[cid]).total_seconds() - if elapsed_seconds >= 3600: # 1 hour (3600 seconds) - last_down_notification_times[cid] = now - send_rocketchat_notification( - text=f"🚨 **System STILL DOWN:** Client `{cid}` remains offline (reminder sent every hour).", - color="#e74c3c" - ) + try: + last_down_dt = datetime.fromisoformat(last_down_str) + elapsed_seconds = (now - last_down_dt).total_seconds() + if elapsed_seconds >= 3600: # 1 hour (3600 seconds) + info["last_down_alert_time"] = now.isoformat() + send_rocketchat_notification( + text=f"🚨 **System STILL DOWN:** Client `{cid}` remains offline (reminder sent every hour).", + color="#e74c3c" + ) + modified = True + except Exception: + pass if modified: save_clients(data) @@ -329,32 +323,40 @@ async def receive_telemetry(client_id: str, payload: TelemetryPayload): # Update last_seen timestamp, active status, and store the current telemetry details mark_client_active(client_id, clients) clients[client_id]["telemetry"] = payload.dict() - save_clients(clients) # Log telemetry history event append_to_logs("telemetry", client_id, payload.dict()) # Storage limits warning checker + storage_alerts = clients[client_id].get("active_storage_alerts", []) + storage_modified = False + for disk in payload.disks: mount = disk.get("mount", "/") percent = disk.get("percent", 0.0) - alert_key = f"{client_id}:{mount}" if percent >= 80.0: - if alert_key not in active_storage_alerts: - active_storage_alerts.add(alert_key) + if mount not in storage_alerts: + storage_alerts.append(mount) + storage_modified = True send_rocketchat_notification( text=f"⚠️ **Storage Warning:** Client `{client_id}` disk `{mount}` is at **{percent}%** capacity.", color="#f39c12" ) else: - if alert_key in active_storage_alerts: - active_storage_alerts.remove(alert_key) + if mount in storage_alerts: + storage_alerts.remove(mount) + storage_modified = True send_rocketchat_notification( text=f"✅ **Storage Recovered:** Client `{client_id}` disk `{mount}` has cleared warning state and is at **{percent}%**.", color="#2ecc71" ) + if storage_modified: + clients[client_id]["active_storage_alerts"] = storage_alerts + + save_clients(clients) + data = payload.dict() client_telemetry[client_id] = data @@ -378,7 +380,8 @@ async def get_all_telemetry(): """ Endpoint to view the live dashboard data of all clients. """ - return client_telemetry + clients = load_clients() + return {cid: info.get("telemetry", {}) for cid, info in clients.items() if info.get("telemetry")} @app.get("/api/clients") async def get_clients_api(): diff --git a/deploy_agent.sh b/deploy_agent.sh index 8731baf..2ad0042 100644 --- a/deploy_agent.sh +++ b/deploy_agent.sh @@ -16,13 +16,13 @@ CLIENT_ID="" # For Production deployment, use: "http://rmm-backend.seekright.com" # For Local WSL testing on your laptop, use: "http://172.18.16.1:8000" -CENTRAL_URL="http://rmm-backend.seekright.com" - # Check if Client ID is set; if not, check command line arguments, otherwise default to hostname if [ -z "$CLIENT_ID" ]; then CLIENT_ID="${1:-$(hostname)}" fi +CENTRAL_URL="${2:-http://rmm-backend.seekright.com}" + echo "🚀 Preparing SeekRight RMM Agent installation..." echo "📍 Target Client ID: $CLIENT_ID" echo "🌐 Central Server URL: $CENTRAL_URL"