Compare commits

...

5 Commits
v2.5.1 ... main

19 changed files with 1180 additions and 180 deletions

View File

@ -34,15 +34,14 @@
### Phase 4: 自動化防護、進階管理與指令精準度 (Automation & Advanced Management)
- [x] **智能視覺診斷 (Visual Diagnostics)**
- CM 一鍵診斷中心整合基礎狀態、PHY 射頻指標與 OFDM MER 頻譜。引入 Chart.js 將 ASCII 報表轉化為紅黃綠狀態圖與長條圖。
- [ ] **Diff 引擎指令精準度強化 (Diff Logic Hardening)**
- 重新 Review `generate_diff_commands` 演算法。
- 實作「指令截斷機制」,確保生成 `no` 移除指令時,能精準剝離多餘的 Value 或參數,避免 CMTS 拒絕執行或引發非預期刪除。
- [ ] **自動備份攔截防護 (Auto-Backup Interceptor)**
- 在執行任何 `generate_cli` (寫入設備變更) 的 API 之前,實作強制背景備份 `running-config` 的防呆機制。
- 將此類備份標記為 `is_auto = true`,確保每次人為變更前都有絕對的還原點。
- [ ] **雙軌制備份保留策略 (Dual-Track Retention Policy)**
- **自動快照滾動 (Rolling Window)**:限制每台設備最多保留 30 份自動快照,採用 FIFO (先進先出) 機制自動清理舊資料。
- **手動快照配額 (Manual Quota)**:限制每台設備最多保留 20 份手動快照,達上限時於前端 UI 提示使用者進行清理,防止資料庫無限膨脹。
- [x] **深度代碼審查與並發加固 (Deep Code Review & Concurrency Hardening)**
- [x] **備份保留策略 (Retention Policy)**
- **手動快照配額 (Manual Quota)**:限制每台設備最多保留 20 份手動快照,達上限時,採用 FIFO (先進先出) 機制自動清理舊資料,防止資料庫無限膨脹。
- [x] **系統日誌動態儀表板 (Log Viewer UI)** :
- 基於 FastAPI 實作一個 WebSocket Log Streamer。
1. **Custom Log Handler**: 在現有的 Python `logging` 模組中,撰寫一個自訂的 Handler能夠攔截系統的 Log 訊息(包含 ANSI 色碼)。
2. **WebSocket Endpoint**: 建立一個 FastAPI WebSocket 路由 `/ws/logs`
3. **Broadcaster**: 實作一個簡單的機制,當 Custom Log Handler 收到新日誌時,能非同步地將訊息推播給所有連線中的 WebSocket 客戶端。
### Phase 5: RPD 快速擴容與部署精靈 (Rapid RPD Provisioning Wizard)
- [ ] **RPD 樣板克隆引擎 (Template Cloning)**
@ -51,6 +50,9 @@
- 透過底層 Tree 架構,完整複製複雜的 RF 與通道設定,並提供 UI 介面供使用者修改唯一識別碼 (如 MAC Address、RPD Name)。
- [ ] **安全寫入與校驗 (Safe Provisioning)**
- 結合 Phase 4 的防護機制,在將全新 RPD 配置寫入 CMTS 前,進行參數衝突檢查(避免 MAC 或 IP 重複),實現零錯誤的設備擴容。
- [ ] **Diff 引擎指令精準度強化 (Diff Logic Hardening)**
- 重新 Review `generate_diff_commands` 演算法。
- 實作「指令截斷機制」,確保生成 `no` 移除指令時,能精準剝離多餘的 Value 或參數,避免 CMTS 拒絕執行或引發非預期刪除。
---

File diff suppressed because it is too large Load Diff

View File

@ -154,7 +154,13 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list):
logger.info(f"🔄 正在處理第 {batch_idx + 1}/{len(batches)} 批次 (共 {len(batch)} 個路徑)...")
try:
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=1.0, prompt_pattern: str = None):
@ -284,7 +290,13 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list):
async def fetch_raw_config(host: str, username: str, password: str, config_type: str = "running") -> str:
try:
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=2.0, prompt_pattern: str = None):

View File

@ -251,7 +251,7 @@ async def insert_config_backup(
return None
async def get_config_backup_list(host: str, config_type: str) -> Optional[List[Dict[str, Any]]]:
"""取得歷史快照列表 (支援 config_type='all' 撈取全部)"""
"""取得歷史快照列表 (支援 config_type='all' 撈取全部,包含 is_pinned 狀態)"""
pool = await get_pool()
if not pool:
return None
@ -259,18 +259,18 @@ async def get_config_backup_list(host: str, config_type: str) -> Optional[List[D
try:
async with pool.acquire() as conn:
if config_type == "all":
# 🟢 SELECT 加入 description
# 🌟 SQL 查詢補上 is_pinned
query = """
SELECT id, host, config_type, timestamp, snapshot_name, description, is_auto
SELECT id, host, config_type, timestamp, snapshot_name, description, is_auto, is_pinned
FROM config_backups
WHERE host = $1
ORDER BY timestamp DESC;
"""
records = await conn.fetch(query, host)
else:
# 🟢 SELECT 加入 description
# 🌟 SQL 查詢補上 is_pinned
query = """
SELECT id, host, config_type, timestamp, snapshot_name, description, is_auto
SELECT id, host, config_type, timestamp, snapshot_name, description, is_auto, is_pinned
FROM config_backups
WHERE host = $1 AND config_type = $2
ORDER BY timestamp DESC;
@ -284,8 +284,9 @@ async def get_config_backup_list(host: str, config_type: str) -> Optional[List[D
"config_type": r["config_type"],
"timestamp": r["timestamp"].isoformat(),
"snapshot_name": r["snapshot_name"],
"description": r["description"], # 🟢 將資料庫的描述放入回傳字典
"is_auto": r["is_auto"]
"description": r["description"],
"is_auto": r["is_auto"],
"is_pinned": r["is_pinned"] # 🌟 放入回傳字典
}
for r in records
]
@ -350,3 +351,111 @@ async def delete_config_backup(backup_id: str) -> bool:
except Exception as e:
logger.error(f"❌ Unknown Error (delete_config_backup): {e}")
return False
# ============================================================================
# 🧹 備份滾動淘汰機制 (Retention Policy)
# ============================================================================
async def cleanup_old_backups(pool, host: str) -> None:
"""
執行手動備份的滾動淘汰機制 (Retention Policy)
- 每個設備 (host) 保留最新 20 筆未釘選 (is_pinned=FALSE) 的手動備份
- 釘選的備份 (is_pinned=True) 永遠不刪除
"""
if not pool:
logger.warning("⚠️ [Retention] 無法執行備份清理Database Pool 未初始化。")
return
# 使用 PostgreSQL CTE (Common Table Expression) 語法
# 先選出該設備最新 20 筆需要保留的 ID再將其餘未釘選的舊備份一次性刪除
query = """
WITH kept_backups AS (
SELECT id FROM config_backups
WHERE host = $1 AND is_pinned = FALSE
ORDER BY timestamp DESC
LIMIT 20
)
DELETE FROM config_backups
WHERE host = $1 AND is_pinned = FALSE
AND id NOT IN (SELECT id FROM kept_backups)
RETURNING id;
"""
try:
async with pool.acquire() as conn:
deleted_records = await conn.fetch(query, host)
total_deleted = len(deleted_records)
if total_deleted > 0:
logger.info(f"🧹 [Retention Policy] 設備 {host} 清理完畢:已自動淘汰 {total_deleted} 筆過期手動備份。")
else:
logger.debug(f" [Retention Policy] 設備 {host} 備份數量未達 20 筆上限,無需清理。")
except Exception as e:
logger.error(f"❌ [Retention Policy] 清理設備 {host} 過期備份時發生錯誤: {e}", exc_info=True)
# ============================================================================
# 📌 釘選防護與容量指標計算 (Pinning & Metrics)
# ============================================================================
async def toggle_config_backup_pin(backup_id: str) -> Optional[bool]:
"""
切換特定備份的釘選狀態 (True -> False, False -> True)
回傳更新後的 is_pinned 狀態若失敗則回傳 None
"""
pool = await get_pool()
if not pool:
return None
# 使用 PostgreSQL 的 UPDATE ... RETURNING 語法,一步完成切換與讀取,保證原子性
query = """
UPDATE config_backups
SET is_pinned = NOT is_pinned
WHERE id = $1
RETURNING is_pinned;
"""
try:
async with pool.acquire() as conn:
new_state = await conn.fetchval(query, backup_id)
return new_state
except Exception as e:
logger.error(f"❌ DB Error (toggle_config_backup_pin): {e}")
return None
async def get_backup_metrics(pool, host: str) -> dict:
"""
計算特定設備的備份容量指標
- total: 總備份數 (含釘選與未釘選)
- pinned: 已釘選保護的數量
- unpinned: 未釘選的數量 (上限為 20 )
- remaining: 剩餘可用手動備份額度 (20 - unpinned)
"""
if not pool:
return {"total": 0, "pinned": 0, "unpinned": 0, "remaining": 20}
query = """
SELECT
COUNT(*) as total,
COUNT(*) FILTER (WHERE is_pinned = TRUE) as pinned,
COUNT(*) FILTER (WHERE is_pinned = FALSE) as unpinned
FROM config_backups
WHERE host = $1;
"""
try:
async with pool.acquire() as conn:
r = await conn.fetchrow(query, host)
total = r["total"] or 0
pinned = r["pinned"] or 0
unpinned = r["unpinned"] or 0
remaining = max(0, 20 - unpinned)
return {
"total": total,
"pinned": pinned,
"unpinned": unpinned,
"remaining": remaining
}
except Exception as e:
logger.error(f"❌ DB Error (get_backup_metrics): {e}")
return {"total": 0, "pinned": 0, "unpinned": 0, "remaining": 20}

View File

@ -449,6 +449,21 @@
<span style="color: #7f8c8d;">⏳ 正在載入日誌設定...</span>
</div>
</div>
<!-- 🌟 新增:🎙️ 系統實時日誌面板 (System Live Logs) -->
<div class="control-group" style="background: #ffffff; padding: 25px 30px; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.05); border: 1px solid #e2e8f0; text-align: left; display: block; margin-top: 20px;">
<h3 style="margin-top: 0; font-size: 18px; color: #2c3e50; border-bottom: 2px solid #ecf0f1; padding-bottom: 10px; margin-bottom: 20px; display: flex; justify-content: space-between; align-items: center;">
<span>🎙️ 系統實時日誌 (System Live Logs)</span>
<div style="display: flex; align-items: center; gap: 15px;">
<span id="log-ws-status" style="font-size: 13px; color: #7f8c8d; font-weight: normal;">狀態:已暫停 ⚪</span>
<button id="btnToggleLog" onclick="toggleLogStream()" class="btn-modern btn-disconnect" style="padding: 4px 12px; font-size: 12px;">🔌 啟動監聽</button>
</div>
</h3>
<!-- 日誌終端機容器 -->
<div id="log-terminal-container" style="width: 100%; height: 300px; border-radius: 6px; background-color: #1e1e1e; box-shadow: inset 0 0 10px rgba(0,0,0,0.8); overflow: hidden; position: relative; padding: 10px; box-sizing: border-box;"></div>
</div>
</div>
</div>
@ -539,10 +554,11 @@
<thead>
<tr style="background-color: #f8f9fa; border-bottom: 2px solid #bdc3c7;">
<th style="padding: 10px; color: #34495e; width: 20%;">時間</th>
<th style="padding: 10px; color: #34495e; width: 25%;">快照名稱</th>
<th style="padding: 10px; color: #34495e; width: 25%;">描述</th>
<th style="padding: 10px; color: #34495e; width: 8%; text-align: center;">保護</th> <!-- 🌟 新增:獨立的保護欄位 -->
<th style="padding: 10px; color: #34495e; width: 22%;">快照名稱</th>
<th style="padding: 10px; color: #34495e; width: 22%;">描述</th>
<th style="padding: 10px; color: #34495e; width: 10%;">類型</th>
<th style="padding: 10px; color: #34495e; text-align: right; width: 20%;">操作</th>
<th style="padding: 10px; color: #34495e; text-align: right; width: 18%;">操作</th>
</tr>
</thead>
<tbody id="backup-history-tbody">

View File

@ -79,7 +79,7 @@ async def init_database(force_reset: bool = False):
);
""")
# 4. 建立設備配置備份表 config_backups
# 4. 建立設備配置備份表 config_backups (手動備份 + 釘選防護版)
logger.info("🛠️ 正在檢查/建立 config_backups 資料表與索引...")
await conn.execute("""
CREATE TABLE IF NOT EXISTS config_backups (
@ -90,17 +90,25 @@ async def init_database(force_reset: bool = False):
snapshot_name VARCHAR(255),
description TEXT DEFAULT '',
is_auto BOOLEAN NOT NULL DEFAULT FALSE,
is_pinned BOOLEAN NOT NULL DEFAULT FALSE, -- 🌟 新增釘選防護欄位
raw_cli TEXT,
parsed_tree JSONB,
CONSTRAINT uq_snapshot_name UNIQUE (host, config_type, snapshot_name)
);
""")
# 建立基礎查詢索引
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_config_backups_host_type_ts
ON config_backups (host, config_type, timestamp DESC);
""")
# 🌟 新增:建立滾動淘汰專用索引
await conn.execute("""
CREATE INDEX IF NOT EXISTS idx_config_backups_retention
ON config_backups (host, is_pinned, timestamp DESC);
""")
await conn.close()
logger.info("✅ 資料庫初始化完成!所有資料表已準備就緒。")

View File

@ -26,7 +26,8 @@ MANAGED_LOGGERS = [
"app.scraper", # 爬蟲與快取
"app.diagnostics", # CM 診斷
"app.backup", # 備份與還原
"app.ssh" # SSH 連線引擎
"app.ssh", # 🔌 SSH 連線引擎 (注意這裡要有逗號!)
"app.auth" # 🔐 上帝模式與安全授權
]
def setup_logger(name: str, level=logging.INFO):
@ -61,3 +62,10 @@ def set_log_level(name: str, level_str: str):
logging.getLogger(name).setLevel(level)
return True
return False
def register_websocket_handler(handler: logging.Handler):
"""🌟 新增:註冊自訂 Handler 到所有受管控的 Logger並防止重複添加"""
for name in MANAGED_LOGGERS:
logger = logging.getLogger(name)
if handler not in logger.handlers:
logger.addHandler(handler)

View File

@ -6,14 +6,17 @@ from fastapi.responses import HTMLResponse
from contextlib import asynccontextmanager
from starlette.middleware.base import BaseHTTPMiddleware # 🌟 新增引入
import database
import asyncio
# 引入我們剛剛拆分出來的路由模組
from routers import query, config, terminal, lock, leaf_options, backup, diagnostics, auth
from routers import query, config, terminal, lock, leaf_options, backup, diagnostics, auth, logs
@asynccontextmanager
async def lifespan(app: FastAPI):
# Startup
await database.init_db_pool()
# 🌟 新增:初始化 Log Broadcaster 的 Event Loop 參考,確保跨執行緒排程安全
logs.log_broadcaster.loop = asyncio.get_running_loop()
yield
# Shutdown
await database.close_db_pool()
@ -54,6 +57,7 @@ app.include_router(auth.router, prefix="/api/v1")
# WebSocket 通常獨立於 API 版本之外,所以不加前綴
app.include_router(terminal.router)
app.include_router(logs.router) # 🌟 新增:掛載日誌 WebSocket 路由
# 根目錄路由:回傳前端 UI
@app.get("/", response_class=HTMLResponse, tags=["UI"])

View File

@ -3,7 +3,7 @@ import asyncssh
import asyncio
import re
import json
from fastapi import APIRouter, HTTPException
from fastapi import APIRouter, HTTPException, BackgroundTasks
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from typing import Optional, List
@ -11,15 +11,19 @@ from logger import get_logger
# 引入 DB 函數
from database import (
get_pool,
insert_config_backup,
get_config_backup_list,
get_config_backup_detail,
delete_config_backup
delete_config_backup,
cleanup_old_backups,
toggle_config_backup_pin, # 🌟 新增
get_backup_metrics # 🌟 新增
)
from cmts_scraper import fetch_raw_config
# 🌟 [Priority 1 修復] 引入 cmts_config_locks
from shared import parse_cli_to_tree, cmts_config_locks
from shared import parse_cli_to_tree, get_cmts_lock
logger = get_logger("app.backup")
@ -149,23 +153,35 @@ async def delete_backup(backup_id: str):
return {"status": "error", "message": "刪除失敗或找不到該筆資料"}
return {"status": "success", "message": "快照已成功刪除"}
@router.post("/{backup_id}/toggle-pin", summary="切換快照的釘選防護狀態")
async def toggle_backup_pin(backup_id: str):
new_state = await toggle_config_backup_pin(backup_id)
if new_state is None:
raise HTTPException(status_code=500, detail="資料庫更新失敗")
status_str = "已啟用釘選保護,系統絕不自動淘汰。" if new_state else "已取消釘選,該備份將納入滾動淘汰範圍。"
return {
"status": "success",
"is_pinned": new_state,
"message": f"快照狀態更新成功!{status_str}"
}
@router.post("/snapshot", summary="手動建立設備快照 (串流版)")
async def create_snapshot(req: SnapshotRequest):
async def create_snapshot(req: SnapshotRequest, background_tasks: BackgroundTasks):
async def backup_streamer():
try:
# 1. 廣播:正在連線
pool = await get_pool()
yield json.dumps({"status": "progress", "message": f"🔌 正在建立 SSH 連線至 {req.host}..."}) + "\n"
await asyncio.sleep(0.1)
# 2. 廣播:正在下載
yield json.dumps({"status": "progress", "message": f"📥 正在下載 {req.config_type} 配置檔 (可能需要 30~60 秒)..."}) + "\n"
raw_cli = await fetch_raw_config(req.host, req.username, req.password, req.config_type)
# 3. 廣播:正在解析
yield json.dumps({"status": "progress", "message": "🧩 正在解析配置並建構樹狀圖結構..."}) + "\n"
await asyncio.sleep(0.1)
parsed_tree = await asyncio.to_thread(parse_cli_to_tree, raw_cli)
# 4. 廣播:寫入資料庫
yield json.dumps({"status": "progress", "message": "💾 正在將快照寫入資料庫..."}) + "\n"
backup_id = await insert_config_backup(
host=req.host,
@ -181,12 +197,23 @@ async def create_snapshot(req: SnapshotRequest):
yield json.dumps({"status": "error", "message": "資料庫寫入失敗,請檢查系統日誌"}) + "\n"
return
# 5. 廣播:成功
# 🌟 關鍵變更:在同一個串流中「立即執行」清理與指標計算,確保回傳最新數據
yield json.dumps({"status": "progress", "message": "🧹 正在執行備份滾動淘汰與容量計算..."}) + "\n"
if pool:
# 1. 立即執行清理
await cleanup_old_backups(pool, req.host)
# 2. 立即計算最新指標
metrics = await get_backup_metrics(pool, req.host)
else:
metrics = {"total": 1, "pinned": 0, "unpinned": 1, "remaining": 19}
# 🌟 3. 在成功訊息中,將 metrics 字典一併回傳給前端
yield json.dumps({
"status": "success",
"message": f"快照 '{req.snapshot_name}' 建立成功!",
"backup_id": backup_id,
"host": req.host
"host": req.host,
"metrics": metrics # 🌟 包含容量指標
}) + "\n"
except Exception as e:
@ -215,8 +242,9 @@ async def analyze_backup_diff(backup_id: str, req: RestoreRequest):
if not raw_current:
return {"status": "error", "message": "無法從設備取得當前配置,請檢查連線狀態"}
current_tree = parse_cli_to_tree(raw_current)
diff_commands = generate_diff_commands(current_tree, snapshot_tree)
# ✅ 安全升級:將 CPU 密集運算移交給 ThreadPool保護 Event Loop 不卡死
current_tree = await asyncio.to_thread(parse_cli_to_tree, raw_current)
diff_commands = await asyncio.to_thread(generate_diff_commands, current_tree, snapshot_tree)
return {
"status": "success",
@ -242,7 +270,7 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
async def restore_generator():
# 🌟 [Priority 1 修復] 使用依 IP 隔離的鎖
host_lock = cmts_config_locks[req.host]
host_lock = get_cmts_lock[req.host]
# 1. 嘗試取得全域寫入鎖 (避免多人同時打指令)
if host_lock.locked():
@ -254,7 +282,13 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
yield json.dumps({"status": "progress", "message": "🔄 正在建立 SSH 安全連線..."}) + "\n"
# 🌟 [Priority 2 提前修復] 使用 async with 確保連線與 Process 絕對會被釋放
async with asyncssh.connect(req.host, username=req.username, password=req.password, known_hosts=None) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=1.0, prompt_pattern: str = None):

View File

@ -10,7 +10,7 @@ from pydantic import BaseModel
from typing import List
from netmiko import ConnectHandler
# 🌟 [Priority 1 修復] 引入 cmts_config_locks
from shared import CMTS_DEVICE, cmts_config_locks, parse_cli_to_tree, deep_split_tree
from shared import CMTS_DEVICE, get_cmts_lock, parse_cli_to_tree, deep_split_tree
from collections import defaultdict
from fastapi.responses import StreamingResponse
from logger import get_all_log_levels, set_log_level
@ -64,18 +64,18 @@ async def execute_cmts_config(req: ConfigRequest):
async def config_streamer():
# 🌟 [Priority 1 修復] 使用依 IP 隔離的鎖
host_lock = cmts_config_locks[req.host]
host_lock = get_cmts_lock[req.host]
async with host_lock:
try:
yield json.dumps({"status": "progress", "message": f"🔌 準備連線至設備 {req.host}..."}) + "\n"
# 使用 asyncssh 建立非同步連線
async with asyncssh.connect(
req.host,
username=req.username,
password=req.password,
known_hosts=None
) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
async with conn.create_process() as process:
# 🌟 建立安全的非同步讀取函數 (讀到安靜為止,避免卡死)

View File

@ -1,4 +1,5 @@
# --- routers/diagnostics.py ---
import asyncio
import re
import asyncssh
from logger import get_logger
@ -34,7 +35,14 @@ async def get_cm_diagnostics(
}
try:
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
logger.debug(f"🚀 [Diagnostics] 開始診斷 MAC: {mac}")
# ==========================================

View File

@ -50,6 +50,9 @@ async def sse_stream(request: Request, host: str):
finally:
if channel_key in active_clients:
active_clients[channel_key].discard(client_queue)
# ✅ 安全升級:如果該設備已經沒有任何客戶端監聽,徹底刪除 Key 釋放記憶體
if not active_clients[channel_key]:
del active_clients[channel_key]
return StreamingResponse(event_generator(), media_type="text/event-stream")

79
routers/logs.py Normal file
View File

@ -0,0 +1,79 @@
# --- routers/logs.py ---
import asyncio
import logging
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
from logger import ColoredFormatter, register_websocket_handler
router = APIRouter()
class LogBroadcaster:
"""管理所有活躍 WebSocket 連線的廣播器 (Connection Manager)"""
def __init__(self):
self.active_connections: set[WebSocket] = set()
self.loop: asyncio.AbstractEventLoop = None
async def connect(self, websocket: WebSocket):
await websocket.accept()
self.active_connections.add(websocket)
def disconnect(self, websocket: WebSocket):
self.active_connections.discard(websocket)
async def broadcast(self, message: str):
"""非同步推播日誌給所有連線中的客戶端"""
if not self.active_connections:
return
# 併發發送,並透過 return_exceptions=True 確保單一連線異常不影響其他客戶端
tasks = [self._safe_send(conn, message) for conn in self.active_connections]
await asyncio.gather(*tasks, return_exceptions=True)
async def _safe_send(self, websocket: WebSocket, message: str):
try:
await websocket.send_text(message)
except Exception:
self.disconnect(websocket)
# 建立全域廣播器實例
log_broadcaster = LogBroadcaster()
class WebSocketLogHandler(logging.Handler):
"""自訂 Log Handler攔截系統日誌並安全地派發至非同步廣播器"""
def __init__(self, broadcaster: LogBroadcaster):
super().__init__()
self.broadcaster = broadcaster
# 沿用系統 ColoredFormatter完美保留 ANSI 色碼
self.setFormatter(ColoredFormatter())
def emit(self, record):
try:
msg = self.format(record)
loop = self.broadcaster.loop
# 確保在 Event Loop 處於執行狀態時,安全地跨執行緒派發任務
if loop and loop.is_running():
loop.call_soon_threadsafe(
lambda: asyncio.create_task(self.broadcaster.broadcast(msg))
)
except Exception:
self.handleError(record)
# 建立 Handler 實例並註冊至所有受管控的 Logger
websocket_log_handler = WebSocketLogHandler(log_broadcaster)
register_websocket_handler(websocket_log_handler)
@router.websocket("/ws/logs")
async def websocket_logs(websocket: WebSocket):
"""WebSocket 實時日誌串流端點"""
# 若尚未綁定 Event Loop則於首次連線時動態綁定
if not log_broadcaster.loop:
log_broadcaster.loop = asyncio.get_running_loop()
await log_broadcaster.connect(websocket)
try:
# 保持連線,監聽客戶端斷線狀態
while True:
await websocket.receive_text()
except WebSocketDisconnect:
log_broadcaster.disconnect(websocket)
except Exception:
log_broadcaster.disconnect(websocket)

View File

@ -17,7 +17,13 @@ async def execute_single_command(host, username, password, command, timeout=15.0
自動防呆確保指令帶有取消分頁的後綴防止 Event Loop 卡死
"""
try:
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
# 確保指令包含取消分頁的參數
if "| nomore" not in command.lower():
command += " | nomore"
@ -38,7 +44,13 @@ async def execute_interactive_command(host, username, password, commands: list,
動態處理終端機的 --More-- 提示
"""
try:
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
# 先用 wait_for 取得連線 (超過 10 秒會拋出 asyncio.TimeoutError)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 再進入 async with 確保資源會被自動關閉
async with conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24)) as process:
async def read_until_quiet(wait_time):

View File

@ -15,7 +15,10 @@ async def websocket_terminal(websocket: WebSocket, host: str, username: str, pas
ssh_task = None
try:
# 建立連線
conn = await asyncssh.connect(host, username=username, password=password, known_hosts=None)
conn = await asyncio.wait_for(
asyncssh.connect(host, username=username, password=password, known_hosts=None),
timeout=10.0
)
# 💡 關鍵修復:使用 term_size 參數來指定寬高 (width, height)
process = await conn.create_process(
@ -37,7 +40,7 @@ async def websocket_terminal(websocket: WebSocket, host: str, username: str, pas
except asyncio.CancelledError:
pass
except Exception as e:
logger.error("❌ [WS Forward Error] 讀取設備畫面時發生錯誤", exc_info=True)
logger.error(f"❌ [WS Forward Error] 讀取設備畫面時發生錯誤: {str(e)}")
async def forward_to_ssh():
try:
@ -60,7 +63,7 @@ async def websocket_terminal(websocket: WebSocket, host: str, username: str, pas
except asyncio.CancelledError:
pass
except Exception as e:
logger.error("❌ [SSH Forward Error] 寫入指令到設備時發生錯誤", exc_info=True)
logger.error(f"❌ [SSH Forward Error] 寫入指令到設備時發生錯誤: {str(e)}")
ws_task = asyncio.create_task(forward_to_ws())
ssh_task = asyncio.create_task(forward_to_ssh())
@ -91,7 +94,7 @@ async def websocket_terminal(websocket: WebSocket, host: str, username: str, pas
await websocket.close(code=4001, reason=safe_reason)
except:
pass
logger.error("❌ [Connection Error] 建立連線或執行過程中發生錯誤", exc_info=True)
logger.error(f"❌ [Connection Error] 建立連線或執行過程中發生錯誤: {str(e)}")
return # 🌟 提早結束,避免下方的 finally 再次執行 close 導致報錯
finally:
# 🌟 確保 process 與 conn 絕對被關閉,防止 Memory Leak

View File

@ -257,5 +257,10 @@ def save_settings(settings_data):
with open(SETTINGS_FILE, "w", encoding="utf-8") as f:
json.dump(settings_data, f, indent=4, ensure_ascii=False)
# 🌟 [Priority 1 修復] 將全域非同步鎖改為依設備 IP (Host) 隔離的鎖
cmts_config_locks = defaultdict(asyncio.Lock)
# 🌟 [Priority 1 修復] 將全域非同步鎖改為依設備 IP (Host) 隔離的鎖 (安全動態獲取版)
_cmts_config_locks = {}
def get_cmts_lock(host: str) -> asyncio.Lock:
if host not in _cmts_config_locks:
_cmts_config_locks[host] = asyncio.Lock()
return _cmts_config_locks[host]

View File

@ -227,3 +227,14 @@ export async function apiVerifyGodMode(password) {
}
return response.json();
}
// ==========================================
// 📌 釘選防護 API (Pinning)
// ==========================================
export async function apiToggleBackupPin(backupId) {
const response = await fetch(`/api/v1/backups/${backupId}/toggle-pin`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' }
});
return response.json();
}

View File

@ -18,7 +18,8 @@ import {
apiGetLogLevels, apiSetLogLevel,
apiVerifyGodMode,
apiGetLeafOptions, // 🌟 補上這個
apiSyncLeafOptions // 🌟 補上這個
apiSyncLeafOptions, // 🌟 補上這個
apiToggleBackupPin // 🌟 新增這行
} from './api.js';
// 3. 共用工具模組 (Utils)
@ -55,6 +56,10 @@ let idleTimer;
const IDLE_TIMEOUT = 300000; // 5 分鐘 (毫秒)
let globalSSE = null;
let logTerm = null;
let logWs = null;
let isLogEnabled = false; // 預設關閉,不主動消耗資源
// 初始化全域 SSE 監聽器 (設備級別)
function initGlobalSSE() {
// 🌟 新增防線:如果尚未連線,強制關閉舊的 SSE 並退出
@ -1238,9 +1243,102 @@ async function switchFilterMode() {
await openSystemSettings();
}
// 開啟系統設定並讀取目前的過濾器清單
function initLogTerminal() {
const container = document.getElementById('log-terminal-container');
if (!container || logTerm) return;
logTerm = new Terminal({
cursorBlink: false,
disableStdin: true,
theme: { background: '#1e1e1e', foreground: '#e0e0e0' },
fontFamily: 'Courier New, monospace',
fontSize: 13,
scrollback: 1000,
cols: 250 // 確保不折行
});
logTerm.open(container);
logTerm.writeln('\x1b[33m[System] Live Log Terminal Ready. Click "啟動監聽" to start streaming.\x1b[0m\r\n');
}
function toggleLogStream() {
isLogEnabled = !isLogEnabled;
const btn = document.getElementById('btnToggleLog');
if (isLogEnabled) {
initLogTerminal();
connectLogWebSocket();
if (btn) {
btn.textContent = '⏸️ 暫停監聽';
btn.className = 'btn-modern btn-clear';
}
} else {
disconnectLogWebSocket();
if (btn) {
btn.textContent = '🔌 啟動監聽';
btn.className = 'btn-modern btn-connect';
}
}
}
function connectLogWebSocket() {
if (!isLogEnabled) return;
if (logWs && logWs.readyState === WebSocket.OPEN) return;
const statusEl = document.getElementById('log-ws-status');
if (statusEl) {
statusEl.textContent = '狀態:正在連線... 🟡';
statusEl.style.color = '#f39c12';
}
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${protocol}//${window.location.host}/ws/logs`;
logWs = new WebSocket(wsUrl);
logWs.onopen = () => {
if (statusEl) {
statusEl.textContent = '狀態:實時監聽中 🟢';
statusEl.style.color = '#27ae60';
}
};
logWs.onmessage = (event) => {
if (logTerm && isLogEnabled) {
const formattedLog = event.data.replace(/\n/g, '\r\n') + '\r\n';
logTerm.write(formattedLog);
}
};
logWs.onclose = () => {
const statusEl = document.getElementById('log-ws-status');
if (isLogEnabled) {
if (statusEl) {
statusEl.textContent = '狀態:連線中斷,嘗試重連... 🔴';
statusEl.style.color = '#c0392b';
}
setTimeout(connectLogWebSocket, 5000);
} else {
if (statusEl) {
statusEl.textContent = '狀態:已暫停 ⚪';
statusEl.style.color = '#7f8c8d';
}
}
};
}
function disconnectLogWebSocket() {
if (logWs) {
logWs.close();
logWs = null;
}
}
// --- 3. 確保 openSystemSettings 正常初始化 ---
async function openSystemSettings() {
// 🌟 動態抓取過濾器模式
setTimeout(() => {
initLogTerminal();
}, 50);
const modeSelect = document.getElementById('filter-mode-select');
const mode = modeSelect ? modeSelect.value : 'running';
@ -1251,7 +1349,6 @@ async function openSystemSettings() {
console.error("讀取設定失敗:", error);
}
// 🌟 新增:同時載入日誌等級
loadLogLevels();
}
@ -1268,7 +1365,8 @@ async function loadLogLevels() {
'app.scraper': '🕷️ 爬蟲與快取模組 (Scraper)',
'app.diagnostics': '🩺 CM 診斷模組 (Diagnostics)',
'app.backup': '💾 備份與還原模組 (Backup)',
'app.ssh': '🔌 SSH 連線引擎 (SSH Engine)'
'app.ssh': '🔌 SSH 連線引擎 (SSH Engine)',
'app.auth': '🔐 安全授權模組 (Auth)' // 🌟 新增這行
};
for (const [module, level] of Object.entries(res.data)) {
@ -1468,7 +1566,7 @@ async function saveSystemSettings() {
// 🌟 新增:全域變數用來暫存所有快照資料,實現前端秒速過濾
let allBackupsData = [];
// 1. 建立新快照 - 🌟 升級為真實動態訊息流 (含百分比動畫)
// 🌟 修正版:手動建立設備快照 (解決容量看板型別解析 Bug)
async function createSnapshot() {
if (!isConnected) {
return alert("⚠️ 請先點擊畫面上方的「連線至 CMTS」成功後再執行備份任務\n(確保您清楚目前正在操作哪一台設備)");
@ -1487,10 +1585,10 @@ async function createSnapshot() {
btn.disabled = true;
msg.style.display = 'inline-block';
msg.style.color = '#f39c12'; // 🌟 確保橘色
msg.style.color = '#f39c12'; // 確保橘色
msg.innerHTML = '⏳ 準備連線至設備...';
let progressInterval = null; // 🌟 新增:進度條計時器
let progressInterval = null;
try {
const response = await fetch('/api/v1/backups/snapshot', {
@ -1512,7 +1610,7 @@ async function createSnapshot() {
const decoder = new TextDecoder("utf-8");
let buffer = "";
let isSuccess = false;
let finalMessage = "";
let finalData = null; // 🌟 修正:改為儲存完整的 JSON 物件,而非只有字串
while (true) {
const { done, value } = await reader.read();
@ -1527,7 +1625,6 @@ async function createSnapshot() {
try {
const data = JSON.parse(line);
if (data.status === 'progress') {
// 🌟 導入百分比動畫邏輯
if (data.message.includes('正在下載')) {
let progress = 0;
if (progressInterval) clearInterval(progressInterval);
@ -1556,7 +1653,7 @@ async function createSnapshot() {
}
} else if (data.status === 'success') {
isSuccess = true;
finalMessage = data.message;
finalData = data; // 🌟 修正:儲存完整的成功資料物件 (包含 metrics)
} else if (data.status === 'error') {
throw new Error(data.message);
}
@ -1568,7 +1665,23 @@ async function createSnapshot() {
}
if (isSuccess) {
alert(`${finalMessage}`);
// 🌟 修正:從 finalData 中安全提取 metrics並補齊所有預設值
const metrics = finalData?.metrics || { total: 0, pinned: 0, unpinned: 0, remaining: 20 };
const successMsg = finalData?.message || `快照 '${snapshotName}' 建立成功!`;
// 建立精美的容量看板 HTML 內容
const alertMsg =
`🎉 ${successMsg}\n\n` +
`📊 【當前設備備份容量看板】\n` +
`----------------------------------------\n` +
`🗄️ 總備份檔案數:${metrics.total}\n` +
`📌 已釘選保護數:${metrics.pinned}\n` +
`🔓 未釘選備份數:${metrics.unpinned}\n` +
`🔋 剩餘可用空間:${metrics.remaining} 筆 (上限 20 筆)\n\n` +
`💡 提示:當可用空間為 0 時,新備份將會自動淘汰最舊的未釘選備份。您可以點擊 📌 圖示來保護重要備份!`;
alert(alertMsg);
document.getElementById('snapshotName').value = '';
if (document.getElementById('snapshotDescription')) {
document.getElementById('snapshotDescription').value = '';
@ -1578,7 +1691,7 @@ async function createSnapshot() {
} catch (error) {
alert(`❌ 發生錯誤: ${error.message}`);
} finally {
if (progressInterval) clearInterval(progressInterval); // 🌟 確保清除計時器
if (progressInterval) clearInterval(progressInterval);
btn.disabled = false;
msg.style.display = 'none';
}
@ -1594,7 +1707,7 @@ async function loadBackupHistory() {
if (!tbody) return;
// 🌟 修正:將原本的灰色改為橘色並加粗
tbody.innerHTML = '<tr><td colspan="5" style="padding: 20px; text-align: center; color: #f39c12; font-weight: bold;">⏳ 正在載入歷史紀錄...</td></tr>';
tbody.innerHTML = '<tr><td colspan="6" style="padding: 20px; text-align: center; color: #f39c12; font-weight: bold;">⏳ 正在載入歷史紀錄...</td></tr>';
try {
// 🌟 關鍵修改:將 config_type 改為 'all',一次把該設備的所有快照撈回來
@ -1652,29 +1765,48 @@ window.applyBackupFilters = function() {
// 渲染過濾後的結果
if (filteredData.length > 0) {
// 🌟 新增:判斷當前是否已解鎖上帝模式 (God Mode)
const isGodMode = sessionStorage.getItem('godModeUnlocked') === 'true';
const displayAdmin = isGodMode ? 'inline-block' : 'none';
tbody.innerHTML = filteredData.map(item => `
tbody.innerHTML = filteredData.map(item => {
// 🌟 根據 is_pinned 狀態,決定 📌 的外觀與提示文字 (移除 margin-right)
const pinIcon = item.is_pinned
? `<span onclick="togglePin('${item.id}')" style="cursor: pointer; font-size: 16px; display: inline-block; transform: scale(1.2); transition: transform 0.1s;" title="📌 已釘選保護(系統絕不自動淘汰),點擊取消保護">📌</span>`
: `<span onclick="togglePin('${item.id}')" class="unpinned-icon" style="cursor: pointer; font-size: 16px; opacity: 0.2; display: inline-block; transition: all 0.2s;" title="📍 點擊釘選保護,防止被滾動淘汰">📌</span>`;
return `
<tr style="border-bottom: 1px solid #ecf0f1; transition: background-color 0.2s;" onmouseover="this.style.backgroundColor='#fcfcfc'" onmouseout="this.style.backgroundColor='transparent'">
<!-- 1. 時間 -->
<td style="padding: 12px 10px; color: #7f8c8d; font-size: 14px;">${new Date(item.timestamp).toLocaleString()}</td>
<td style="padding: 12px 10px; font-weight: bold; color: #2c3e50;">${item.snapshot_name}</td>
<!-- 🟢 描述欄位 -->
<!-- 2. 🌟 獨立的保護 (釘選) 欄位採水平與垂直置中對齊 -->
<td style="padding: 12px 10px; text-align: center; vertical-align: middle;">
${pinIcon}
</td>
<!-- 3. 快照名稱 (純文字完美對齊) -->
<td style="padding: 12px 10px; font-weight: bold; color: #2c3e50;">
${item.snapshot_name}
</td>
<!-- 4. 描述 -->
<td style="padding: 12px 10px; color: #7f8c8d; max-width: 250px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;" title="${item.description || ''}">
${item.description || '<span style="color: #bdc3c7; font-style: italic;">無</span>'}
</td>
<!-- 5. 類型 -->
<td style="padding: 12px 10px; color: #34495e;"><span style="background: #e8f8f5; color: #27ae60; padding: 3px 8px; border-radius: 12px; font-size: 12px;">${item.config_type}</span></td>
<!-- 6. 操作 -->
<td style="padding: 12px 10px; text-align: right;">
<button onclick="viewBackup('${item.id}')" class="btn-modern btn-scan" style="padding: 4px 8px; font-size: 12px; margin-right: 5px;">👁 檢視</button>
<!-- 🌟 加上 admin-only-action class 與動態 display 樣式只有解鎖後才會出現 -->
<button onclick="restoreBackup('${item.id}', '${item.snapshot_name}', '${item.host}')" class="btn-modern btn-save admin-only-action" style="display: ${displayAdmin}; padding: 4px 8px; font-size: 12px; margin-right: 5px; background-color: #8e44ad;" title="將設備還原至此狀態"> 還原</button>
<button onclick="deleteBackup('${item.id}')" class="btn-modern btn-disconnect admin-only-action" style="display: ${displayAdmin}; padding: 4px 8px; font-size: 12px;">🗑 刪除</button>
</td>
</tr>
`).join('');
`;
}).join('');
} else {
// 🟢 修正沒有資料時colspan 改為 5
tbody.innerHTML = '<tr><td colspan="5" style="padding: 20px; text-align: center; color: #7f8c8d;">找不到符合條件的備份紀錄。</td></tr>';
}
};
@ -1730,6 +1862,21 @@ async function deleteBackup(backupId) {
}
}
// 🌟 新增:切換釘選狀態
window.togglePin = async function(backupId) {
try {
const result = await apiToggleBackupPin(backupId);
if (result.status === 'success') {
// 重新載入歷史紀錄,即時更新 📌 狀態
await loadBackupHistory();
} else {
alert(`❌ 操作失敗: ${result.message}`);
}
} catch (error) {
alert(`❌ 連線失敗: ${error.message}`);
}
};
// 5. 請求設備差異分析 (安全還原第一步)
// 🌟 接收第三個參數 backupHost
async function restoreBackup(snapshotId, snapshotName, backupHost) {
@ -2193,6 +2340,7 @@ window.scanMissingOptions = scanMissingOptions;
window.clearOptionsCache = clearOptionsCache;
window.previewNodeCLI = previewNodeCLI;
window.switchFilterMode = switchFilterMode;
window.toggleLogStream = toggleLogStream;
// --- 樹狀圖展開與編輯操作 ---
window.expandAll = expandAll;
@ -2219,3 +2367,4 @@ window.deleteBackup = deleteBackup;
window.restoreBackup = restoreBackup;
window.applyBackupFilters = applyBackupFilters; // 🌟 確保過濾器綁定到全域
window.changeLogLevel = changeLogLevel; // 🌟 新增這行,統一在這裡綁定!
window.togglePin = togglePin;

View File

@ -303,3 +303,40 @@ button:hover { background-color: #3498db; }
content-visibility: auto;
contain-intrinsic-size: 0 24px;
}
/* ============================================================================
🎙 系統實時日誌終端機 - 滾動條隔離優化 (Scrollbar Isolation - 修正版)
============================================================================ */
#log-terminal-container .xterm {
width: 100% !important;
}
#log-terminal-container .xterm-screen {
/* 🌟 關鍵:限制文字區最大寬度,扣除右側垂直滾動條的寬度 */
max-width: calc(100% - 18px) !important;
/* 🌟 關鍵:只允許水平滾動,強制關閉垂直滾動,消除多餘的藍灰色軌道 */
overflow-x: auto !important;
overflow-y: hidden !important;
}
/* 讓水平滾動條呈現精美的扁平化現代風格 */
#log-terminal-container .xterm-screen::-webkit-scrollbar {
height: 8px;
}
#log-terminal-container .xterm-screen::-webkit-scrollbar-track {
background: #1e1e1e;
border-radius: 4px;
}
#log-terminal-container .xterm-screen::-webkit-scrollbar-thumb {
background: #475569;
border-radius: 4px;
}
#log-terminal-container .xterm-screen::-webkit-scrollbar-thumb:hover {
background: #64748b;
}
.unpinned-icon:hover { opacity: 1 !important; transform: scale(1.2); }