perf(memory): 實施 Priority 2 記憶體洩漏修復與資源回收優化

- 前端 OOM 防護:實作 clearTreeCache 並於切換設備時觸發,釋放巨量樹狀圖快取 (tree-ui.js, app.js)。
- 後端 SSE 資源回收:延長輪詢 timeout 降低 CPU 負載,並透過 finally 確保斷線時 Queue 絕對釋放 (leaf_options.py)。
- SSH 連線生命週期:全面導入 async with Context Manager,確保爬蟲任務中 AsyncSSH 連線與 Process 乾淨關閉 (cmts_scraper.py)。
This commit is contained in:
swpa 2026-05-31 16:15:44 +08:00
parent 4437e4e3fa
commit 60195a735e
5 changed files with 481 additions and 420 deletions

View File

@ -663,8 +663,8 @@ def save_settings(settings_data):
with open(SETTINGS_FILE, "w", encoding="utf-8") as f:
json.dump(settings_data, f, indent=4, ensure_ascii=False)
# 全域非同步鎖:確保同一時間只有一人能執行設定寫入
cmts_config_lock = asyncio.Lock()
# 🌟 [Priority 1 修復] 將全域非同步鎖改為依設備 IP (Host) 隔離的鎖
cmts_config_locks = defaultdict(asyncio.Lock)
================================================================================
@ -1423,11 +1423,11 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list, con
for batch_idx, batch in enumerate(batches):
print(f"🔄 正在處理第 {batch_idx + 1}/{len(batches)} 批次 (共 {len(batch)} 個路徑)...")
conn = None
try:
conn = await asyncssh.connect(host, username=username, password=password, known_hosts=None)
process = await conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8')
# 🧹 [穩定性修復] 全面改用 async with 管理生命週期,確保連線與 process 絕對釋放
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=1.0):
output = ""
@ -1523,8 +1523,6 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list, con
except Exception as e:
yield json.dumps({"event": "error", "message": f"批次錯誤: {str(e)}"}) + "\n"
continue
finally:
if conn: conn.close()
# --- 步驟 3寫入快取 (DB or JSON) ---
try:
@ -1606,10 +1604,10 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list, con
async def fetch_raw_config(host: str, username: str, password: str, config_type: str = "running") -> str:
"""連線至設備並抓取完整的純文字設定檔"""
conn = None
try:
conn = await asyncssh.connect(host, username=username, password=password, known_hosts=None)
process = await conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8')
# 🧹 [穩定性修復] 全面改用 async with 管理生命週期
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=2.0):
output = ""
@ -1683,6 +1681,7 @@ async def fetch_raw_config(host: str, username: str, password: str, config_type:
except Exception as e:
raise Exception(f"SSH 連線或抓取設定失敗: {str(e)}")
finally:
if conn:
conn.close()
@ -2038,7 +2037,7 @@ import {
import { getGlobalConnectionInfo } from './utils.js';
// 4. 樹狀圖 UI 模組 (Tree UI)
import { buildTree, buildRealFilterTree, expandAll, collapseAll } from './tree-ui.js';
import { buildTree, buildRealFilterTree, expandAll, collapseAll, clearTreeCache } from './tree-ui.js';
// 5. 編輯模式與鎖定模組 (Edit Mode)
import {
@ -2339,6 +2338,9 @@ function toggleQueryInputs(mode) {
function resetAllManagerData() {
resetMacDomainData();
// 🧹 [修復 Leak] 清空前端樹狀圖快取,避免切換設備時發生 OOM
clearTreeCache();
const configArea = document.getElementById('macDomainConfigArea');
if (configArea) configArea.style.display = 'none';
@ -5057,6 +5059,12 @@ FILE: static/tree-ui.js
export const folderDataCache = {};
export const filterFolderCache = {};
// 🧹 [修復 Leak] 徹底清空樹狀圖快取,釋放記憶體
export function clearTreeCache() {
for (let key in folderDataCache) delete folderDataCache[key];
for (let key in filterFolderCache) delete filterFolderCache[key];
}
// 魔法函數:當資料夾被點擊展開時,才將 HTML 渲染進去
window.lazyLoadFolder = function(elementId, isFilter = false) {
const detailsEl = document.getElementById(`details-${elementId}`);
@ -5404,6 +5412,7 @@ export function buildRealFilterTree(data, parentPath = '', hiddenKeys = [], isPa
================================================================================
FILE: routers/backup.py
================================================================================
# --- routers/backup.py ---
import logging
import asyncssh
import asyncio
@ -5423,7 +5432,8 @@ from database import (
)
from cmts_scraper import fetch_raw_config
from shared import parse_cli_to_tree, cmts_config_lock # <== 引入真正的解析器與全域鎖
# 🌟 [Priority 1 修復] 引入 cmts_config_locks
from shared import parse_cli_to_tree, cmts_config_locks
logger = logging.getLogger(__name__)
@ -5629,16 +5639,21 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
return StreamingResponse(empty_success(), media_type="application/x-ndjson")
async def restore_generator():
# 🌟 [Priority 1 修復] 使用依 IP 隔離的鎖
host_lock = cmts_config_locks[req.host]
# 1. 嘗試取得全域寫入鎖 (避免多人同時打指令)
if cmts_config_lock.locked():
if host_lock.locked():
yield json.dumps({"status": "error", "message": "❌ 設備目前正由其他使用者進行配置,請稍後再試。"}) + "\n"
return
async with cmts_config_lock:
async with host_lock:
try:
yield json.dumps({"status": "progress", "message": "🔄 正在建立 SSH 安全連線..."}) + "\n"
conn = await asyncssh.connect(req.host, username=req.username, password=req.password, known_hosts=None)
process = await conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8')
# 🌟 [Priority 2 提前修復] 使用 async with 確保連線與 Process 絕對會被釋放
async with asyncssh.connect(req.host, username=req.username, password=req.password, known_hosts=None) as conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=1.0):
output = ""
@ -5687,7 +5702,7 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
"message": f"❌ 寫入中斷且已安全撤銷!失敗指令: {cmd}",
"output": clean_out
}) + "\n"
conn.close()
# 🌟 移除手動 conn.close(),交由 async with 處理
return
yield json.dumps({
@ -5709,7 +5724,7 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
# 5. 退出並關閉連線
process.stdin.write("exit\n")
await process.stdin.drain()
conn.close()
# 🌟 移除手動 conn.close(),交由 async with 處理
yield json.dumps({
"status": "success",
@ -5724,7 +5739,6 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
return StreamingResponse(restore_generator(), media_type="application/x-ndjson")
================================================================================
FILE: routers/terminal.py
================================================================================
@ -5954,26 +5968,32 @@ def parse_cm_output(raw_text: str) -> list:
@router.get("/cable-modems")
async def get_cable_modems():
net_connect = None
try:
net_connect = ConnectHandler(**CMTS_DEVICE)
raw_output = str(net_connect.send_command("show cable modem"))
net_connect.disconnect()
structured_data = parse_cm_output(raw_output)
return {"status": "success", "total_count": len(structured_data), "data": structured_data}
except Exception as e:
raise HTTPException(status_code=500, detail=f"CMTS Connection Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/configuration")
async def get_configuration():
net_connect = None
try:
net_connect = ConnectHandler(**CMTS_DEVICE)
net_connect.send_command_timing("config")
raw_output = net_connect.send_command("show full-configuration | nomore", expect_string=r"#", read_timeout=120)
net_connect.send_command_timing("exit")
net_connect.disconnect()
return {"status": "success", "data": raw_output}
except Exception as e:
raise HTTPException(status_code=500, detail=f"CMTS Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-query")
async def get_cmts_query(
@ -5983,6 +6003,7 @@ async def get_cmts_query(
username: str = Query(...),
password: str = Query(...)
):
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
@ -6031,14 +6052,17 @@ async def get_cmts_query(
cli_command = commands[query_type] + " | nomore"
net_connect = ConnectHandler(**device)
raw_output = net_connect.send_command(cli_command, read_timeout=15)
net_connect.disconnect()
return {"status": "success", "command": cli_command, "data": raw_output}
except Exception as e:
raise HTTPException(status_code=500, detail=f"CMTS Query Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-mac-domain-config")
async def get_mac_domain_config(target: str, host: str, username: str, password: str):
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
@ -6046,7 +6070,6 @@ async def get_mac_domain_config(target: str, host: str, username: str, password:
cli_command = f"show running-config cable mac-domain {target.strip()} | nomore"
net_connect = ConnectHandler(**device)
raw_output = str(net_connect.send_command(cli_command, read_timeout=15))
net_connect.disconnect()
# 💡 正名工程:更新字典的 Key 為一致性的命名
config = {
@ -6102,6 +6125,9 @@ async def get_mac_domain_config(target: str, host: str, username: str, password:
return {"status": "success", "data": config}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Parse Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-mac-domain-list")
async def get_cmts_mac_domain_list(
@ -6109,13 +6135,13 @@ async def get_cmts_mac_domain_list(
username: str = Query(...),
password: str = Query(...)
):
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
net_connect = ConnectHandler(**device)
raw_output = str(net_connect.send_command_timing("show running-config cable mac-domain ?"))
net_connect.disconnect()
import re
matches = re.findall(r'\b\d+:\d+/\d+\.\d+\b', raw_output)
@ -6124,6 +6150,9 @@ async def get_cmts_mac_domain_list(
return {"status": "success", "data": mac_domains}
except Exception as e:
return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-version")
async def get_cmts_version(
@ -6131,13 +6160,13 @@ async def get_cmts_version(
username: str = Query(...),
password: str = Query(...)
):
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
net_connect = ConnectHandler(**device)
raw_output = str(net_connect.send_command("show version", read_timeout=15))
net_connect.disconnect()
import re
# 🌟 使用 Regex 尋找 infra 或 vcmts-cd-0 後面的版本號
@ -6150,6 +6179,9 @@ async def get_cmts_version(
except Exception as e:
return {"status": "error", "message": f"CMTS Connection Error: {str(e)}"}
finally:
if net_connect:
net_connect.disconnect()
================================================================================
@ -6166,8 +6198,8 @@ from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import List
from netmiko import ConnectHandler
# 🌟 移除了舊的 load_settings, save_settings改用專屬的雙軌機制
from shared import CMTS_DEVICE, cmts_config_lock, parse_cli_to_tree, deep_split_tree, USE_DB
# 🌟 [Priority 1 修復] 引入 cmts_config_locks
from shared import CMTS_DEVICE, cmts_config_locks, parse_cli_to_tree, deep_split_tree, USE_DB
from collections import defaultdict
from fastapi.responses import StreamingResponse
@ -6246,8 +6278,9 @@ async def execute_cmts_config(req: ConfigRequest):
commands = [cmd.strip() for cmd in req.script.splitlines() if cmd.strip() and not cmd.strip().startswith('!')]
async def config_streamer():
# 🛡️ 繼承原本的非同步鎖,確保同一時間只有一個寫入任務執行
async with cmts_config_lock:
# 🌟 [Priority 1 修復] 使用依 IP 隔離的鎖
host_lock = cmts_config_locks[req.host]
async with host_lock:
try:
yield json.dumps({"status": "progress", "message": f"🔌 準備連線至設備 {req.host}..."}) + "\n"
@ -6392,6 +6425,7 @@ async def generate_cli(req: GenerateCliRequest):
@router.get("/cmts-ds-rf-port-list")
async def get_ds_rf_port_list(host: str, username: str, password: str = ""):
"""獲取設備上所有的 DS RF Port 清單"""
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
@ -6399,7 +6433,6 @@ async def get_ds_rf_port_list(host: str, username: str, password: str = ""):
net_connect = ConnectHandler(**device)
command = 'show running-config | include "cable ds-rf-port"'
raw_output = str(net_connect.send_command(command, read_timeout=60))
net_connect.disconnect()
pattern = r"cable ds-rf-port\s+([0-9:/]+)"
ports = re.findall(pattern, raw_output)
@ -6412,11 +6445,15 @@ async def get_ds_rf_port_list(host: str, username: str, password: str = ""):
}
except Exception as e:
return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-ds-rf-port-config")
async def get_ds_rf_port_config(target: str, host: str, username: str, password: str = ""):
"""獲取特定 DS RF Port 的配置,並轉換為樹狀 JSON"""
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
@ -6424,10 +6461,10 @@ async def get_ds_rf_port_config(target: str, host: str, username: str, password:
net_connect = ConnectHandler(**device)
command = f"show running-config interface cable ds-rf-port {target} | nomore"
raw_output = str(net_connect.send_command(command, read_timeout=60))
net_connect.disconnect()
base_tree = parse_cli_to_tree(raw_output)
perfect_tree = deep_split_tree(base_tree)
# 🌟 [Priority 1 修復] 將 CPU-Bound 任務移至 ThreadPool
base_tree = await asyncio.to_thread(parse_cli_to_tree, raw_output)
perfect_tree = await asyncio.to_thread(deep_split_tree, base_tree)
return {
"status": "success",
@ -6436,11 +6473,15 @@ async def get_ds_rf_port_config(target: str, host: str, username: str, password:
}
except Exception as e:
return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-full-config")
async def get_full_config(host: str, username: str, password: str = "", skip_filter: bool = False, config_type: str = "running"):
"""獲取整台 CMTS 的完整配置,並轉換為大型樹狀 JSON"""
net_connect = None
try:
device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password})
@ -6456,13 +6497,12 @@ async def get_full_config(host: str, username: str, password: str = "", skip_fil
command = "show running-config | nomore"
raw_output = str(net_connect.send_command(command, read_timeout=180))
net_connect.disconnect()
clean_output = re.sub(r"^(?:Building configuration\.\.\.|Current configuration.*?)\n+", "", raw_output, flags=re.IGNORECASE | re.MULTILINE)
clean_output = clean_output.lstrip()
base_tree = parse_cli_to_tree(clean_output)
perfect_tree = deep_split_tree(base_tree)
# 🌟 [Priority 1 修復] 將 CPU-Bound 任務移至 ThreadPool
base_tree = await asyncio.to_thread(parse_cli_to_tree, clean_output)
perfect_tree = await asyncio.to_thread(deep_split_tree, base_tree)
# ==========================================
# 🌟 深度過濾攔截器:改用雙軌過濾器讀取邏輯
@ -6488,12 +6528,16 @@ async def get_full_config(host: str, username: str, password: str = "", skip_fil
return {"status": "success", "data": perfect_tree}
except Exception as e:
return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-full-config/stream")
async def get_full_config_stream(host: str, username: str, password: str = "", skip_filter: bool = False, config_type: str = "running"):
"""獲取整台 CMTS 的完整配置 (串流進度回報版)"""
async def config_streamer():
net_connect = None
try:
# 1. 廣播:正在連線
yield json.dumps({"status": "progress", "message": f"🔌 正在建立 SSH 連線至 {host}..."}) + "\n"
@ -6517,8 +6561,6 @@ async def get_full_config_stream(host: str, username: str, password: str = "", s
command = "show running-config | nomore"
raw_output = await asyncio.to_thread(net_connect.send_command, command, read_timeout=180)
await asyncio.to_thread(net_connect.disconnect)
# 3. 廣播:正在解析
yield json.dumps({"status": "progress", "message": "🧩 正在解析配置並建構樹狀圖結構..."}) + "\n"
await asyncio.sleep(0.1)
@ -6526,8 +6568,9 @@ async def get_full_config_stream(host: str, username: str, password: str = "", s
clean_output = re.sub(r"^(?:Building configuration\.\.\.|Current configuration.*?)\n+", "", raw_output, flags=re.IGNORECASE | re.MULTILINE)
clean_output = clean_output.lstrip()
base_tree = parse_cli_to_tree(clean_output)
perfect_tree = deep_split_tree(base_tree)
# 🌟 [Priority 1 修復] 將 CPU-Bound 任務移至 ThreadPool
base_tree = await asyncio.to_thread(parse_cli_to_tree, clean_output)
perfect_tree = await asyncio.to_thread(deep_split_tree, base_tree)
# ==========================================
# 🌟 深度過濾攔截器 (維持原樣)
@ -6556,6 +6599,10 @@ async def get_full_config_stream(host: str, username: str, password: str = "", s
except Exception as e:
yield json.dumps({"status": "error", "message": f"載入失敗: {str(e)}"}) + "\n"
finally:
# 🌟 [Priority 1 修復] 確保背景連線被正確釋放
if net_connect:
await asyncio.to_thread(net_connect.disconnect)
# 回傳 NDJSON 串流格式
return StreamingResponse(config_streamer(), media_type="application/x-ndjson")
@ -6638,10 +6685,12 @@ async def sse_stream(request: Request, host: str, config_type: str = "running"):
async def event_generator():
try:
while True:
# 主動檢查斷線,避免死鎖
if await request.is_disconnected():
break
try:
message = await asyncio.wait_for(client_queue.get(), timeout=1.0)
# [修復 Leak] 延長 timeout 降低輪詢負擔
message = await asyncio.wait_for(client_queue.get(), timeout=5.0)
yield message
except asyncio.TimeoutError:
yield ": keepalive\n\n"
@ -6649,6 +6698,7 @@ async def sse_stream(request: Request, host: str, config_type: str = "running"):
except asyncio.CancelledError:
pass
finally:
# [修復 Leak] 確保斷線時 Queue 絕對會被移出 active_clients 釋放記憶體
if channel_key in active_clients:
active_clients[channel_key].discard(client_queue)

View File

@ -157,11 +157,11 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list, con
for batch_idx, batch in enumerate(batches):
print(f"🔄 正在處理第 {batch_idx + 1}/{len(batches)} 批次 (共 {len(batch)} 個路徑)...")
conn = None
try:
conn = await asyncssh.connect(host, username=username, password=password, known_hosts=None)
process = await conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8')
# 🧹 [穩定性修復] 全面改用 async with 管理生命週期,確保連線與 process 絕對釋放
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=1.0):
output = ""
@ -257,8 +257,6 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list, con
except Exception as e:
yield json.dumps({"event": "error", "message": f"批次錯誤: {str(e)}"}) + "\n"
continue
finally:
if conn: conn.close()
# --- 步驟 3寫入快取 (DB or JSON) ---
try:
@ -340,10 +338,10 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list, con
async def fetch_raw_config(host: str, username: str, password: str, config_type: str = "running") -> str:
"""連線至設備並抓取完整的純文字設定檔"""
conn = None
try:
conn = await asyncssh.connect(host, username=username, password=password, known_hosts=None)
process = await conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8')
# 🧹 [穩定性修復] 全面改用 async with 管理生命週期
async with asyncssh.connect(host, username=username, password=password, known_hosts=None) as conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=2.0):
output = ""
@ -417,6 +415,7 @@ async def fetch_raw_config(host: str, username: str, password: str, config_type:
except Exception as e:
raise Exception(f"SSH 連線或抓取設定失敗: {str(e)}")
finally:
if conn:
conn.close()

View File

@ -51,10 +51,12 @@ async def sse_stream(request: Request, host: str, config_type: str = "running"):
async def event_generator():
try:
while True:
# 主動檢查斷線,避免死鎖
if await request.is_disconnected():
break
try:
message = await asyncio.wait_for(client_queue.get(), timeout=1.0)
# [修復 Leak] 延長 timeout 降低輪詢負擔
message = await asyncio.wait_for(client_queue.get(), timeout=5.0)
yield message
except asyncio.TimeoutError:
yield ": keepalive\n\n"
@ -62,6 +64,7 @@ async def sse_stream(request: Request, host: str, config_type: str = "running"):
except asyncio.CancelledError:
pass
finally:
# [修復 Leak] 確保斷線時 Queue 絕對會被移出 active_clients 釋放記憶體
if channel_key in active_clients:
active_clients[channel_key].discard(client_queue)

View File

@ -19,7 +19,7 @@ import {
import { getGlobalConnectionInfo } from './utils.js';
// 4. 樹狀圖 UI 模組 (Tree UI)
import { buildTree, buildRealFilterTree, expandAll, collapseAll } from './tree-ui.js';
import { buildTree, buildRealFilterTree, expandAll, collapseAll, clearTreeCache } from './tree-ui.js';
// 5. 編輯模式與鎖定模組 (Edit Mode)
import {
@ -320,6 +320,9 @@ function toggleQueryInputs(mode) {
function resetAllManagerData() {
resetMacDomainData();
// 🧹 [修復 Leak] 清空前端樹狀圖快取,避免切換設備時發生 OOM
clearTreeCache();
const configArea = document.getElementById('macDomainConfigArea');
if (configArea) configArea.style.display = 'none';

View File

@ -6,6 +6,12 @@
export const folderDataCache = {};
export const filterFolderCache = {};
// 🧹 [修復 Leak] 徹底清空樹狀圖快取,釋放記憶體
export function clearTreeCache() {
for (let key in folderDataCache) delete folderDataCache[key];
for (let key in filterFolderCache) delete filterFolderCache[key];
}
// 魔法函數:當資料夾被點擊展開時,才將 HTML 渲染進去
window.lazyLoadFolder = function(elementId, isFilter = false) {
const detailsEl = document.getElementById(`details-${elementId}`);