refactor(backend): 實施 Priority 1 核心穩定性升級與連線防護

- 實作設備級別鎖 (Per-Host Lock):將全域鎖改為 defaultdict(asyncio.Lock),大幅提升多人併發操作能力 (shared.py, config.py, backup.py)。
- 修復 Netmiko 連線洩漏:全面導入 try...finally 機制,確保發生異常時 disconnect() 絕對會被執行 (query.py, config.py)。
- 解除 Event Loop 阻塞:將解析樹狀圖等 CPU-Bound 任務移至 asyncio.to_thread 背景執行,防止 API 請求卡死 (config.py)。
- 強化 AsyncSSH 生命週期:於備份還原路由導入 async with Context Manager,確保 SSH 連線與 Process 絕對乾淨釋放 (backup.py)。
This commit is contained in:
swpa 2026-05-31 14:24:10 +08:00
parent d0d80b26ac
commit 4437e4e3fa
4 changed files with 141 additions and 102 deletions

View File

@ -1,3 +1,4 @@
# --- routers/backup.py ---
import logging import logging
import asyncssh import asyncssh
import asyncio import asyncio
@ -17,7 +18,8 @@ from database import (
) )
from cmts_scraper import fetch_raw_config from cmts_scraper import fetch_raw_config
from shared import parse_cli_to_tree, cmts_config_lock # <== 引入真正的解析器與全域鎖 # 🌟 [Priority 1 修復] 引入 cmts_config_locks
from shared import parse_cli_to_tree, cmts_config_locks
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -223,16 +225,21 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
return StreamingResponse(empty_success(), media_type="application/x-ndjson") return StreamingResponse(empty_success(), media_type="application/x-ndjson")
async def restore_generator(): async def restore_generator():
# 🌟 [Priority 1 修復] 使用依 IP 隔離的鎖
host_lock = cmts_config_locks[req.host]
# 1. 嘗試取得全域寫入鎖 (避免多人同時打指令) # 1. 嘗試取得全域寫入鎖 (避免多人同時打指令)
if cmts_config_lock.locked(): if host_lock.locked():
yield json.dumps({"status": "error", "message": "❌ 設備目前正由其他使用者進行配置,請稍後再試。"}) + "\n" yield json.dumps({"status": "error", "message": "❌ 設備目前正由其他使用者進行配置,請稍後再試。"}) + "\n"
return return
async with cmts_config_lock: async with host_lock:
try: try:
yield json.dumps({"status": "progress", "message": "🔄 正在建立 SSH 安全連線..."}) + "\n" yield json.dumps({"status": "progress", "message": "🔄 正在建立 SSH 安全連線..."}) + "\n"
conn = await asyncssh.connect(req.host, username=req.username, password=req.password, known_hosts=None)
process = await conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') # 🌟 [Priority 2 提前修復] 使用 async with 確保連線與 Process 絕對會被釋放
async with asyncssh.connect(req.host, username=req.username, password=req.password, known_hosts=None) as conn:
async with conn.create_process(term_type='xterm-256color', term_size=(200, 24), encoding='utf-8') as process:
async def read_until_quiet(timeout=1.0): async def read_until_quiet(timeout=1.0):
output = "" output = ""
@ -281,7 +288,7 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
"message": f"❌ 寫入中斷且已安全撤銷!失敗指令: {cmd}", "message": f"❌ 寫入中斷且已安全撤銷!失敗指令: {cmd}",
"output": clean_out "output": clean_out
}) + "\n" }) + "\n"
conn.close() # 🌟 移除手動 conn.close(),交由 async with 處理
return return
yield json.dumps({ yield json.dumps({
@ -303,7 +310,7 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
# 5. 退出並關閉連線 # 5. 退出並關閉連線
process.stdin.write("exit\n") process.stdin.write("exit\n")
await process.stdin.drain() await process.stdin.drain()
conn.close() # 🌟 移除手動 conn.close(),交由 async with 處理
yield json.dumps({ yield json.dumps({
"status": "success", "status": "success",
@ -316,4 +323,3 @@ async def execute_restore(backup_id: str, req: ExecuteRestoreRequest):
yield json.dumps({"status": "error", "message": f"SSH 連線或執行發生例外錯誤: {str(e)}"}) + "\n" yield json.dumps({"status": "error", "message": f"SSH 連線或執行發生例外錯誤: {str(e)}"}) + "\n"
return StreamingResponse(restore_generator(), media_type="application/x-ndjson") return StreamingResponse(restore_generator(), media_type="application/x-ndjson")

View File

@ -9,8 +9,8 @@ from fastapi import APIRouter, HTTPException
from pydantic import BaseModel from pydantic import BaseModel
from typing import List from typing import List
from netmiko import ConnectHandler from netmiko import ConnectHandler
# 🌟 移除了舊的 load_settings, save_settings改用專屬的雙軌機制 # 🌟 [Priority 1 修復] 引入 cmts_config_locks
from shared import CMTS_DEVICE, cmts_config_lock, parse_cli_to_tree, deep_split_tree, USE_DB from shared import CMTS_DEVICE, cmts_config_locks, parse_cli_to_tree, deep_split_tree, USE_DB
from collections import defaultdict from collections import defaultdict
from fastapi.responses import StreamingResponse from fastapi.responses import StreamingResponse
@ -89,8 +89,9 @@ async def execute_cmts_config(req: ConfigRequest):
commands = [cmd.strip() for cmd in req.script.splitlines() if cmd.strip() and not cmd.strip().startswith('!')] commands = [cmd.strip() for cmd in req.script.splitlines() if cmd.strip() and not cmd.strip().startswith('!')]
async def config_streamer(): async def config_streamer():
# 🛡️ 繼承原本的非同步鎖,確保同一時間只有一個寫入任務執行 # 🌟 [Priority 1 修復] 使用依 IP 隔離的鎖
async with cmts_config_lock: host_lock = cmts_config_locks[req.host]
async with host_lock:
try: try:
yield json.dumps({"status": "progress", "message": f"🔌 準備連線至設備 {req.host}..."}) + "\n" yield json.dumps({"status": "progress", "message": f"🔌 準備連線至設備 {req.host}..."}) + "\n"
@ -235,6 +236,7 @@ async def generate_cli(req: GenerateCliRequest):
@router.get("/cmts-ds-rf-port-list") @router.get("/cmts-ds-rf-port-list")
async def get_ds_rf_port_list(host: str, username: str, password: str = ""): async def get_ds_rf_port_list(host: str, username: str, password: str = ""):
"""獲取設備上所有的 DS RF Port 清單""" """獲取設備上所有的 DS RF Port 清單"""
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
@ -242,7 +244,6 @@ async def get_ds_rf_port_list(host: str, username: str, password: str = ""):
net_connect = ConnectHandler(**device) net_connect = ConnectHandler(**device)
command = 'show running-config | include "cable ds-rf-port"' command = 'show running-config | include "cable ds-rf-port"'
raw_output = str(net_connect.send_command(command, read_timeout=60)) raw_output = str(net_connect.send_command(command, read_timeout=60))
net_connect.disconnect()
pattern = r"cable ds-rf-port\s+([0-9:/]+)" pattern = r"cable ds-rf-port\s+([0-9:/]+)"
ports = re.findall(pattern, raw_output) ports = re.findall(pattern, raw_output)
@ -255,11 +256,15 @@ async def get_ds_rf_port_list(host: str, username: str, password: str = ""):
} }
except Exception as e: except Exception as e:
return {"status": "error", "message": str(e)} return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-ds-rf-port-config") @router.get("/cmts-ds-rf-port-config")
async def get_ds_rf_port_config(target: str, host: str, username: str, password: str = ""): async def get_ds_rf_port_config(target: str, host: str, username: str, password: str = ""):
"""獲取特定 DS RF Port 的配置,並轉換為樹狀 JSON""" """獲取特定 DS RF Port 的配置,並轉換為樹狀 JSON"""
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
@ -267,10 +272,10 @@ async def get_ds_rf_port_config(target: str, host: str, username: str, password:
net_connect = ConnectHandler(**device) net_connect = ConnectHandler(**device)
command = f"show running-config interface cable ds-rf-port {target} | nomore" command = f"show running-config interface cable ds-rf-port {target} | nomore"
raw_output = str(net_connect.send_command(command, read_timeout=60)) raw_output = str(net_connect.send_command(command, read_timeout=60))
net_connect.disconnect()
base_tree = parse_cli_to_tree(raw_output) # 🌟 [Priority 1 修復] 將 CPU-Bound 任務移至 ThreadPool
perfect_tree = deep_split_tree(base_tree) base_tree = await asyncio.to_thread(parse_cli_to_tree, raw_output)
perfect_tree = await asyncio.to_thread(deep_split_tree, base_tree)
return { return {
"status": "success", "status": "success",
@ -279,11 +284,15 @@ async def get_ds_rf_port_config(target: str, host: str, username: str, password:
} }
except Exception as e: except Exception as e:
return {"status": "error", "message": str(e)} return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-full-config") @router.get("/cmts-full-config")
async def get_full_config(host: str, username: str, password: str = "", skip_filter: bool = False, config_type: str = "running"): async def get_full_config(host: str, username: str, password: str = "", skip_filter: bool = False, config_type: str = "running"):
"""獲取整台 CMTS 的完整配置,並轉換為大型樹狀 JSON""" """獲取整台 CMTS 的完整配置,並轉換為大型樹狀 JSON"""
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
@ -299,13 +308,12 @@ async def get_full_config(host: str, username: str, password: str = "", skip_fil
command = "show running-config | nomore" command = "show running-config | nomore"
raw_output = str(net_connect.send_command(command, read_timeout=180)) raw_output = str(net_connect.send_command(command, read_timeout=180))
net_connect.disconnect()
clean_output = re.sub(r"^(?:Building configuration\.\.\.|Current configuration.*?)\n+", "", raw_output, flags=re.IGNORECASE | re.MULTILINE) clean_output = re.sub(r"^(?:Building configuration\.\.\.|Current configuration.*?)\n+", "", raw_output, flags=re.IGNORECASE | re.MULTILINE)
clean_output = clean_output.lstrip() clean_output = clean_output.lstrip()
base_tree = parse_cli_to_tree(clean_output) # 🌟 [Priority 1 修復] 將 CPU-Bound 任務移至 ThreadPool
perfect_tree = deep_split_tree(base_tree) base_tree = await asyncio.to_thread(parse_cli_to_tree, clean_output)
perfect_tree = await asyncio.to_thread(deep_split_tree, base_tree)
# ========================================== # ==========================================
# 🌟 深度過濾攔截器:改用雙軌過濾器讀取邏輯 # 🌟 深度過濾攔截器:改用雙軌過濾器讀取邏輯
@ -331,12 +339,16 @@ async def get_full_config(host: str, username: str, password: str = "", skip_fil
return {"status": "success", "data": perfect_tree} return {"status": "success", "data": perfect_tree}
except Exception as e: except Exception as e:
return {"status": "error", "message": str(e)} return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-full-config/stream") @router.get("/cmts-full-config/stream")
async def get_full_config_stream(host: str, username: str, password: str = "", skip_filter: bool = False, config_type: str = "running"): async def get_full_config_stream(host: str, username: str, password: str = "", skip_filter: bool = False, config_type: str = "running"):
"""獲取整台 CMTS 的完整配置 (串流進度回報版)""" """獲取整台 CMTS 的完整配置 (串流進度回報版)"""
async def config_streamer(): async def config_streamer():
net_connect = None
try: try:
# 1. 廣播:正在連線 # 1. 廣播:正在連線
yield json.dumps({"status": "progress", "message": f"🔌 正在建立 SSH 連線至 {host}..."}) + "\n" yield json.dumps({"status": "progress", "message": f"🔌 正在建立 SSH 連線至 {host}..."}) + "\n"
@ -360,8 +372,6 @@ async def get_full_config_stream(host: str, username: str, password: str = "", s
command = "show running-config | nomore" command = "show running-config | nomore"
raw_output = await asyncio.to_thread(net_connect.send_command, command, read_timeout=180) raw_output = await asyncio.to_thread(net_connect.send_command, command, read_timeout=180)
await asyncio.to_thread(net_connect.disconnect)
# 3. 廣播:正在解析 # 3. 廣播:正在解析
yield json.dumps({"status": "progress", "message": "🧩 正在解析配置並建構樹狀圖結構..."}) + "\n" yield json.dumps({"status": "progress", "message": "🧩 正在解析配置並建構樹狀圖結構..."}) + "\n"
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
@ -369,8 +379,9 @@ async def get_full_config_stream(host: str, username: str, password: str = "", s
clean_output = re.sub(r"^(?:Building configuration\.\.\.|Current configuration.*?)\n+", "", raw_output, flags=re.IGNORECASE | re.MULTILINE) clean_output = re.sub(r"^(?:Building configuration\.\.\.|Current configuration.*?)\n+", "", raw_output, flags=re.IGNORECASE | re.MULTILINE)
clean_output = clean_output.lstrip() clean_output = clean_output.lstrip()
base_tree = parse_cli_to_tree(clean_output) # 🌟 [Priority 1 修復] 將 CPU-Bound 任務移至 ThreadPool
perfect_tree = deep_split_tree(base_tree) base_tree = await asyncio.to_thread(parse_cli_to_tree, clean_output)
perfect_tree = await asyncio.to_thread(deep_split_tree, base_tree)
# ========================================== # ==========================================
# 🌟 深度過濾攔截器 (維持原樣) # 🌟 深度過濾攔截器 (維持原樣)
@ -399,6 +410,10 @@ async def get_full_config_stream(host: str, username: str, password: str = "", s
except Exception as e: except Exception as e:
yield json.dumps({"status": "error", "message": f"載入失敗: {str(e)}"}) + "\n" yield json.dumps({"status": "error", "message": f"載入失敗: {str(e)}"}) + "\n"
finally:
# 🌟 [Priority 1 修復] 確保背景連線被正確釋放
if net_connect:
await asyncio.to_thread(net_connect.disconnect)
# 回傳 NDJSON 串流格式 # 回傳 NDJSON 串流格式
return StreamingResponse(config_streamer(), media_type="application/x-ndjson") return StreamingResponse(config_streamer(), media_type="application/x-ndjson")

View File

@ -30,26 +30,32 @@ def parse_cm_output(raw_text: str) -> list:
@router.get("/cable-modems") @router.get("/cable-modems")
async def get_cable_modems(): async def get_cable_modems():
net_connect = None
try: try:
net_connect = ConnectHandler(**CMTS_DEVICE) net_connect = ConnectHandler(**CMTS_DEVICE)
raw_output = str(net_connect.send_command("show cable modem")) raw_output = str(net_connect.send_command("show cable modem"))
net_connect.disconnect()
structured_data = parse_cm_output(raw_output) structured_data = parse_cm_output(raw_output)
return {"status": "success", "total_count": len(structured_data), "data": structured_data} return {"status": "success", "total_count": len(structured_data), "data": structured_data}
except Exception as e: except Exception as e:
raise HTTPException(status_code=500, detail=f"CMTS Connection Error: {str(e)}") raise HTTPException(status_code=500, detail=f"CMTS Connection Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/configuration") @router.get("/configuration")
async def get_configuration(): async def get_configuration():
net_connect = None
try: try:
net_connect = ConnectHandler(**CMTS_DEVICE) net_connect = ConnectHandler(**CMTS_DEVICE)
net_connect.send_command_timing("config") net_connect.send_command_timing("config")
raw_output = net_connect.send_command("show full-configuration | nomore", expect_string=r"#", read_timeout=120) raw_output = net_connect.send_command("show full-configuration | nomore", expect_string=r"#", read_timeout=120)
net_connect.send_command_timing("exit") net_connect.send_command_timing("exit")
net_connect.disconnect()
return {"status": "success", "data": raw_output} return {"status": "success", "data": raw_output}
except Exception as e: except Exception as e:
raise HTTPException(status_code=500, detail=f"CMTS Error: {str(e)}") raise HTTPException(status_code=500, detail=f"CMTS Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-query") @router.get("/cmts-query")
async def get_cmts_query( async def get_cmts_query(
@ -59,6 +65,7 @@ async def get_cmts_query(
username: str = Query(...), username: str = Query(...),
password: str = Query(...) password: str = Query(...)
): ):
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
@ -107,14 +114,17 @@ async def get_cmts_query(
cli_command = commands[query_type] + " | nomore" cli_command = commands[query_type] + " | nomore"
net_connect = ConnectHandler(**device) net_connect = ConnectHandler(**device)
raw_output = net_connect.send_command(cli_command, read_timeout=15) raw_output = net_connect.send_command(cli_command, read_timeout=15)
net_connect.disconnect()
return {"status": "success", "command": cli_command, "data": raw_output} return {"status": "success", "command": cli_command, "data": raw_output}
except Exception as e: except Exception as e:
raise HTTPException(status_code=500, detail=f"CMTS Query Error: {str(e)}") raise HTTPException(status_code=500, detail=f"CMTS Query Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-mac-domain-config") @router.get("/cmts-mac-domain-config")
async def get_mac_domain_config(target: str, host: str, username: str, password: str): async def get_mac_domain_config(target: str, host: str, username: str, password: str):
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
@ -122,7 +132,6 @@ async def get_mac_domain_config(target: str, host: str, username: str, password:
cli_command = f"show running-config cable mac-domain {target.strip()} | nomore" cli_command = f"show running-config cable mac-domain {target.strip()} | nomore"
net_connect = ConnectHandler(**device) net_connect = ConnectHandler(**device)
raw_output = str(net_connect.send_command(cli_command, read_timeout=15)) raw_output = str(net_connect.send_command(cli_command, read_timeout=15))
net_connect.disconnect()
# 💡 正名工程:更新字典的 Key 為一致性的命名 # 💡 正名工程:更新字典的 Key 為一致性的命名
config = { config = {
@ -178,6 +187,9 @@ async def get_mac_domain_config(target: str, host: str, username: str, password:
return {"status": "success", "data": config} return {"status": "success", "data": config}
except Exception as e: except Exception as e:
raise HTTPException(status_code=500, detail=f"Parse Error: {str(e)}") raise HTTPException(status_code=500, detail=f"Parse Error: {str(e)}")
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-mac-domain-list") @router.get("/cmts-mac-domain-list")
async def get_cmts_mac_domain_list( async def get_cmts_mac_domain_list(
@ -185,13 +197,13 @@ async def get_cmts_mac_domain_list(
username: str = Query(...), username: str = Query(...),
password: str = Query(...) password: str = Query(...)
): ):
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
net_connect = ConnectHandler(**device) net_connect = ConnectHandler(**device)
raw_output = str(net_connect.send_command_timing("show running-config cable mac-domain ?")) raw_output = str(net_connect.send_command_timing("show running-config cable mac-domain ?"))
net_connect.disconnect()
import re import re
matches = re.findall(r'\b\d+:\d+/\d+\.\d+\b', raw_output) matches = re.findall(r'\b\d+:\d+/\d+\.\d+\b', raw_output)
@ -200,6 +212,9 @@ async def get_cmts_mac_domain_list(
return {"status": "success", "data": mac_domains} return {"status": "success", "data": mac_domains}
except Exception as e: except Exception as e:
return {"status": "error", "message": str(e)} return {"status": "error", "message": str(e)}
finally:
if net_connect:
net_connect.disconnect()
@router.get("/cmts-version") @router.get("/cmts-version")
async def get_cmts_version( async def get_cmts_version(
@ -207,13 +222,13 @@ async def get_cmts_version(
username: str = Query(...), username: str = Query(...),
password: str = Query(...) password: str = Query(...)
): ):
net_connect = None
try: try:
device = CMTS_DEVICE.copy() device = CMTS_DEVICE.copy()
device.update({'host': host, 'username': username, 'password': password}) device.update({'host': host, 'username': username, 'password': password})
net_connect = ConnectHandler(**device) net_connect = ConnectHandler(**device)
raw_output = str(net_connect.send_command("show version", read_timeout=15)) raw_output = str(net_connect.send_command("show version", read_timeout=15))
net_connect.disconnect()
import re import re
# 🌟 使用 Regex 尋找 infra 或 vcmts-cd-0 後面的版本號 # 🌟 使用 Regex 尋找 infra 或 vcmts-cd-0 後面的版本號
@ -226,3 +241,6 @@ async def get_cmts_version(
except Exception as e: except Exception as e:
return {"status": "error", "message": f"CMTS Connection Error: {str(e)}"} return {"status": "error", "message": f"CMTS Connection Error: {str(e)}"}
finally:
if net_connect:
net_connect.disconnect()

View File

@ -258,5 +258,5 @@ def save_settings(settings_data):
with open(SETTINGS_FILE, "w", encoding="utf-8") as f: with open(SETTINGS_FILE, "w", encoding="utf-8") as f:
json.dump(settings_data, f, indent=4, ensure_ascii=False) json.dump(settings_data, f, indent=4, ensure_ascii=False)
# 全域非同步鎖:確保同一時間只有一人能執行設定寫入 # 🌟 [Priority 1 修復] 將全域非同步鎖改為依設備 IP (Host) 隔離的鎖
cmts_config_lock = asyncio.Lock() cmts_config_locks = defaultdict(asyncio.Lock)