feat: 導入 SSE 串流機制與 API 路由標準化
1. 實作 StreamingResponse 即時回傳爬蟲進度。2. 新增 IS_SCANNING 全域變數防止併發掃描。3. 統一 main.py 路由前綴，修復心跳機制 404 錯誤。
This commit is contained in:
swpa
parent
93eb15ac9c
commit
3fea4a972a
File diff suppressed because it is too large
Load Diff
|
|
@ -142,8 +142,10 @@ def parse_device_response(output: str) -> dict:
|
|||
|
||||
return {"type": "unknown", "options": [], "current_value": None, "raw_output": output.strip()}
|
||||
|
||||
async def sync_cmts_leaves_async(host, username, password, leaf_paths: list) -> dict:
|
||||
async def sync_cmts_leaves_async(host, username, password, leaf_paths: list):
|
||||
try:
|
||||
total_paths = len(leaf_paths)
|
||||
processed_count = 0
|
||||
result_data = {}
|
||||
BATCH_SIZE = 30
|
||||
batches = [leaf_paths[i:i + BATCH_SIZE] for i in range(0, len(leaf_paths), BATCH_SIZE)]
|
||||
|
|
@ -229,12 +231,22 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list) ->
|
|||
|
||||
result_data[path] = parsed_data
|
||||
|
||||
# 🌟 核心修改：每處理完一個路徑，就推播一次進度！
|
||||
processed_count += 1
|
||||
event_data = {
|
||||
"event": "progress",
|
||||
"current": processed_count,
|
||||
"total": total_paths,
|
||||
"current_path": path
|
||||
}
|
||||
yield json.dumps(event_data) + "\n"
|
||||
|
||||
process.stdin.write("exit\n")
|
||||
await process.stdin.drain()
|
||||
await read_until_quiet(timeout=1.0)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ 第 {batch_idx + 1} 批次發生錯誤: {str(e)}")
|
||||
yield json.dumps({"event": "error", "message": f"批次錯誤: {str(e)}"}) + "\n"
|
||||
continue
|
||||
finally:
|
||||
if conn: conn.close()
|
||||
|
|
@ -261,7 +273,7 @@ async def sync_cmts_leaves_async(host, username, password, leaf_paths: list) ->
|
|||
if batch_idx < len(batches) - 1:
|
||||
await asyncio.sleep(2)
|
||||
|
||||
return {"status": "success", "data": result_data}
|
||||
yield json.dumps({"event": "done", "message": "快取更新完成"}) + "\n"
|
||||
|
||||
except Exception as e:
|
||||
return {"status": "error", "message": f"AsyncSSH 爬蟲錯誤: {str(e)}"}
|
||||
yield json.dumps({"event": "error", "message": f"爬蟲嚴重錯誤: {str(e)}"}) + "\n"
|
||||
|
|
|
|||
14
main.py
14
main.py
|
|
@ -12,12 +12,14 @@ app = FastAPI(title="Harmonic CMTS Manager", version="2.0")
|
|||
# 掛載靜態檔案目錄 (對應 static/style.css 與 static/app.js)
|
||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||
|
||||
# 註冊 API 路由
|
||||
app.include_router(query.router, prefix="/api/v1", tags=["Query"])
|
||||
app.include_router(config.router, prefix="/api/v1", tags=["Config"])
|
||||
app.include_router(terminal.router, tags=["Terminal"])
|
||||
app.include_router(lock.router, prefix="/api/v1/locks")
|
||||
app.include_router(leaf_options.router)
|
||||
# 🌟 統一標準：所有 REST API 都在這裡掛載 /api/v1 前綴
|
||||
app.include_router(query.router, prefix="/api/v1")
|
||||
app.include_router(config.router, prefix="/api/v1")
|
||||
app.include_router(leaf_options.router, prefix="/api/v1")
|
||||
app.include_router(lock.router, prefix="/api/v1")
|
||||
|
||||
# WebSocket 通常獨立於 API 版本之外，所以不加前綴
|
||||
app.include_router(terminal.router)
|
||||
|
||||
# 根目錄路由：回傳前端 UI
|
||||
@app.get("/", response_class=HTMLResponse, tags=["UI"])
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
# --- routers/leaf_options.py ---
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Body
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
import json, os, asyncio, time, re
|
||||
|
|
@ -8,69 +9,16 @@ import json, os, asyncio, time, re
|
|||
from cmts_scraper import sync_cmts_leaves_async
|
||||
from shared import CMTS_DEVICE
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["Options"])
|
||||
# 🌟 移除原本硬寫的 prefix="/api/v1"，只留下 tags
|
||||
router = APIRouter(tags=["Options"])
|
||||
CACHE_FILE = "cmts_leaf_options_cache.json"
|
||||
scraper_semaphore = asyncio.Semaphore(1)
|
||||
|
||||
# 🌟 1. 改用全域布林變數，狀態切換更即時，消滅非同步時間差
|
||||
IS_SCANNING = False
|
||||
|
||||
class SyncOptionsRequest(BaseModel):
|
||||
leaf_paths: List[str]
|
||||
|
||||
async def run_scraper_task_async(leaf_paths: List[str]):
|
||||
if scraper_semaphore.locked():
|
||||
print("⚠️ [Debug] 爬蟲正在執行中，本次請求被忽略")
|
||||
return
|
||||
|
||||
async with scraper_semaphore:
|
||||
try:
|
||||
print(f"🚀 [Debug] 背景爬蟲開始啟動，準備抓取: {leaf_paths}")
|
||||
|
||||
# 🌟 雙重防護：確保傳給 CMTS 的是空白分隔，並在後端也徹底拔除 [0], [1] 等虛擬索引
|
||||
cmts_query_paths = []
|
||||
for p in leaf_paths:
|
||||
clean_p = p.replace("::", " ")
|
||||
clean_p = re.sub(r"\s*\[\d+\]", "", clean_p) # 拔除虛擬索引
|
||||
cmts_query_paths.append(clean_p)
|
||||
|
||||
# 去除重複的路徑 (例如 alias [0] 和 alias [1] 淨化後都會變成 alias)
|
||||
cmts_query_paths = list(dict.fromkeys(cmts_query_paths))
|
||||
|
||||
print(f"⏳ [Debug] 正在連線 CMTS 執行 sync_cmts_leaves_async...")
|
||||
|
||||
# 🌟 核心修改：直接 await 非同步爬蟲，不再需要 Netmiko 與 to_thread
|
||||
# 從共用的 CMTS_DEVICE 字典中提取連線資訊
|
||||
result = await sync_cmts_leaves_async(
|
||||
host=CMTS_DEVICE.get("host"),
|
||||
username=CMTS_DEVICE.get("username"),
|
||||
password=CMTS_DEVICE.get("password"),
|
||||
leaf_paths=cmts_query_paths
|
||||
)
|
||||
|
||||
print(f"✅ [Debug] CMTS 抓取完成，結果: {result['status']}")
|
||||
|
||||
if result["status"] == "success":
|
||||
cache_data = {}
|
||||
if os.path.exists(CACHE_FILE):
|
||||
with open(CACHE_FILE, "r", encoding="utf-8") as f:
|
||||
cache_data = json.load(f)
|
||||
|
||||
current_timestamp = int(time.time())
|
||||
|
||||
# 存入快取
|
||||
for cmts_path, data in result["data"].items():
|
||||
data["updated_at"] = current_timestamp
|
||||
cache_data[cmts_path] = data
|
||||
|
||||
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(cache_data, f, indent=4, ensure_ascii=False)
|
||||
print("💾 [Debug] 快取檔案寫入成功！")
|
||||
|
||||
else:
|
||||
error_msg = result.get('message', '沒有提供錯誤訊息')
|
||||
print(f"❌ [Debug] 抓取失敗，CMTS 拒絕了我們！原因: {error_msg}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"💥 [Debug] 背景爬蟲發生嚴重錯誤: {e}")
|
||||
|
||||
@router.get("/cmts-leaf-options")
|
||||
async def get_leaf_options():
|
||||
if not os.path.exists(CACHE_FILE): return {}
|
||||
|
|
@ -78,10 +26,49 @@ async def get_leaf_options():
|
|||
return json.load(f)
|
||||
|
||||
@router.post("/cmts-leaf-options/sync")
|
||||
async def sync_leaf_options(request: SyncOptionsRequest, background_tasks: BackgroundTasks):
|
||||
if not request.leaf_paths: raise HTTPException(status_code=400)
|
||||
background_tasks.add_task(run_scraper_task_async, request.leaf_paths)
|
||||
return {"status": "processing"}
|
||||
async def sync_leaf_options(request: SyncOptionsRequest):
|
||||
global IS_SCANNING # 宣告我們要修改全域變數
|
||||
|
||||
if not request.leaf_paths:
|
||||
raise HTTPException(status_code=400)
|
||||
|
||||
# 🌟 2. 第一時間檢查並「立即」阻擋
|
||||
if IS_SCANNING:
|
||||
async def busy_stream():
|
||||
yield json.dumps({"event": "error", "message": "⚠️ 掃描任務正在執行中，請勿重複點擊！"}) + "\n"
|
||||
return StreamingResponse(busy_stream(), media_type="text/event-stream")
|
||||
|
||||
# 如果沒人掃描，立刻把狀態改為 True (在建立串流之前就上鎖)
|
||||
IS_SCANNING = True
|
||||
|
||||
async def event_generator():
|
||||
global IS_SCANNING
|
||||
try:
|
||||
cmts_query_paths = []
|
||||
for p in request.leaf_paths:
|
||||
clean_p = p.replace("::", " ")
|
||||
clean_p = re.sub(r"\s*\[\d+\]", "", clean_p)
|
||||
cmts_query_paths.append(clean_p)
|
||||
|
||||
cmts_query_paths = list(dict.fromkeys(cmts_query_paths))
|
||||
|
||||
# 🌟 因為爬蟲自己會存檔，我們只要專心把進度轉發給前端就好！
|
||||
async for chunk in sync_cmts_leaves_async(
|
||||
host=CMTS_DEVICE.get("host"),
|
||||
username=CMTS_DEVICE.get("username"),
|
||||
password=CMTS_DEVICE.get("password"),
|
||||
leaf_paths=cmts_query_paths
|
||||
):
|
||||
yield chunk
|
||||
|
||||
except Exception as e:
|
||||
yield json.dumps({"event": "error", "message": str(e)}) + "\n"
|
||||
|
||||
finally:
|
||||
# 🌟 3. 無論成功、失敗、或使用者提早關閉網頁，都確保會解鎖
|
||||
IS_SCANNING = False
|
||||
|
||||
return StreamingResponse(event_generator(), media_type="text/event-stream")
|
||||
|
||||
# 🌟 新增：清除特定選項快取的 API (使用您原本定義的 CACHE_FILE)
|
||||
@router.post("/clear_cache")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ from fastapi import APIRouter, HTTPException
|
|||
from pydantic import BaseModel
|
||||
from typing import Dict, Optional
|
||||
|
||||
router = APIRouter()
|
||||
# 🌟 只定義自己的子路徑
|
||||
router = APIRouter(prefix="/locks", tags=["Lock Management"])
|
||||
|
||||
# ==========================================
|
||||
# 💡 全域鎖定表 (In-Memory Lock Table)
|
||||
|
|
|
|||
|
|
@ -119,3 +119,40 @@ export async function apiExecuteQuery(queryType, target, host, username, passwor
|
|||
const response = await fetch(url);
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// ==========================================
|
||||
// 6. 專門處理串流的 API 呼叫函數 (UI 可以即時更新)
|
||||
// ==========================================
|
||||
export async function apiSyncLeafOptionsStream(paths, onProgress) {
|
||||
const response = await fetch('/api/v1/cmts-leaf-options/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ leaf_paths: paths })
|
||||
});
|
||||
|
||||
if (!response.body) throw new Error("瀏覽器不支援 ReadableStream");
|
||||
|
||||
// 🌟 核心修改：讀取串流資料
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break; // 伺服器斷開連線 (任務完成)
|
||||
|
||||
// 解碼二進位資料為文字
|
||||
const chunk = decoder.decode(value, { stream: true });
|
||||
|
||||
// 因為一次可能收到多行 JSON，我們用換行符號切開
|
||||
const lines = chunk.split("\n").filter(line => line.trim() !== "");
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const data = JSON.parse(line);
|
||||
onProgress(data); // 將解析後的資料傳給 UI 介面
|
||||
} catch (e) {
|
||||
console.error("JSON 解析錯誤:", e, line);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -10,7 +10,8 @@ import { initTerminal, connectWebSocket, disconnectWebSocket, injectCommand, dow
|
|||
// 2. API 網路請求模組 (API)
|
||||
import {
|
||||
apiGetFullConfig, apiClearCache, apiExecuteQuery,
|
||||
apiGetTreeFilters, apiSaveTreeFilters
|
||||
apiGetTreeFilters, apiSaveTreeFilters,
|
||||
apiSyncLeafOptionsStream
|
||||
} from './api.js';
|
||||
|
||||
// 3. 共用工具模組 (Utils)
|
||||
|
|
@ -334,49 +335,20 @@ async function scanAllMissingOptions() {
|
|||
return;
|
||||
}
|
||||
|
||||
statusEl.innerHTML = `⏳ 正在背景掃描 <b>${pathsToSync.length}</b> 個欄位的選項...`;
|
||||
statusEl.innerHTML = `⏳ 正在建立即時連線...`;
|
||||
statusEl.style.color = "#e67e22";
|
||||
|
||||
const response = await fetch('/api/v1/cmts-leaf-options/sync', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ leaf_paths: pathsToSync })
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
statusEl.innerHTML = `✅ 任務已送出！正在監控背景執行進度...`;
|
||||
statusEl.style.color = "#3498db";
|
||||
checkScanProgress(pathsToSync);
|
||||
} else {
|
||||
throw new Error("API 回應錯誤");
|
||||
// 🌟 核心修改：改用串流 API，並傳入進度更新的回呼函數
|
||||
await apiSyncLeafOptionsStream(pathsToSync, (data) => {
|
||||
if (data.event === 'progress') {
|
||||
// 收到進度：即時更新畫面數字與當前路徑
|
||||
statusEl.innerHTML = `⏳ 背景掃描進度：<b>${data.current} / ${data.total}</b> (正在處理: ${data.current_path})`;
|
||||
statusEl.style.color = "#f39c12";
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("掃描請求失敗:", error);
|
||||
statusEl.textContent = "❌ 掃描請求發送失敗，請檢查網路連線。";
|
||||
statusEl.style.color = "#e74c3c";
|
||||
}
|
||||
}
|
||||
|
||||
// 智慧監控背景掃描進度
|
||||
async function checkScanProgress(pendingPaths) {
|
||||
const btn = document.getElementById('global-scan-btn');
|
||||
const statusEl = document.getElementById('scan-status');
|
||||
|
||||
let attempts = 0;
|
||||
const maxAttempts = 60; // 最多監控 5 分鐘 (每 5 秒查一次)
|
||||
|
||||
const interval = setInterval(async () => {
|
||||
attempts++;
|
||||
try {
|
||||
const optRes = await fetch('/api/v1/cmts-leaf-options');
|
||||
const optData = await optRes.json();
|
||||
|
||||
const remainingPaths = pendingPaths.filter(path => !optData[path]);
|
||||
const doneCount = pendingPaths.length - remainingPaths.length;
|
||||
|
||||
if (remainingPaths.length === 0) {
|
||||
clearInterval(interval);
|
||||
localStorage.removeItem('scanLockUntil');
|
||||
else if (data.event === 'done') {
|
||||
// 掃描完成：恢復按鈕狀態並顯示成功
|
||||
statusEl.innerHTML = `✅ 掃描完美達成！快取已全面更新。`;
|
||||
statusEl.style.color = "#27ae60";
|
||||
|
||||
if (btn) {
|
||||
btn.disabled = false;
|
||||
|
|
@ -384,24 +356,29 @@ async function checkScanProgress(pendingPaths) {
|
|||
btn.style.cursor = "pointer";
|
||||
btn.innerText = "🔍 一鍵掃描缺失選項";
|
||||
}
|
||||
statusEl.innerHTML = `✅ 掃描完美達成！快取已全面更新。`;
|
||||
statusEl.style.color = "#27ae60";
|
||||
setTimeout(() => statusEl.textContent = "", 5000);
|
||||
|
||||
} else if (attempts >= maxAttempts) {
|
||||
clearInterval(interval);
|
||||
localStorage.removeItem('scanLockUntil');
|
||||
if (typeof enableGlobalScanButton === 'function') enableGlobalScanButton();
|
||||
setTimeout(() => statusEl.textContent = "", 5000);
|
||||
}
|
||||
else if (data.event === 'error') {
|
||||
// 發生錯誤：顯示紅字並恢復按鈕
|
||||
statusEl.innerHTML = `❌ 錯誤: ${data.message}`;
|
||||
statusEl.style.color = "#e74c3c";
|
||||
|
||||
statusEl.innerHTML = `⚠️ 掃描耗時較長 (${doneCount}/${pendingPaths.length})，已轉為純背景執行。`;
|
||||
statusEl.style.color = "#f39c12";
|
||||
} else {
|
||||
statusEl.innerHTML = `⏳ 背景掃描進度：<b>${doneCount} / ${pendingPaths.length}</b> 完成...`;
|
||||
if (btn) {
|
||||
btn.disabled = false;
|
||||
btn.style.backgroundColor = "#8e44ad";
|
||||
btn.style.cursor = "pointer";
|
||||
btn.innerText = "🔍 一鍵掃描缺失選項";
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("監控進度時發生網路錯誤，稍後重試...", e);
|
||||
localStorage.removeItem('scanLockUntil');
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error("掃描請求失敗:", error);
|
||||
statusEl.textContent = "❌ 掃描請求發送失敗，請檢查網路連線。";
|
||||
statusEl.style.color = "#e74c3c";
|
||||
}
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
// 清除畫面選項快取功能 (精準可見度判定版)
|
||||
|
|
|
|||
Loading…
Reference in New Issue