From 0cf78d3d61dceebf04bfc52e787ed3b2caddfb61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=AE=E7=94=9F=EF=BC=88=E5=AD=90=E8=99=9A=EF=BC=89?= <2234839456@qq.com> Date: Wed, 11 Nov 2026 20:08:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8A=E4=BC=A0=E9=93=BE?= =?UTF-8?q?=E8=B7=AF=E4=B8=8E=E4=B8=B4=E6=97=B6=E5=AD=97=E4=BD=93=E6=B8=85?= =?UTF-8?q?=E7=90=86=EF=BC=88dynaPM=20=E6=9E=B6=E6=9E=84=E9=80=82=E9=85=8D?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - webfont-rs 支持 chunked 请求体:dynaPM 网关流式转发剥 Content-Length 后用 Transfer-Encoding: chunked,原 extract_frame 只认 Content-Length 导致 POST body 恒空(上传接口 400 请求体为空)。取帧支持 chunked + 解码重组为等价 Content-Length 帧,含单测 - 上传限制 50MB → 90MB(TS/rust 双端),连接缓冲 1MB → 100MB 适配大上传 - temp-cleaner 双坑修复(TS/rust 双端):删字体时同步删伴生 .meta.json + 孤儿 meta 兜底扫描(此前孤儿元数据无限堆积);首轮清理由延迟 60s 改为 启动立即执行(dynaPM 闲置 30s 停服,延迟首轮导致清理器几乎从未运行) --- backend/app.ts | 6 +- backend/temp_cleaner.ts | 42 ++++++++---- rust-spike/src/main.rs | 142 ++++++++++++++++++++++++++++++++++++++-- rust-spike/src/misc.rs | 33 ++++++++-- 4 files changed, 198 insertions(+), 25 deletions(-) diff --git a/backend/app.ts b/backend/app.ts index 8528cd2..5bab276 100644 --- a/backend/app.ts +++ b/backend/app.ts @@ -190,8 +190,8 @@ const fontApiMiddleware: cMiddleware = async (req, res, next) => { return next(req, res); }; -/** 上传文件大小限制 50MB */ -const MAX_UPLOAD_SIZE = 50 * 1024 * 1024; +/** 上传文件大小限制 90MB */ +const MAX_UPLOAD_SIZE = 90 * 1024 * 1024; const uploadSizeMiddleware: cMiddleware = async (req, res, next) => { if (req.method === "POST" && parseUrl(req).pathname === "/api/upload") { @@ -199,7 +199,7 @@ const uploadSizeMiddleware: cMiddleware = async (req, res, next) => { if (contentLength > MAX_UPLOAD_SIZE) { return { req, - res: jsonResponse({ success: false, error: "文件过大,最大 50MB" }, 413), + res: jsonResponse({ success: false, error: "文件过大,最大 90MB" }, 413), }; } } diff --git a/backend/temp_cleaner.ts b/backend/temp_cleaner.ts index f90e779..76c7a39 100644 --- a/backend/temp_cleaner.ts +++ b/backend/temp_cleaner.ts @@ -43,28 +43,46 @@ async function cleanOnce(): Promise { return; } - let cleaned = 0; + /** 在册字体文件名集合:用于识别孤儿 meta(字体已删但 meta 残留) */ + const liveFonts = new Set(); + /** 孤儿 meta 清单(.meta.json 后缀且主文件是字体扩展名) */ + const orphanMetas: Array<{ base: string; name: string }> = []; for (const entry of entries) { if (!entry.isFile()) continue; - if (!/\.(ttf|otf|woff|woff2)$/i.test(entry.name)) continue; + if (/\.(ttf|otf|woff|woff2)$/i.test(entry.name)) { + liveFonts.add(entry.name); + } else if (/\.(ttf|otf|woff|woff2)\.meta\.json$/i.test(entry.name)) { + orphanMetas.push({ base: entry.name.replace(/\.meta\.json$/i, ""), name: entry.name }); + } + } - const filePath = path_join(TEMP_DIR, entry.name); + let cleaned = 0; + for (const name of liveFonts) { + const filePath = path_join(TEMP_DIR, name); try { const s = await stat(filePath); /** 取"最后使用时间"和"文件修改时间"的较大值作为活跃判定基准 */ - const lastUsed = lastUsedMap.get(entry.name) ?? 0; + const lastUsed = lastUsedMap.get(name) ?? 0; const lastActive = Math.max(lastUsed, s.mtimeMs); if (now - lastActive > retentionMs) { await unlink(filePath); - lastUsedMap.delete(entry.name); + /** 伴生 meta 一并删除,避免孤儿残留 */ + await unlink(path_join(TEMP_DIR, `${name}.meta.json`)).catch(() => {}); + lastUsedMap.delete(name); cleaned++; } } catch (err) { - console.log("[temp-cleaner] 处理文件失败:", entry.name, err); + console.log("[temp-cleaner] 处理文件失败:", name, err); } } + /** 孤儿 meta 兜底:字体本体已不存在的 meta 直接删(本体不在,meta 无引用价值) */ + for (const { base, name } of orphanMetas) { + if (liveFonts.has(base)) continue; + await unlink(path_join(TEMP_DIR, name)).catch(() => {}); + cleaned++; + } if (cleaned > 0) { - console.log(`[temp-cleaner] 本次清理 ${cleaned} 个过期临时字体`); + console.log(`[temp-cleaner] 本次清理 ${cleaned} 个过期临时字体/孤儿元数据`); } } @@ -73,18 +91,18 @@ const CLEAN_INTERVAL = Math.max(tempRetentionSeconds * 500, 300_000); /** * 启动定时清理器。 - * 首次延迟 1 分钟执行(避免启动峰),之后按周期循环。 + * 首轮立即执行(按需启停下服务可能活不过 60s 延迟,冷启动即扫保证不漏清),之后按周期循环。 */ export function startTempCleaner(): void { const intervalSec = Math.round(CLEAN_INTERVAL / 1000); console.log(`[temp-cleaner] 启动,保留时限 ${tempRetentionSeconds}s,清理周期 ${intervalSec}s`); - /** 首次延迟 60 秒 */ + /** 首轮立即执行 */ + cleanOnce().catch(() => {}); + /** 首个周期点起按周期循环 */ setTimeout(() => { - cleanOnce().catch(() => {}); - /** 后续按周期循环 */ setInterval(() => { cleanOnce().catch(() => {}); }, CLEAN_INTERVAL); - }, 60_000); + }, CLEAN_INTERVAL); } diff --git a/rust-spike/src/main.rs b/rust-spike/src/main.rs index 80cbebd..50d7721 100644 --- a/rust-spike/src/main.rs +++ b/rust-spike/src/main.rs @@ -198,14 +198,51 @@ struct Request<'a> { if_none_match: Option, } +/// chunked 帧解码重组:剔除 Transfer-Encoding 头、补 Content-Length、body 展平为连续字节, +/// 返回重组后的帧总长(非 chunked 帧原样返回 None) +fn normalize_chunked_frame(buf: &mut Vec, total: usize) -> Option { + let hdr_end = find(&buf[..total], b"\r\n\r\n")?; + let head = buf[..hdr_end].to_vec(); + if !head_is_chunked(&head) { + return None; + } + let decoded = decode_chunked(&buf[hdr_end + 4..total])?; + let mut new_head: Vec = Vec::with_capacity(hdr_end + 24); + for line in head.split(|&b| b == b'\n') { + let l = trim_cr(line); + if l.is_empty() { + continue; + } + let is_te = l + .iter() + .position(|&b| b == b':') + .map(|colon| l[..colon].eq_ignore_ascii_case(b"transfer-encoding")) + .unwrap_or(false); + if is_te { + continue; + } + new_head.extend_from_slice(l); + new_head.extend_from_slice(b"\r\n"); + } + new_head.extend_from_slice(format!("Content-Length: {}\r\n\r\n", decoded.len()).as_bytes()); + let head_len = new_head.len(); + let mut frame = new_head; + frame.extend_from_slice(&decoded); + frame.extend_from_slice(&buf[total..]); + *buf = frame; + Some(head_len + decoded.len()) +} + /// 连接线程主体:keep-alive 循环,按 Content-Length 完整取帧后处理(对译 connThread/extractFrame) fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()> { let mut buf: Vec = Vec::with_capacity(16 * 1024); let mut rbuf = [0u8; 16 * 1024]; loop { if let Some(total) = extract_frame(&buf) { - let req_bytes = &buf[..total]; - let close = process_request(app, &mut stream, req_bytes)?; + // 反向代理(如 dynaPM)流式转发无 Content-Length,用 chunked 编码——解码重组后再路由 + let total = normalize_chunked_frame(&mut buf, total).unwrap_or(total); + let req_bytes = buf[..total].to_vec(); + let close = process_request(app, &mut stream, &req_bytes)?; // 移除已消费字节(剩余是下一请求的开头) buf.drain(..total); if close { @@ -213,7 +250,8 @@ fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()> } continue; } - if buf.len() > 1024 * 1024 { + // 缓冲上限须覆盖最大上传(90MB + 头部余量) + if buf.len() > 100 * 1024 * 1024 { let _ = http::send_text(&mut stream, 400, "request too large", true, &[]); return Ok(()); } @@ -225,10 +263,78 @@ fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()> } } -/// 判定缓冲里是否有完整请求帧(头结束 + Content-Length 全 body);无则 None(继续等读) +/** 请求头是否声明 Transfer-Encoding: chunked(取帧与重组共用) */ +fn head_is_chunked(head: &[u8]) -> bool { + head.split(|&b| b == b'\n').any(|h| { + let h = trim_cr(h); + let Some(colon) = h.iter().position(|&b| b == b':') else { return false }; + if !h[..colon].eq_ignore_ascii_case(b"transfer-encoding") { + return false; + } + let mut v = &h[colon + 1..]; + while let Some(&f) = v.first() { + if f == b' ' || f == b'\t' { v = &v[1..]; } else { break; } + } + v.eq_ignore_ascii_case(b"chunked") + }) +} + +/// 解码 chunked body(非法数据返 None) +fn decode_chunked(body: &[u8]) -> Option> { + let mut out = Vec::new(); + let mut i = 0; + loop { + let line_end = find(&body[i..], b"\r\n")? + i; + let size_str = &body[i..line_end]; + let semi = size_str.iter().position(|&b| b == b';').unwrap_or(size_str.len()); + let size = usize::from_str_radix(std::str::from_utf8(&size_str[..semi]).ok()?.trim(), 16).ok()?; + i = line_end + 2; + if size == 0 { + return Some(out); + } + if i + size + 2 > body.len() { + return None; + } + out.extend_from_slice(&body[i..i + size]); + i += size + 2; + } +} + +/// chunked 帧完整长度(0-chunk + 可选 trailer + 空行;数据不足返 None 继续等读) +fn chunked_frame_len(body: &[u8]) -> Option { + let mut i = 0; + loop { + let line_end = find(&body[i..], b"\r\n")? + i; + let size_str = &body[i..line_end]; + let semi = size_str.iter().position(|&b| b == b';').unwrap_or(size_str.len()); + let size = usize::from_str_radix(std::str::from_utf8(&size_str[..semi]).ok()?.trim(), 16).ok()?; + i = line_end + 2; + if size == 0 { + // 0-chunk 后:直接空行结束,或若干 trailer 行 + 空行 + loop { + let e = find(&body[i..], b"\r\n")? + i; + let line = &body[i..e]; + i = e + 2; + if line.is_empty() { + return Some(i); + } + } + } + if i + size + 2 > body.len() { + return None; + } + i += size + 2; + } +} + +/// 判定缓冲里是否有完整请求帧(头结束 + Content-Length 全 body / chunked 整帧);无则 None(继续等读) fn extract_frame(req: &[u8]) -> Option { let hdr_end = find(req, b"\r\n\r\n")?; let head = &req[..hdr_end]; + if head_is_chunked(head) { + let body_len = chunked_frame_len(&req[hdr_end + 4..])?; + return Some(hdr_end + 4 + body_len); + } let mut content_length = 0usize; for h in head.split(|&b| b == b'\n') { let h = trim_cr(h); @@ -642,6 +748,34 @@ mod tests { assert_eq!(extract_frame(b"GET / HTTP/1.1\r\n"), None); // 头未结束 } + #[test] + fn chunked_frame_extract_and_normalize() { + // 两个 chunk + 0 结束(无 trailer) + let head = b"POST /api/upload HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"; + let body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"; + let mut full = head.to_vec(); + full.extend_from_slice(body); + assert_eq!(extract_frame(&full), Some(full.len())); + // 数据不足:截断最后一部分后不完整 + assert_eq!(extract_frame(&full[..full.len() - 2]), None); + + // 重组:Transfer-Encoding 剔除、补 Content-Length、body 展平 + let total = extract_frame(&full).unwrap(); + let new_total = normalize_chunked_frame(&mut full, total).unwrap(); + assert_eq!(new_total, full.len()); + let r = parse_request(&full).unwrap(); + assert_eq!(r.method, "POST"); + assert_eq!(r.body, b"Wikipedia"); + assert!(full.starts_with(b"POST /api/upload HTTP/1.1\r\nContent-Length: 9\r\n\r\n")); + + // 带 trailer 的 0-chunk + let mut t = head.to_vec(); + t.extend_from_slice(b"3\r\nabc\r\n0\r\nX-T: 1\r\n\r\n"); + let total = extract_frame(&t).unwrap(); + normalize_chunked_frame(&mut t, total).unwrap(); + assert_eq!(parse_request(&t).unwrap().body, b"abc"); + } + #[test] fn parse_request_fields() { let req = b"GET /api?font=a.ttf&text=%E5%AD%97 HTTP/1.1\r\nConnection: close\r\n\r\nbody"; diff --git a/rust-spike/src/misc.rs b/rust-spike/src/misc.rs index ced1a8f..ec92aa7 100644 --- a/rust-spike/src/misc.rs +++ b/rust-spike/src/misc.rs @@ -285,6 +285,10 @@ fn clean_once() { let retention_ms = config().temp_retention_seconds * 1000; let Ok(entries) = std::fs::read_dir("font/temp") else { return }; let mut names: Vec = Vec::new(); + // 在册字体文件名集合:用于识别孤儿 meta(字体已删但 meta 残留) + let mut live_fonts: Vec = Vec::new(); + // 孤儿 meta 清单(字体本体不存在或同样过期的 .meta.json) + let mut orphan_metas: Vec<(String, String)> = Vec::new(); for entry in entries.flatten() { // 只删普通文件(对齐 entry.kind != .file 跳过) let Ok(ft) = entry.file_type() else { continue }; @@ -293,10 +297,14 @@ fn clean_once() { } let fname = entry.file_name(); let Some(fname) = fname.to_str() else { continue }; - if !crate::has_font_ext(fname) { - continue; + if crate::has_font_ext(fname) { + live_fonts.push(fname.to_string()); + names.push(fname.to_string()); + } else if let Some(base) = fname.strip_suffix(".meta.json") { + if crate::has_font_ext(base) { + orphan_metas.push((base.to_string(), fname.to_string())); + } } - names.push(fname.to_string()); } let mut cleaned: usize = 0; for name in names { @@ -313,12 +321,22 @@ fn clean_once() { if std::fs::remove_file(format!("font/temp/{name}")).is_err() { continue; } + // 伴生 meta 一并删除,避免孤儿残留 + let _ = std::fs::remove_file(format!("font/temp/{name}.meta.json")); with_last_used(|m| m.remove(&name)); cleaned += 1; } } + // 孤儿 meta 兜底:字体本体已不存在的 meta 直接删(本体都不在了,meta 无引用价值) + for (base, meta_name) in orphan_metas { + if !live_fonts.contains(&base) { + if std::fs::remove_file(format!("font/temp/{meta_name}")).is_ok() { + cleaned += 1; + } + } + } if cleaned > 0 { - eprintln!("[temp-cleaner] 本次清理 {cleaned} 个过期临时字体"); + eprintln!("[temp-cleaner] 本次清理 {cleaned} 个过期临时字体/孤儿元数据"); } } @@ -330,6 +348,9 @@ fn temp_cleaner_thread() { config().temp_retention_seconds, clean_interval_ms / 1000 ); + // 首轮清理立即执行:按需启停(如 dynaPM 闲置回收)下服务可能活不过 60s 延迟, + // 冷启动即扫一遍保证过期文件不因进程短命而漏清 + clean_once(); std::thread::sleep(Duration::from_secs(60)); loop { clean_once(); @@ -1142,7 +1163,7 @@ fn match_quoted_attr<'a>(header: &'a [u8], attr: &[u8]) -> Option<&'a [u8]> { // ===== POST /api/upload(对齐 backend/routes/upload.ts + upload.ts)===== -const MAX_UPLOAD_SIZE: usize = 50 * 1024 * 1024; +const MAX_UPLOAD_SIZE: usize = 90 * 1024 * 1024; /// 允许的字体扩展名(对齐 isAllowedFontFile) fn is_allowed_font_file(filename: &str) -> bool { @@ -1340,7 +1361,7 @@ pub fn handle_upload( .and_then(|cl| cl.parse::().ok()) .unwrap_or(0); if content_length > MAX_UPLOAD_SIZE as i64 { - let body = write_upload_result(&UploadResult { success: false, err: Some("文件过大,最大 50MB") }); + let body = write_upload_result(&UploadResult { success: false, err: Some("文件过大,最大 90MB") }); http::send_json(stream, 413, &body, close)?; return Ok(close); }