修复上传链路与临时字体清理(dynaPM 架构适配)

- webfont-rs 支持 chunked 请求体:dynaPM 网关流式转发剥 Content-Length 后用
  Transfer-Encoding: chunked,原 extract_frame 只认 Content-Length 导致 POST
  body 恒空(上传接口 400 请求体为空)。取帧支持 chunked + 解码重组为等价
  Content-Length 帧,含单测
- 上传限制 50MB → 90MB(TS/rust 双端),连接缓冲 1MB → 100MB 适配大上传
- temp-cleaner 双坑修复(TS/rust 双端):删字体时同步删伴生 .meta.json +
  孤儿 meta 兜底扫描(此前孤儿元数据无限堆积);首轮清理由延迟 60s 改为
  启动立即执行(dynaPM 闲置 30s 停服,延迟首轮导致清理器几乎从未运行)
This commit is contained in:
崮生(子虚) 2026-11-11 20:08:50 +08:00
parent c630d5b5c9
commit 0cf78d3d61
4 changed files with 198 additions and 25 deletions

View File

@ -190,8 +190,8 @@ const fontApiMiddleware: cMiddleware = async (req, res, next) => {
return next(req, res);
};
/** 上传文件大小限制 50MB */
const MAX_UPLOAD_SIZE = 50 * 1024 * 1024;
/** 上传文件大小限制 90MB */
const MAX_UPLOAD_SIZE = 90 * 1024 * 1024;
const uploadSizeMiddleware: cMiddleware = async (req, res, next) => {
if (req.method === "POST" && parseUrl(req).pathname === "/api/upload") {
@ -199,7 +199,7 @@ const uploadSizeMiddleware: cMiddleware = async (req, res, next) => {
if (contentLength > MAX_UPLOAD_SIZE) {
return {
req,
res: jsonResponse({ success: false, error: "文件过大,最大 50MB" }, 413),
res: jsonResponse({ success: false, error: "文件过大,最大 90MB" }, 413),
};
}
}

View File

@ -43,28 +43,46 @@ async function cleanOnce(): Promise<void> {
return;
}
let cleaned = 0;
/** 在册字体文件名集合:用于识别孤儿 meta(字体已删但 meta 残留) */
const liveFonts = new Set<string>();
/** 孤儿 meta 清单(.meta.json 后缀且主文件是字体扩展名) */
const orphanMetas: Array<{ base: string; name: string }> = [];
for (const entry of entries) {
if (!entry.isFile()) continue;
if (!/\.(ttf|otf|woff|woff2)$/i.test(entry.name)) continue;
if (/\.(ttf|otf|woff|woff2)$/i.test(entry.name)) {
liveFonts.add(entry.name);
} else if (/\.(ttf|otf|woff|woff2)\.meta\.json$/i.test(entry.name)) {
orphanMetas.push({ base: entry.name.replace(/\.meta\.json$/i, ""), name: entry.name });
}
}
const filePath = path_join(TEMP_DIR, entry.name);
let cleaned = 0;
for (const name of liveFonts) {
const filePath = path_join(TEMP_DIR, name);
try {
const s = await stat(filePath);
/** 取"最后使用时间"和"文件修改时间"的较大值作为活跃判定基准 */
const lastUsed = lastUsedMap.get(entry.name) ?? 0;
const lastUsed = lastUsedMap.get(name) ?? 0;
const lastActive = Math.max(lastUsed, s.mtimeMs);
if (now - lastActive > retentionMs) {
await unlink(filePath);
lastUsedMap.delete(entry.name);
/** 伴生 meta 一并删除,避免孤儿残留 */
await unlink(path_join(TEMP_DIR, `${name}.meta.json`)).catch(() => {});
lastUsedMap.delete(name);
cleaned++;
}
} catch (err) {
console.log("[temp-cleaner] 处理文件失败:", entry.name, err);
console.log("[temp-cleaner] 处理文件失败:", name, err);
}
}
/** 孤儿 meta 兜底:字体本体已不存在的 meta 直接删(本体不在,meta 无引用价值) */
for (const { base, name } of orphanMetas) {
if (liveFonts.has(base)) continue;
await unlink(path_join(TEMP_DIR, name)).catch(() => {});
cleaned++;
}
if (cleaned > 0) {
console.log(`[temp-cleaner] 本次清理 ${cleaned} 个过期临时字体`);
console.log(`[temp-cleaner] 本次清理 ${cleaned} 个过期临时字体/孤儿元数据`);
}
}
@ -73,18 +91,18 @@ const CLEAN_INTERVAL = Math.max(tempRetentionSeconds * 500, 300_000);
/**
* 启动定时清理器。
* 首次延迟 1 分钟执行(避免启动峰),之后按周期循环。
* 首轮立即执行(按需启停下服务可能活不过 60s 延迟,冷启动即扫保证不漏清),之后按周期循环。
*/
export function startTempCleaner(): void {
const intervalSec = Math.round(CLEAN_INTERVAL / 1000);
console.log(`[temp-cleaner] 启动,保留时限 ${tempRetentionSeconds}s,清理周期 ${intervalSec}s`);
/** 首次延迟 60 秒 */
/** 首轮立即执行 */
cleanOnce().catch(() => {});
/** 首个周期点起按周期循环 */
setTimeout(() => {
cleanOnce().catch(() => {});
/** 后续按周期循环 */
setInterval(() => {
cleanOnce().catch(() => {});
}, CLEAN_INTERVAL);
}, 60_000);
}, CLEAN_INTERVAL);
}

View File

@ -198,14 +198,51 @@ struct Request<'a> {
if_none_match: Option<String>,
}
/// chunked 帧解码重组:剔除 Transfer-Encoding 头、补 Content-Length、body 展平为连续字节,
/// 返回重组后的帧总长(非 chunked 帧原样返回 None)
fn normalize_chunked_frame(buf: &mut Vec<u8>, total: usize) -> Option<usize> {
let hdr_end = find(&buf[..total], b"\r\n\r\n")?;
let head = buf[..hdr_end].to_vec();
if !head_is_chunked(&head) {
return None;
}
let decoded = decode_chunked(&buf[hdr_end + 4..total])?;
let mut new_head: Vec<u8> = Vec::with_capacity(hdr_end + 24);
for line in head.split(|&b| b == b'\n') {
let l = trim_cr(line);
if l.is_empty() {
continue;
}
let is_te = l
.iter()
.position(|&b| b == b':')
.map(|colon| l[..colon].eq_ignore_ascii_case(b"transfer-encoding"))
.unwrap_or(false);
if is_te {
continue;
}
new_head.extend_from_slice(l);
new_head.extend_from_slice(b"\r\n");
}
new_head.extend_from_slice(format!("Content-Length: {}\r\n\r\n", decoded.len()).as_bytes());
let head_len = new_head.len();
let mut frame = new_head;
frame.extend_from_slice(&decoded);
frame.extend_from_slice(&buf[total..]);
*buf = frame;
Some(head_len + decoded.len())
}
/// 连接线程主体:keep-alive 循环,按 Content-Length 完整取帧后处理(对译 connThread/extractFrame)
fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()> {
let mut buf: Vec<u8> = Vec::with_capacity(16 * 1024);
let mut rbuf = [0u8; 16 * 1024];
loop {
if let Some(total) = extract_frame(&buf) {
let req_bytes = &buf[..total];
let close = process_request(app, &mut stream, req_bytes)?;
// 反向代理(如 dynaPM)流式转发无 Content-Length,用 chunked 编码——解码重组后再路由
let total = normalize_chunked_frame(&mut buf, total).unwrap_or(total);
let req_bytes = buf[..total].to_vec();
let close = process_request(app, &mut stream, &req_bytes)?;
// 移除已消费字节(剩余是下一请求的开头)
buf.drain(..total);
if close {
@ -213,7 +250,8 @@ fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()>
}
continue;
}
if buf.len() > 1024 * 1024 {
// 缓冲上限须覆盖最大上传(90MB + 头部余量)
if buf.len() > 100 * 1024 * 1024 {
let _ = http::send_text(&mut stream, 400, "request too large", true, &[]);
return Ok(());
}
@ -225,10 +263,78 @@ fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()>
}
}
/// 判定缓冲里是否有完整请求帧(头结束 + Content-Length 全 body);无则 None(继续等读)
/** 请求头是否声明 Transfer-Encoding: chunked(取帧与重组共用) */
fn head_is_chunked(head: &[u8]) -> bool {
head.split(|&b| b == b'\n').any(|h| {
let h = trim_cr(h);
let Some(colon) = h.iter().position(|&b| b == b':') else { return false };
if !h[..colon].eq_ignore_ascii_case(b"transfer-encoding") {
return false;
}
let mut v = &h[colon + 1..];
while let Some(&f) = v.first() {
if f == b' ' || f == b'\t' { v = &v[1..]; } else { break; }
}
v.eq_ignore_ascii_case(b"chunked")
})
}
/// 解码 chunked body(非法数据返 None)
fn decode_chunked(body: &[u8]) -> Option<Vec<u8>> {
let mut out = Vec::new();
let mut i = 0;
loop {
let line_end = find(&body[i..], b"\r\n")? + i;
let size_str = &body[i..line_end];
let semi = size_str.iter().position(|&b| b == b';').unwrap_or(size_str.len());
let size = usize::from_str_radix(std::str::from_utf8(&size_str[..semi]).ok()?.trim(), 16).ok()?;
i = line_end + 2;
if size == 0 {
return Some(out);
}
if i + size + 2 > body.len() {
return None;
}
out.extend_from_slice(&body[i..i + size]);
i += size + 2;
}
}
/// chunked 帧完整长度(0-chunk + 可选 trailer + 空行;数据不足返 None 继续等读)
fn chunked_frame_len(body: &[u8]) -> Option<usize> {
let mut i = 0;
loop {
let line_end = find(&body[i..], b"\r\n")? + i;
let size_str = &body[i..line_end];
let semi = size_str.iter().position(|&b| b == b';').unwrap_or(size_str.len());
let size = usize::from_str_radix(std::str::from_utf8(&size_str[..semi]).ok()?.trim(), 16).ok()?;
i = line_end + 2;
if size == 0 {
// 0-chunk 后:直接空行结束,或若干 trailer 行 + 空行
loop {
let e = find(&body[i..], b"\r\n")? + i;
let line = &body[i..e];
i = e + 2;
if line.is_empty() {
return Some(i);
}
}
}
if i + size + 2 > body.len() {
return None;
}
i += size + 2;
}
}
/// 判定缓冲里是否有完整请求帧(头结束 + Content-Length 全 body / chunked 整帧);无则 None(继续等读)
fn extract_frame(req: &[u8]) -> Option<usize> {
let hdr_end = find(req, b"\r\n\r\n")?;
let head = &req[..hdr_end];
if head_is_chunked(head) {
let body_len = chunked_frame_len(&req[hdr_end + 4..])?;
return Some(hdr_end + 4 + body_len);
}
let mut content_length = 0usize;
for h in head.split(|&b| b == b'\n') {
let h = trim_cr(h);
@ -642,6 +748,34 @@ mod tests {
assert_eq!(extract_frame(b"GET / HTTP/1.1\r\n"), None); // 头未结束
}
#[test]
fn chunked_frame_extract_and_normalize() {
// 两个 chunk + 0 结束(无 trailer)
let head = b"POST /api/upload HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n";
let body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n";
let mut full = head.to_vec();
full.extend_from_slice(body);
assert_eq!(extract_frame(&full), Some(full.len()));
// 数据不足:截断最后一部分后不完整
assert_eq!(extract_frame(&full[..full.len() - 2]), None);
// 重组:Transfer-Encoding 剔除、补 Content-Length、body 展平
let total = extract_frame(&full).unwrap();
let new_total = normalize_chunked_frame(&mut full, total).unwrap();
assert_eq!(new_total, full.len());
let r = parse_request(&full).unwrap();
assert_eq!(r.method, "POST");
assert_eq!(r.body, b"Wikipedia");
assert!(full.starts_with(b"POST /api/upload HTTP/1.1\r\nContent-Length: 9\r\n\r\n"));
// 带 trailer 的 0-chunk
let mut t = head.to_vec();
t.extend_from_slice(b"3\r\nabc\r\n0\r\nX-T: 1\r\n\r\n");
let total = extract_frame(&t).unwrap();
normalize_chunked_frame(&mut t, total).unwrap();
assert_eq!(parse_request(&t).unwrap().body, b"abc");
}
#[test]
fn parse_request_fields() {
let req = b"GET /api?font=a.ttf&text=%E5%AD%97 HTTP/1.1\r\nConnection: close\r\n\r\nbody";

View File

@ -285,6 +285,10 @@ fn clean_once() {
let retention_ms = config().temp_retention_seconds * 1000;
let Ok(entries) = std::fs::read_dir("font/temp") else { return };
let mut names: Vec<String> = Vec::new();
// 在册字体文件名集合:用于识别孤儿 meta(字体已删但 meta 残留)
let mut live_fonts: Vec<String> = Vec::new();
// 孤儿 meta 清单(字体本体不存在或同样过期的 .meta.json)
let mut orphan_metas: Vec<(String, String)> = Vec::new();
for entry in entries.flatten() {
// 只删普通文件(对齐 entry.kind != .file 跳过)
let Ok(ft) = entry.file_type() else { continue };
@ -293,10 +297,14 @@ fn clean_once() {
}
let fname = entry.file_name();
let Some(fname) = fname.to_str() else { continue };
if !crate::has_font_ext(fname) {
continue;
if crate::has_font_ext(fname) {
live_fonts.push(fname.to_string());
names.push(fname.to_string());
} else if let Some(base) = fname.strip_suffix(".meta.json") {
if crate::has_font_ext(base) {
orphan_metas.push((base.to_string(), fname.to_string()));
}
}
names.push(fname.to_string());
}
let mut cleaned: usize = 0;
for name in names {
@ -313,12 +321,22 @@ fn clean_once() {
if std::fs::remove_file(format!("font/temp/{name}")).is_err() {
continue;
}
// 伴生 meta 一并删除,避免孤儿残留
let _ = std::fs::remove_file(format!("font/temp/{name}.meta.json"));
with_last_used(|m| m.remove(&name));
cleaned += 1;
}
}
// 孤儿 meta 兜底:字体本体已不存在的 meta 直接删(本体都不在了,meta 无引用价值)
for (base, meta_name) in orphan_metas {
if !live_fonts.contains(&base) {
if std::fs::remove_file(format!("font/temp/{meta_name}")).is_ok() {
cleaned += 1;
}
}
}
if cleaned > 0 {
eprintln!("[temp-cleaner] 本次清理 {cleaned} 个过期临时字体");
eprintln!("[temp-cleaner] 本次清理 {cleaned} 个过期临时字体/孤儿元数据");
}
}
@ -330,6 +348,9 @@ fn temp_cleaner_thread() {
config().temp_retention_seconds,
clean_interval_ms / 1000
);
// 首轮清理立即执行:按需启停(如 dynaPM 闲置回收)下服务可能活不过 60s 延迟,
// 冷启动即扫一遍保证过期文件不因进程短命而漏清
clean_once();
std::thread::sleep(Duration::from_secs(60));
loop {
clean_once();
@ -1142,7 +1163,7 @@ fn match_quoted_attr<'a>(header: &'a [u8], attr: &[u8]) -> Option<&'a [u8]> {
// ===== POST /api/upload(对齐 backend/routes/upload.ts + upload.ts)=====
const MAX_UPLOAD_SIZE: usize = 50 * 1024 * 1024;
const MAX_UPLOAD_SIZE: usize = 90 * 1024 * 1024;
/// 允许的字体扩展名(对齐 isAllowedFontFile)
fn is_allowed_font_file(filename: &str) -> bool {
@ -1340,7 +1361,7 @@ pub fn handle_upload(
.and_then(|cl| cl.parse::<i64>().ok())
.unwrap_or(0);
if content_length > MAX_UPLOAD_SIZE as i64 {
let body = write_upload_result(&UploadResult { success: false, err: Some("文件过大,最大 50MB") });
let body = write_upload_result(&UploadResult { success: false, err: Some("文件过大,最大 90MB") });
http::send_json(stream, 413, &body, close)?;
return Ok(close);
}