mirror of
https://github.com/2234839/web-font.git
synced 2026-11-21 06:24:05 +08:00
修复上传链路与临时字体清理(dynaPM 架构适配)
- webfont-rs 支持 chunked 请求体:dynaPM 网关流式转发剥 Content-Length 后用 Transfer-Encoding: chunked,原 extract_frame 只认 Content-Length 导致 POST body 恒空(上传接口 400 请求体为空)。取帧支持 chunked + 解码重组为等价 Content-Length 帧,含单测 - 上传限制 50MB → 90MB(TS/rust 双端),连接缓冲 1MB → 100MB 适配大上传 - temp-cleaner 双坑修复(TS/rust 双端):删字体时同步删伴生 .meta.json + 孤儿 meta 兜底扫描(此前孤儿元数据无限堆积);首轮清理由延迟 60s 改为 启动立即执行(dynaPM 闲置 30s 停服,延迟首轮导致清理器几乎从未运行)
This commit is contained in:
parent
c630d5b5c9
commit
0cf78d3d61
@ -190,8 +190,8 @@ const fontApiMiddleware: cMiddleware = async (req, res, next) => {
|
||||
return next(req, res);
|
||||
};
|
||||
|
||||
/** 上传文件大小限制 50MB */
|
||||
const MAX_UPLOAD_SIZE = 50 * 1024 * 1024;
|
||||
/** 上传文件大小限制 90MB */
|
||||
const MAX_UPLOAD_SIZE = 90 * 1024 * 1024;
|
||||
|
||||
const uploadSizeMiddleware: cMiddleware = async (req, res, next) => {
|
||||
if (req.method === "POST" && parseUrl(req).pathname === "/api/upload") {
|
||||
@ -199,7 +199,7 @@ const uploadSizeMiddleware: cMiddleware = async (req, res, next) => {
|
||||
if (contentLength > MAX_UPLOAD_SIZE) {
|
||||
return {
|
||||
req,
|
||||
res: jsonResponse({ success: false, error: "文件过大,最大 50MB" }, 413),
|
||||
res: jsonResponse({ success: false, error: "文件过大,最大 90MB" }, 413),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,28 +43,46 @@ async function cleanOnce(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
let cleaned = 0;
|
||||
/** 在册字体文件名集合:用于识别孤儿 meta(字体已删但 meta 残留) */
|
||||
const liveFonts = new Set<string>();
|
||||
/** 孤儿 meta 清单(.meta.json 后缀且主文件是字体扩展名) */
|
||||
const orphanMetas: Array<{ base: string; name: string }> = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (!/\.(ttf|otf|woff|woff2)$/i.test(entry.name)) continue;
|
||||
if (/\.(ttf|otf|woff|woff2)$/i.test(entry.name)) {
|
||||
liveFonts.add(entry.name);
|
||||
} else if (/\.(ttf|otf|woff|woff2)\.meta\.json$/i.test(entry.name)) {
|
||||
orphanMetas.push({ base: entry.name.replace(/\.meta\.json$/i, ""), name: entry.name });
|
||||
}
|
||||
}
|
||||
|
||||
const filePath = path_join(TEMP_DIR, entry.name);
|
||||
let cleaned = 0;
|
||||
for (const name of liveFonts) {
|
||||
const filePath = path_join(TEMP_DIR, name);
|
||||
try {
|
||||
const s = await stat(filePath);
|
||||
/** 取"最后使用时间"和"文件修改时间"的较大值作为活跃判定基准 */
|
||||
const lastUsed = lastUsedMap.get(entry.name) ?? 0;
|
||||
const lastUsed = lastUsedMap.get(name) ?? 0;
|
||||
const lastActive = Math.max(lastUsed, s.mtimeMs);
|
||||
if (now - lastActive > retentionMs) {
|
||||
await unlink(filePath);
|
||||
lastUsedMap.delete(entry.name);
|
||||
/** 伴生 meta 一并删除,避免孤儿残留 */
|
||||
await unlink(path_join(TEMP_DIR, `${name}.meta.json`)).catch(() => {});
|
||||
lastUsedMap.delete(name);
|
||||
cleaned++;
|
||||
}
|
||||
} catch (err) {
|
||||
console.log("[temp-cleaner] 处理文件失败:", entry.name, err);
|
||||
console.log("[temp-cleaner] 处理文件失败:", name, err);
|
||||
}
|
||||
}
|
||||
/** 孤儿 meta 兜底:字体本体已不存在的 meta 直接删(本体不在,meta 无引用价值) */
|
||||
for (const { base, name } of orphanMetas) {
|
||||
if (liveFonts.has(base)) continue;
|
||||
await unlink(path_join(TEMP_DIR, name)).catch(() => {});
|
||||
cleaned++;
|
||||
}
|
||||
if (cleaned > 0) {
|
||||
console.log(`[temp-cleaner] 本次清理 ${cleaned} 个过期临时字体`);
|
||||
console.log(`[temp-cleaner] 本次清理 ${cleaned} 个过期临时字体/孤儿元数据`);
|
||||
}
|
||||
}
|
||||
|
||||
@ -73,18 +91,18 @@ const CLEAN_INTERVAL = Math.max(tempRetentionSeconds * 500, 300_000);
|
||||
|
||||
/**
|
||||
* 启动定时清理器。
|
||||
* 首次延迟 1 分钟执行(避免启动峰),之后按周期循环。
|
||||
* 首轮立即执行(按需启停下服务可能活不过 60s 延迟,冷启动即扫保证不漏清),之后按周期循环。
|
||||
*/
|
||||
export function startTempCleaner(): void {
|
||||
const intervalSec = Math.round(CLEAN_INTERVAL / 1000);
|
||||
console.log(`[temp-cleaner] 启动,保留时限 ${tempRetentionSeconds}s,清理周期 ${intervalSec}s`);
|
||||
|
||||
/** 首次延迟 60 秒 */
|
||||
/** 首轮立即执行 */
|
||||
cleanOnce().catch(() => {});
|
||||
/** 首个周期点起按周期循环 */
|
||||
setTimeout(() => {
|
||||
cleanOnce().catch(() => {});
|
||||
/** 后续按周期循环 */
|
||||
setInterval(() => {
|
||||
cleanOnce().catch(() => {});
|
||||
}, CLEAN_INTERVAL);
|
||||
}, 60_000);
|
||||
}, CLEAN_INTERVAL);
|
||||
}
|
||||
|
||||
@ -198,14 +198,51 @@ struct Request<'a> {
|
||||
if_none_match: Option<String>,
|
||||
}
|
||||
|
||||
/// chunked 帧解码重组:剔除 Transfer-Encoding 头、补 Content-Length、body 展平为连续字节,
|
||||
/// 返回重组后的帧总长(非 chunked 帧原样返回 None)
|
||||
fn normalize_chunked_frame(buf: &mut Vec<u8>, total: usize) -> Option<usize> {
|
||||
let hdr_end = find(&buf[..total], b"\r\n\r\n")?;
|
||||
let head = buf[..hdr_end].to_vec();
|
||||
if !head_is_chunked(&head) {
|
||||
return None;
|
||||
}
|
||||
let decoded = decode_chunked(&buf[hdr_end + 4..total])?;
|
||||
let mut new_head: Vec<u8> = Vec::with_capacity(hdr_end + 24);
|
||||
for line in head.split(|&b| b == b'\n') {
|
||||
let l = trim_cr(line);
|
||||
if l.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let is_te = l
|
||||
.iter()
|
||||
.position(|&b| b == b':')
|
||||
.map(|colon| l[..colon].eq_ignore_ascii_case(b"transfer-encoding"))
|
||||
.unwrap_or(false);
|
||||
if is_te {
|
||||
continue;
|
||||
}
|
||||
new_head.extend_from_slice(l);
|
||||
new_head.extend_from_slice(b"\r\n");
|
||||
}
|
||||
new_head.extend_from_slice(format!("Content-Length: {}\r\n\r\n", decoded.len()).as_bytes());
|
||||
let head_len = new_head.len();
|
||||
let mut frame = new_head;
|
||||
frame.extend_from_slice(&decoded);
|
||||
frame.extend_from_slice(&buf[total..]);
|
||||
*buf = frame;
|
||||
Some(head_len + decoded.len())
|
||||
}
|
||||
|
||||
/// 连接线程主体:keep-alive 循环,按 Content-Length 完整取帧后处理(对译 connThread/extractFrame)
|
||||
fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()> {
|
||||
let mut buf: Vec<u8> = Vec::with_capacity(16 * 1024);
|
||||
let mut rbuf = [0u8; 16 * 1024];
|
||||
loop {
|
||||
if let Some(total) = extract_frame(&buf) {
|
||||
let req_bytes = &buf[..total];
|
||||
let close = process_request(app, &mut stream, req_bytes)?;
|
||||
// 反向代理(如 dynaPM)流式转发无 Content-Length,用 chunked 编码——解码重组后再路由
|
||||
let total = normalize_chunked_frame(&mut buf, total).unwrap_or(total);
|
||||
let req_bytes = buf[..total].to_vec();
|
||||
let close = process_request(app, &mut stream, &req_bytes)?;
|
||||
// 移除已消费字节(剩余是下一请求的开头)
|
||||
buf.drain(..total);
|
||||
if close {
|
||||
@ -213,7 +250,8 @@ fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()>
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if buf.len() > 1024 * 1024 {
|
||||
// 缓冲上限须覆盖最大上传(90MB + 头部余量)
|
||||
if buf.len() > 100 * 1024 * 1024 {
|
||||
let _ = http::send_text(&mut stream, 400, "request too large", true, &[]);
|
||||
return Ok(());
|
||||
}
|
||||
@ -225,10 +263,78 @@ fn conn_loop(app: &App, mut stream: std::net::TcpStream) -> std::io::Result<()>
|
||||
}
|
||||
}
|
||||
|
||||
/// 判定缓冲里是否有完整请求帧(头结束 + Content-Length 全 body);无则 None(继续等读)
|
||||
/** 请求头是否声明 Transfer-Encoding: chunked(取帧与重组共用) */
|
||||
fn head_is_chunked(head: &[u8]) -> bool {
|
||||
head.split(|&b| b == b'\n').any(|h| {
|
||||
let h = trim_cr(h);
|
||||
let Some(colon) = h.iter().position(|&b| b == b':') else { return false };
|
||||
if !h[..colon].eq_ignore_ascii_case(b"transfer-encoding") {
|
||||
return false;
|
||||
}
|
||||
let mut v = &h[colon + 1..];
|
||||
while let Some(&f) = v.first() {
|
||||
if f == b' ' || f == b'\t' { v = &v[1..]; } else { break; }
|
||||
}
|
||||
v.eq_ignore_ascii_case(b"chunked")
|
||||
})
|
||||
}
|
||||
|
||||
/// 解码 chunked body(非法数据返 None)
|
||||
fn decode_chunked(body: &[u8]) -> Option<Vec<u8>> {
|
||||
let mut out = Vec::new();
|
||||
let mut i = 0;
|
||||
loop {
|
||||
let line_end = find(&body[i..], b"\r\n")? + i;
|
||||
let size_str = &body[i..line_end];
|
||||
let semi = size_str.iter().position(|&b| b == b';').unwrap_or(size_str.len());
|
||||
let size = usize::from_str_radix(std::str::from_utf8(&size_str[..semi]).ok()?.trim(), 16).ok()?;
|
||||
i = line_end + 2;
|
||||
if size == 0 {
|
||||
return Some(out);
|
||||
}
|
||||
if i + size + 2 > body.len() {
|
||||
return None;
|
||||
}
|
||||
out.extend_from_slice(&body[i..i + size]);
|
||||
i += size + 2;
|
||||
}
|
||||
}
|
||||
|
||||
/// chunked 帧完整长度(0-chunk + 可选 trailer + 空行;数据不足返 None 继续等读)
|
||||
fn chunked_frame_len(body: &[u8]) -> Option<usize> {
|
||||
let mut i = 0;
|
||||
loop {
|
||||
let line_end = find(&body[i..], b"\r\n")? + i;
|
||||
let size_str = &body[i..line_end];
|
||||
let semi = size_str.iter().position(|&b| b == b';').unwrap_or(size_str.len());
|
||||
let size = usize::from_str_radix(std::str::from_utf8(&size_str[..semi]).ok()?.trim(), 16).ok()?;
|
||||
i = line_end + 2;
|
||||
if size == 0 {
|
||||
// 0-chunk 后:直接空行结束,或若干 trailer 行 + 空行
|
||||
loop {
|
||||
let e = find(&body[i..], b"\r\n")? + i;
|
||||
let line = &body[i..e];
|
||||
i = e + 2;
|
||||
if line.is_empty() {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
if i + size + 2 > body.len() {
|
||||
return None;
|
||||
}
|
||||
i += size + 2;
|
||||
}
|
||||
}
|
||||
|
||||
/// 判定缓冲里是否有完整请求帧(头结束 + Content-Length 全 body / chunked 整帧);无则 None(继续等读)
|
||||
fn extract_frame(req: &[u8]) -> Option<usize> {
|
||||
let hdr_end = find(req, b"\r\n\r\n")?;
|
||||
let head = &req[..hdr_end];
|
||||
if head_is_chunked(head) {
|
||||
let body_len = chunked_frame_len(&req[hdr_end + 4..])?;
|
||||
return Some(hdr_end + 4 + body_len);
|
||||
}
|
||||
let mut content_length = 0usize;
|
||||
for h in head.split(|&b| b == b'\n') {
|
||||
let h = trim_cr(h);
|
||||
@ -642,6 +748,34 @@ mod tests {
|
||||
assert_eq!(extract_frame(b"GET / HTTP/1.1\r\n"), None); // 头未结束
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunked_frame_extract_and_normalize() {
|
||||
// 两个 chunk + 0 结束(无 trailer)
|
||||
let head = b"POST /api/upload HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n";
|
||||
let body = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n";
|
||||
let mut full = head.to_vec();
|
||||
full.extend_from_slice(body);
|
||||
assert_eq!(extract_frame(&full), Some(full.len()));
|
||||
// 数据不足:截断最后一部分后不完整
|
||||
assert_eq!(extract_frame(&full[..full.len() - 2]), None);
|
||||
|
||||
// 重组:Transfer-Encoding 剔除、补 Content-Length、body 展平
|
||||
let total = extract_frame(&full).unwrap();
|
||||
let new_total = normalize_chunked_frame(&mut full, total).unwrap();
|
||||
assert_eq!(new_total, full.len());
|
||||
let r = parse_request(&full).unwrap();
|
||||
assert_eq!(r.method, "POST");
|
||||
assert_eq!(r.body, b"Wikipedia");
|
||||
assert!(full.starts_with(b"POST /api/upload HTTP/1.1\r\nContent-Length: 9\r\n\r\n"));
|
||||
|
||||
// 带 trailer 的 0-chunk
|
||||
let mut t = head.to_vec();
|
||||
t.extend_from_slice(b"3\r\nabc\r\n0\r\nX-T: 1\r\n\r\n");
|
||||
let total = extract_frame(&t).unwrap();
|
||||
normalize_chunked_frame(&mut t, total).unwrap();
|
||||
assert_eq!(parse_request(&t).unwrap().body, b"abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_request_fields() {
|
||||
let req = b"GET /api?font=a.ttf&text=%E5%AD%97 HTTP/1.1\r\nConnection: close\r\n\r\nbody";
|
||||
|
||||
@ -285,6 +285,10 @@ fn clean_once() {
|
||||
let retention_ms = config().temp_retention_seconds * 1000;
|
||||
let Ok(entries) = std::fs::read_dir("font/temp") else { return };
|
||||
let mut names: Vec<String> = Vec::new();
|
||||
// 在册字体文件名集合:用于识别孤儿 meta(字体已删但 meta 残留)
|
||||
let mut live_fonts: Vec<String> = Vec::new();
|
||||
// 孤儿 meta 清单(字体本体不存在或同样过期的 .meta.json)
|
||||
let mut orphan_metas: Vec<(String, String)> = Vec::new();
|
||||
for entry in entries.flatten() {
|
||||
// 只删普通文件(对齐 entry.kind != .file 跳过)
|
||||
let Ok(ft) = entry.file_type() else { continue };
|
||||
@ -293,10 +297,14 @@ fn clean_once() {
|
||||
}
|
||||
let fname = entry.file_name();
|
||||
let Some(fname) = fname.to_str() else { continue };
|
||||
if !crate::has_font_ext(fname) {
|
||||
continue;
|
||||
if crate::has_font_ext(fname) {
|
||||
live_fonts.push(fname.to_string());
|
||||
names.push(fname.to_string());
|
||||
} else if let Some(base) = fname.strip_suffix(".meta.json") {
|
||||
if crate::has_font_ext(base) {
|
||||
orphan_metas.push((base.to_string(), fname.to_string()));
|
||||
}
|
||||
}
|
||||
names.push(fname.to_string());
|
||||
}
|
||||
let mut cleaned: usize = 0;
|
||||
for name in names {
|
||||
@ -313,12 +321,22 @@ fn clean_once() {
|
||||
if std::fs::remove_file(format!("font/temp/{name}")).is_err() {
|
||||
continue;
|
||||
}
|
||||
// 伴生 meta 一并删除,避免孤儿残留
|
||||
let _ = std::fs::remove_file(format!("font/temp/{name}.meta.json"));
|
||||
with_last_used(|m| m.remove(&name));
|
||||
cleaned += 1;
|
||||
}
|
||||
}
|
||||
// 孤儿 meta 兜底:字体本体已不存在的 meta 直接删(本体都不在了,meta 无引用价值)
|
||||
for (base, meta_name) in orphan_metas {
|
||||
if !live_fonts.contains(&base) {
|
||||
if std::fs::remove_file(format!("font/temp/{meta_name}")).is_ok() {
|
||||
cleaned += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if cleaned > 0 {
|
||||
eprintln!("[temp-cleaner] 本次清理 {cleaned} 个过期临时字体");
|
||||
eprintln!("[temp-cleaner] 本次清理 {cleaned} 个过期临时字体/孤儿元数据");
|
||||
}
|
||||
}
|
||||
|
||||
@ -330,6 +348,9 @@ fn temp_cleaner_thread() {
|
||||
config().temp_retention_seconds,
|
||||
clean_interval_ms / 1000
|
||||
);
|
||||
// 首轮清理立即执行:按需启停(如 dynaPM 闲置回收)下服务可能活不过 60s 延迟,
|
||||
// 冷启动即扫一遍保证过期文件不因进程短命而漏清
|
||||
clean_once();
|
||||
std::thread::sleep(Duration::from_secs(60));
|
||||
loop {
|
||||
clean_once();
|
||||
@ -1142,7 +1163,7 @@ fn match_quoted_attr<'a>(header: &'a [u8], attr: &[u8]) -> Option<&'a [u8]> {
|
||||
|
||||
// ===== POST /api/upload(对齐 backend/routes/upload.ts + upload.ts)=====
|
||||
|
||||
const MAX_UPLOAD_SIZE: usize = 50 * 1024 * 1024;
|
||||
const MAX_UPLOAD_SIZE: usize = 90 * 1024 * 1024;
|
||||
|
||||
/// 允许的字体扩展名(对齐 isAllowedFontFile)
|
||||
fn is_allowed_font_file(filename: &str) -> bool {
|
||||
@ -1340,7 +1361,7 @@ pub fn handle_upload(
|
||||
.and_then(|cl| cl.parse::<i64>().ok())
|
||||
.unwrap_or(0);
|
||||
if content_length > MAX_UPLOAD_SIZE as i64 {
|
||||
let body = write_upload_result(&UploadResult { success: false, err: Some("文件过大,最大 50MB") });
|
||||
let body = write_upload_result(&UploadResult { success: false, err: Some("文件过大,最大 90MB") });
|
||||
http::send_json(stream, 413, &body, close)?;
|
||||
return Ok(close);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user