web-font/backend/subset_queue.ts
崮生(子虚) 5d6d71c447 feat: 找字体改为流式接口 + 内存对象评分,全库搜索内存降至 ~270MB
- /api/glyph-match 改为 NDJSON 流式:start/font/glyph/fontDone/done 事件,
  前端逐字体渲染预览与分数,消除整包等待
- 评分直接消费裁剪管线内存 ttf 对象(按字形 unicode 重建子集 cmap),
  免去序列化再解析;源界明朝/装甲明朝等 re-parse 失败字体首次可评分
- glyph_bitmap 对快路径字形(_origBuf 原始字节引用)按需调用 vendor
  parseGlyf 解码坐标,修复内存对象方案下「借」字镂空回归
- 所有裁剪调用点接入 withMemoryGate 内存闸门(cgroup 探测并发控制)
- HTTP 服务支持流式响应(LLRT 下经 _stream 附加属性传递),
  客户端断连 EPIPE/Controller closed 全链路容错,不再打死进程
- 单字体处理失败 try-catch 隔离,不中断整个搜索流
- app.ts 端口支持 PORT 环境变量覆盖,便于并行测试实例
2026-11-01 19:47:26 +08:00

122 lines
4.2 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { freemem, totalmem } from "os";
import { readFile } from "./interface";
/** 内存快照:同时考虑宿主机空闲内存和容器 cgroup 剩余额度 */
interface MemorySnapshot {
limit: number;
free: number;
reserve: number;
}
const MIB = 1024 * 1024;
const decoder = new TextDecoder();
let activeCount = 0;
let waitingCount = 0;
let reservedBytes = 0;
let maxConcurrency = 4;
/** 读取一个内核数值文件;memory.max 在未限制时为 max,会返回 NaN 并被忽略 */
async function readMemoryNumber(path: string): Promise<number> {
return Number(decoder.decode(await readFile(path)).trim());
}
/** 优先读取 cgroup v2,再兼容 v1;非容器环境没有这些文件 */
async function readCgroupMemory(): Promise<{ limit: number; used: number } | null> {
const candidates = [
["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.current"],
["/sys/fs/cgroup/memory/memory.limit_in_bytes", "/sys/fs/cgroup/memory/memory.usage_in_bytes"],
] as const;
for (const [limitPath, usedPath] of candidates) {
try {
const [limit, used] = await Promise.all([readMemoryNumber(limitPath), readMemoryNumber(usedPath)]);
if (Number.isFinite(limit) && limit > 0 && Number.isFinite(used) && used >= 0) {
return { limit, used };
}
} catch {
/** 当前运行环境没有这一版 cgroup 接口,继续探测下一版 */
}
}
return null;
}
/** 每次准入都读取当前值,不缓存内存水位 */
async function readMemorySnapshot(): Promise<MemorySnapshot> {
const hostLimit = totalmem();
const hostFree = freemem();
const cgroup = await readCgroupMemory();
const limit = cgroup ? Math.min(hostLimit, cgroup.limit) : hostLimit;
const cgroupFree = cgroup ? Math.max(0, cgroup.limit - cgroup.used) : hostLimit;
const free = Math.max(0, Math.min(hostFree, cgroupFree));
/** 给运行时和静态请求保留内存,绝不把水位用满 */
const reserve = Math.max(32 * MIB, Math.min(128 * MIB, limit * 0.1));
return { limit, free, reserve };
}
/** 小内存机器最多同时执行两个重任务;大内存机器仍受配置上限约束 */
function concurrencyLimit(memory: MemorySnapshot): number {
return memory.limit <= 2 * 1024 * MIB ? Math.min(maxConcurrency, 2) : maxConcurrency;
}
function delay(milliseconds: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, milliseconds));
}
/**
* 所有字体重任务共用的内存闸门。
*
* 在读取完整字体之前按源文件预估内存;并发槽位和实际可用内存任一不足都排队。
* 没有任务运行时允许启动一个大任务,避免小内存机器因预估值偏大而永久饥饿。
* 任务超时返回 null,由调用方按现有接口语义返回 503 或流式错误。
*/
export async function withMemoryGate<T>(
task: () => Promise<T>,
queueTimeoutMs: number,
estimatedBytes = 64 * MIB,
): Promise<T | null> {
const deadline = Date.now() + queueTimeoutMs;
let queued = false;
while (true) {
const memory = await readMemorySnapshot();
const hasSlot = activeCount < concurrencyLimit(memory);
const available = memory.free - memory.reserve - reservedBytes;
const hasMemory = activeCount === 0 ? available > 0 : available >= estimatedBytes;
if (hasSlot && hasMemory) {
if (queued) waitingCount--;
activeCount++;
reservedBytes += estimatedBytes;
try {
return await task();
} finally {
activeCount--;
reservedBytes -= estimatedBytes;
}
}
if (!queued) {
queued = true;
waitingCount++;
}
const remaining = deadline - Date.now();
if (remaining <= 0) {
waitingCount--;
return null;
}
await delay(Math.min(50, remaining));
}
}
/** 初始化最大并发数;实际并发还会随每次任务的内存水位进一步降低 */
export function initMemoryGate(concurrency?: number): void {
if (concurrency && concurrency > 0) maxConcurrency = concurrency;
console.log(`[subset-queue] maxConcurrency=${maxConcurrency}, memoryAware=true`);
}
/** 获取当前队列状态(用于 stats / 日志) */
export function getQueueStats() {
return {
active: activeCount,
waiting: waitingCount,
reservedBytes,
maxConcurrency,
};
}