// IntegrityDetector(veridrop detectors/integrity.py):同一题非流式、流式各问一次,五项各 20 分: // 1 两次文字相似度 ≥ 85(rapidfuzz ratio) 2 字符 / 输出 token 在 1.2–10 之间 // 3 流式 output_tokens 与非流式接近 4 流式 stop_reason 是 end_turn / max_tokens // 5 流式 message_start 的 input_tokens 与非流式接近 import { ratio } from "../fuzz.js"; import type { DetectorResult } from "../models.js"; import { cpLen, cpSlice, isInt, joinText, pyRound } from "../py.js"; import { ActiveDetector, errText, type DetectClient } from "./base.js"; const PROBE_PROMPT = 'Reply with exactly this JSON literal and nothing else: {"verify":"abc123","n":42}'; const PROBE_MAX_TOKENS = 80; const CHARS_PER_TOKEN_MIN = 1.2; const CHARS_PER_TOKEN_MAX = 10.0; const SIMILARITY_PASS = 85.0; const INPUT_TOKEN_TOLERANCE_FRAC = 0.2; const INPUT_TOKEN_TOLERANCE_FLOOR = 3; const SUB_CHECK_WEIGHT = 20.0; /** Python 的 `x or 0`:真值留着,假值变 0。这里只在数上用。 */ function orZero(v: unknown): number { return typeof v === "number" && v ? v : 0; } export class IntegrityDetector extends ActiveDetector { readonly name = "integrity"; readonly displayName = "响应完整性"; readonly weight = 5; async run(client: DetectClient, model: string): Promise { const details: Record = {}; const body = { model, max_tokens: PROBE_MAX_TOKENS, temperature: 0, messages: [{ role: "user", content: PROBE_PROMPT }], }; let nsResp: Record; try { ({ resp: nsResp } = await client.messagesCreate(body)); } catch (e) { return this.result("error", 0, { phase: "non_stream" }, null, errText(e)); } const nsText = joinText(nsResp["content"]); const nsUsage = (nsResp["usage"] && typeof nsResp["usage"] === "object" ? nsResp["usage"] : {}) as Record; const nsInput = orZero(nsUsage["input_tokens"]); const nsOutput = orZero(nsUsage["output_tokens"]); details["ns_text"] = cpSlice(nsText, 200); details["ns_usage"] = { input: nsInput, output: nsOutput }; let streamText = ""; let streamStopReason: unknown = null; let deltaUsage: Record | null = null; let startInput: number | null = null; let startSeen = false; try { for await (const [ev] of client.messagesStream(body)) { if (ev.event === "message_start") { startSeen = true; const msg = (ev.data["message"] ?? {}) as Record; const msu = (msg["usage"] ?? {}) as Record; if (isInt(msu["input_tokens"])) startInput = msu["input_tokens"]; } else if (ev.event === "content_block_delta") { const delta = (ev.data["delta"] ?? {}) as Record; if (delta["type"] === "text_delta") streamText += typeof delta["text"] === "string" ? delta["text"] : ""; } else if (ev.event === "message_delta") { const delta = (ev.data["delta"] ?? {}) as Record; if ("stop_reason" in delta) streamStopReason = delta["stop_reason"]; const u = ev.data["usage"]; if (u && typeof u === "object" && !Array.isArray(u)) deltaUsage = u as Record; } } } catch (e) { return this.result("error", 0, { phase: "stream", ...details }, null, errText(e)); } details["stream_text"] = cpSlice(streamText, 200); details["stream_stop_reason"] = streamStopReason ?? null; details["stream_message_delta_usage"] = deltaUsage; details["stream_message_start_input_tokens"] = startInput; details["stream_message_start_seen"] = startSeen; let score = 0; const sub: Record> = {}; // 1 文字相似度 const sim = nsText && streamText ? ratio(nsText, streamText) : 0; sub["similarity"] = { ratio: pyRound(sim, 1), pass: sim >= SIMILARITY_PASS }; if (sim >= SIMILARITY_PASS) score += SUB_CHECK_WEIGHT; // 2 字符 / token if (nsOutput > 0 && nsText) { const cpt = cpLen(nsText) / nsOutput; const ok = cpt >= CHARS_PER_TOKEN_MIN && cpt <= CHARS_PER_TOKEN_MAX; sub["char_per_token"] = { value: pyRound(cpt, 2), pass: ok }; if (ok) score += SUB_CHECK_WEIGHT; } else { sub["char_per_token"] = { value: null, pass: false }; } // 3 流式 output_tokens 累计且接近非流式 if (deltaUsage && isInt(deltaUsage["output_tokens"])) { const streamOut = deltaUsage["output_tokens"]; const tol = Math.max(2, Math.trunc(nsOutput * 0.5)); const ok = Math.abs(streamOut - nsOutput) <= tol && streamOut > 0; sub["stream_output_tokens"] = { stream: streamOut, ns: nsOutput, pass: ok }; if (ok) score += SUB_CHECK_WEIGHT; } else { sub["stream_output_tokens"] = { stream: null, pass: false }; } // 4 stop_reason { const ok = streamStopReason === "end_turn" || streamStopReason === "max_tokens"; sub["stop_reason"] = { value: streamStopReason ?? null, pass: ok }; if (ok) score += SUB_CHECK_WEIGHT; } // 5 input_tokens:同一题、temperature 0,两次报的输入应当几乎一样;差得多通常是流式时注入了系统提示或者虚报。 if (startInput !== null && nsInput > 0) { const tol = Math.max(INPUT_TOKEN_TOLERANCE_FLOOR, Math.trunc(nsInput * INPUT_TOKEN_TOLERANCE_FRAC)); const diff = Math.abs(startInput - nsInput); const ok = diff <= tol; sub["input_tokens"] = { ns: nsInput, stream: startInput, diff, tolerance: tol, pass: ok }; if (ok) score += SUB_CHECK_WEIGHT; } else { sub["input_tokens"] = { ns: nsInput, stream: startInput, pass: false }; } details["sub_checks"] = sub; return this.result(score >= 70 ? "pass" : "fail", score, details); } }