// ProtocolDetector(veridrop detectors/protocol.py):被动,看所有应答是否符合 Messages API 的字段形状。 // 每一种问题只扣一次(10 分),衡量的是合不合规,不是出现了几次。 import type { Body } from "../client.js"; import type { DetectorResult } from "../models.js"; import { isInt, pyRepr } from "../py.js"; import { PassiveDetector } from "./base.js"; const VALID_STOP_REASONS = new Set(["end_turn", "max_tokens", "stop_sequence", "tool_use", null, undefined]); const VALID_CONTENT_BLOCK_TYPES = new Set([ "text", "tool_use", "thinking", "redacted_thinking", "server_tool_use", "web_search_tool_result", ]); const ISSUE_PENALTY = 10; function nonNegInt(v: unknown): boolean { return isInt(v) && v >= 0; } export class ProtocolDetector extends PassiveDetector { readonly name = "protocol"; readonly displayName = "协议规范性"; readonly weight = 5; private readonly issues = new Set(); private observations = 0; override observe(_request: Body, response: Record): void { this.observations++; const rid = response["id"]; if (typeof rid !== "string" || !rid) this.issues.add("id_missing_or_not_string"); if (response["type"] !== "message") this.issues.add(`type_invalid:${pyRepr(response["type"])}`); if (response["role"] !== "assistant") this.issues.add(`role_invalid:${pyRepr(response["role"])}`); const model = response["model"]; if (typeof model !== "string" || !model) this.issues.add("model_missing_or_not_string"); const content = response["content"]; if (!Array.isArray(content)) { this.issues.add("content_not_array"); } else { for (const block of content) { if (!block || typeof block !== "object" || Array.isArray(block)) { this.issues.add("content_block_not_object"); continue; } const t = (block as Record)["type"]; if (typeof t !== "string") this.issues.add("content_block_missing_type"); else if (!VALID_CONTENT_BLOCK_TYPES.has(t)) this.issues.add(`content_block_unknown_type:${pyRepr(t)}`); } } if (!VALID_STOP_REASONS.has(response["stop_reason"])) { this.issues.add(`stop_reason_invalid:${pyRepr(response["stop_reason"])}`); } const ss = response["stop_sequence"]; if (!(ss === null || ss === undefined || typeof ss === "string")) this.issues.add("stop_sequence_wrong_type"); const usage = response["usage"]; if (!usage || typeof usage !== "object" || Array.isArray(usage)) { this.issues.add("usage_missing_or_not_object"); } else { const u = usage as Record; if (!nonNegInt(u["input_tokens"])) this.issues.add("usage_input_tokens_invalid"); if (!nonNegInt(u["output_tokens"])) this.issues.add("usage_output_tokens_invalid"); for (const opt of ["cache_read_input_tokens", "cache_creation_input_tokens"]) { if (opt in u && !nonNegInt(u[opt])) this.issues.add(`${opt}_wrong_type`); } } // 不查 anthropic-request-id 头:原仓库实测 api.anthropic.com 的 /v1/messages 也不回它。 } finalize(): DetectorResult { if (this.observations === 0) return this.skip("no observations"); const score = Math.max(0, 100 - this.issues.size * ISSUE_PENALTY); return this.result(score >= 70 ? "pass" : "fail", score, { observation_count: this.observations, issue_count: this.issues.size, issues: [...this.issues].sort(), }); } }