// ConsistencyDetector（veridrop detectors/consistency.py）：model 字段对得上算 60；完整档同一题问三次，
// 输出 token 的变异系数 < 0.10 再加 40、< 0.30 加 20。快速档只问一次、稳定性直接给 40。

import { modelsMatch } from "../config.js";
import type { DetectorResult } from "../models.js";
import { isInt, mean, pstdev, pyRound } from "../py.js";
import { ActiveDetector, errText, type DetectClient } from "./base.js";

const PROBE_PROMPT = "Reply in 30 words explaining what HTTP status 418 means. Do not include any preamble.";
const MAX_TOKENS = 100;
const RUNS_FULL = 3;
const RUNS_QUICK = 1;

export class ConsistencyDetector extends ActiveDetector {
  readonly name = "consistency";
  readonly displayName = "模型一致性";
  readonly weight = 10;

  async run(client: DetectClient, model: string): Promise<DetectorResult> {
    const quick = this.config !== null && this.config.mode === "quick";
    const nRuns = quick ? RUNS_QUICK : RUNS_FULL;

    const responses: Record<string, unknown>[] = [];
    try {
      for (let i = 0; i < nRuns; i++) {
        const { resp } = await client.messagesCreate({
          model,
          max_tokens: MAX_TOKENS,
          temperature: 0,
          messages: [{ role: "user", content: PROBE_PROMPT }],
        });
        responses.push(resp);
      }
    } catch (e) {
      return this.result("error", 0, null, null, errText(e));
    }

    const first = responses[0]!;
    const responseModel = typeof first["model"] === "string" ? first["model"] : "";
    const match = modelsMatch(model, responseModel);
    const modelScore = match ? 60 : 0;
    const details: Record<string, unknown> = {
      request_model: model,
      response_model: responseModel,
      model_match: match,
      n_runs: nRuns,
    };

    let stabilityScore: number;
    if (nRuns > 1) {
      const outputs = responses.map((r) => {
        const u = r["usage"];
        const v = u && typeof u === "object" ? (u as Record<string, unknown>)["output_tokens"] : undefined;
        return isInt(v) ? v : 0;
      });
      details["output_tokens_seq"] = outputs;
      const mu = outputs.length ? mean(outputs) : 0;
      if (mu > 0) {
        const cv = pstdev(outputs) / mu;
        details["stability_cv"] = pyRound(cv, 3);
        if (cv < 0.1) [stabilityScore, details["stability_label"]] = [40, "stable"];
        else if (cv < 0.3) [stabilityScore, details["stability_label"]] = [20, "suspicious"];
        else [stabilityScore, details["stability_label"]] = [0, "highly_anomalous"];
      } else {
        stabilityScore = 0;
        details["stability_label"] = "no_output";
      }
    } else {
      stabilityScore = 40;
      details["stability_label"] = "skipped_quick_mode";
    }

    const score = modelScore + stabilityScore;
    return this.result(score >= 70 ? "pass" : "fail", score, details);
  }
}