# -*- coding: utf-8 -*-
"""
dataset_sha256.py — deterministic SHA-256 fingerprint of the dataset.
By default it selects 42 windows with a seeded RNG (seed=42) and hashes their
canonical JSON serialization (sorted keys, compact separators). The selection is
fully deterministic: all window IDs are sorted, then sampled with random.Random(42).
Usage:
python dataset_sha256.py # compute and store the fingerprint
python dataset_sha256.py --verify # recompute and compare with the stored hash
python dataset_sha256.py --all # hash the ENTIRE dataset (all windows)
python dataset_sha256.py --seed 42 --n 42
The manifest is written to dataset/reports/dataset_sha256.json.
"""
import argparse
import hashlib
import json
import os
import random
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import common # noqa: E402
MANIFEST_PATH = os.path.join(common.REPORTS_DIR, "dataset_sha256.json")
def canonical(record):
"""Canonical JSON serialization: sorted keys, compact separators, UTF-8."""
return json.dumps(record, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
def hash_records(records):
"""SHA-256 over the canonical serialization of a list of records."""
payload = "\n".join(canonical(r) for r in records).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def load_all():
"""Returns ({window_id: record}, sorted_window_ids, {split: n_windows})."""
by_wid = {}
counts = {}
for split in common.SPLITS:
cnt = 0
for r in common.iter_split(split):
by_wid[r["window_id"]] = r
cnt += 1
counts[split] = cnt
return by_wid, sorted(by_wid), counts
def main():
ap = argparse.ArgumentParser(description="Deterministic SHA-256 fingerprint of the dataset.")
ap.add_argument("--seed", type=int, default=42, help="RNG seed for the sample")
ap.add_argument("--n", type=int, default=42, help="number of windows to sample")
ap.add_argument("--all", action="store_true",
help="hash every window (full fingerprint, no manifest)")
ap.add_argument("--verify", action="store_true",
help="compare against the stored sha256 and exit")
args = ap.parse_args()
by_wid, wids, counts = load_all()
if args.all:
digest = hash_records([by_wid[w] for w in wids])
print(f"sha256 (all {len(wids)} windows) = {digest}")
return
rng = random.Random(args.seed)
selected = sorted(rng.sample(wids, min(args.n, len(wids))))
digest = hash_records([by_wid[w] for w in selected])
if args.verify:
if not os.path.exists(MANIFEST_PATH):
print("no stored manifest; run without --verify first")
sys.exit(1)
stored = json.load(open(MANIFEST_PATH, encoding="utf-8"))
if stored.get("sha256") == digest:
print("OK: dataset matches stored sha256")
sys.exit(0)
print("MISMATCH: dataset differs from stored sha256")
print(" stored :", stored.get("sha256"))
print(" current:", digest)
sys.exit(1)
manifest = {
"algorithm": "sha256",
"seed": args.seed,
"n_selected": len(selected),
"total_windows": len(wids),
"total_cases": len({r["case_id"] for r in by_wid.values()}),
"windows_by_split": counts,
"selected_window_ids": selected,
"sha256": digest,
}
with open(MANIFEST_PATH, "w", encoding="utf-8") as f:
json.dump(manifest, f, ensure_ascii=False, indent=2)
print(f"seed={args.seed} selected={len(selected)} windows "
f"sha256={digest}")
print(f"manifest -> {os.path.relpath(MANIFEST_PATH)}")
if __name__ == "__main__":
main()