{
  "run_id": "random__sms_spam__20261006T043915Z",
  "backend": "random",
  "model_version": "s1bench-random-1",
  "suite_id": "sms_spam",
  "suite_version": "1.0.0",
  "suite_title": "SMS spam",
  "primitive": "noul",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 0.0,
  "latency_p50_ms": 0.008317000038005062,
  "latency_p95_ms": 0.00965350005799337,
  "latency_mean_ms": 0.008604433014461914,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": 0.49
    }
  },
  "warnings": [],
  "license": "The Hugging Face card lists the license as unknown. This is the public SMS Spam Collection (Almeida, Hidalgo, Yamakami), distributed for spam-filtering research.",
  "n": 836,
  "n_valid": 836,
  "schema_validity": 1.0,
  "accuracy": 0.507177033492823,
  "accuracy_ci95": [
    0.47368421052631576,
    0.5430622009569378
  ],
  "macro_f1": 0.4250267094017094,
  "macro_f1_ci95": [
    0.3939020431399036,
    0.4577745921356787
  ],
  "ece": 0.17426533701250826,
  "ece_ci95": [
    0.13715750611137661,
    0.20841288175280723
  ],
  "brier": 0.5936027016369594,
  "brier_ci95": [
    0.5620638233350728,
    0.6264328888316362
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 70.69381883683245,
  "s1_score_ci95": [
    68.7449723048887,
    72.73820323248057
  ],
  "components": {
    "accuracy": 0.507177033492823,
    "macro_f1": 0.4250267094017094,
    "calibration": 0.8257346629874918,
    "brier_quality": 0.7031986491815203,
    "quality": 0.5813402690976064,
    "schema_validity": 1.0,
    "robustness": 1.0,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 315,
      "confidence": 0.548696629606817,
      "accuracy": 0.4793650793650794
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 186,
      "confidence": 0.6426511660405188,
      "accuracy": 0.510752688172043
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 141,
      "confidence": 0.7465398415258516,
      "accuracy": 0.5390070921985816
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 117,
      "confidence": 0.8465727862435789,
      "accuracy": 0.5982905982905983
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 77,
      "confidence": 0.9480796478847958,
      "accuracy": 0.4155844155844156
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 836,
      "coverage": 1.0,
      "risk": 0.49282296650717705
    },
    {
      "threshold": 0.6,
      "n": 521,
      "coverage": 0.6232057416267942,
      "risk": 0.4760076775431862
    },
    {
      "threshold": 0.7,
      "n": 335,
      "coverage": 0.4007177033492823,
      "risk": 0.4686567164179104
    },
    {
      "threshold": 0.8,
      "n": 194,
      "coverage": 0.23205741626794257,
      "risk": 0.4742268041237113
    },
    {
      "threshold": 0.9,
      "n": 77,
      "coverage": 0.09210526315789473,
      "risk": 0.5844155844155844
    },
    {
      "threshold": 0.95,
      "n": 40,
      "coverage": 0.04784688995215311,
      "risk": 0.675
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 84,
      "risk": 0.5833333333333333
    },
    {
      "coverage": 0.2,
      "n": 167,
      "risk": 0.5029940119760479
    },
    {
      "coverage": 0.3,
      "n": 251,
      "risk": 0.4701195219123506
    },
    {
      "coverage": 0.4,
      "n": 334,
      "risk": 0.47005988023952094
    },
    {
      "coverage": 0.5,
      "n": 418,
      "risk": 0.47846889952153115
    },
    {
      "coverage": 0.6,
      "n": 502,
      "risk": 0.47808764940239046
    },
    {
      "coverage": 0.7,
      "n": 585,
      "risk": 0.4717948717948718
    },
    {
      "coverage": 0.8,
      "n": 669,
      "risk": 0.4828101644245142
    },
    {
      "coverage": 0.9,
      "n": 752,
      "risk": 0.48404255319148937
    },
    {
      "coverage": 1.0,
      "n": 836,
      "risk": 0.49282296650717705
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}