{
  "run_id": "random__wiki_toxic__20261006T043916Z",
  "backend": "random",
  "model_version": "s1bench-random-1",
  "suite_id": "wiki_toxic",
  "suite_version": "1.0.0",
  "suite_title": "Wiki comment toxicity",
  "primitive": "noul",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 0.0,
  "latency_p50_ms": 0.008362500011571683,
  "latency_p95_ms": 0.01072204985348435,
  "latency_mean_ms": 0.008755954000207566,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": 0.515
    }
  },
  "warnings": [],
  "license": "CC0-1.0",
  "n": 500,
  "n_valid": 500,
  "schema_validity": 1.0,
  "accuracy": 0.494,
  "accuracy_ci95": [
    0.452,
    0.536
  ],
  "macro_f1": 0.493997975991904,
  "macro_f1_ci95": [
    0.45196382952724307,
    0.5359710445468511
  ],
  "ece": 0.20293181158365817,
  "ece_ci95": [
    0.1604386996003484,
    0.24684527009628812
  ],
  "brier": 0.616002665446712,
  "brier_ci95": [
    0.5765754763609446,
    0.6584023526732187
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 71.59488639150345,
  "s1_score_ci95": [
    69.01691471167429,
    74.11889958322539
  ],
  "components": {
    "accuracy": 0.494,
    "macro_f1": 0.493997975991904,
    "calibration": 0.7970681884163419,
    "brier_quality": 0.691998667276644,
    "quality": 0.5942126627357636,
    "schema_validity": 1.0,
    "robustness": 1.0,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 153,
      "confidence": 0.5440697718313956,
      "accuracy": 0.45751633986928103
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 128,
      "confidence": 0.6482484720625686,
      "accuracy": 0.5390625
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 90,
      "confidence": 0.7476885375578183,
      "accuracy": 0.5222222222222223
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 80,
      "confidence": 0.8536746238297687,
      "accuracy": 0.425
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 49,
      "confidence": 0.9522752651230945,
      "accuracy": 0.5510204081632653
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 500,
      "coverage": 1.0,
      "risk": 0.506
    },
    {
      "threshold": 0.6,
      "n": 347,
      "coverage": 0.694,
      "risk": 0.48991354466858794
    },
    {
      "threshold": 0.7,
      "n": 219,
      "coverage": 0.438,
      "risk": 0.5068493150684932
    },
    {
      "threshold": 0.8,
      "n": 129,
      "coverage": 0.258,
      "risk": 0.5271317829457365
    },
    {
      "threshold": 0.9,
      "n": 49,
      "coverage": 0.098,
      "risk": 0.44897959183673475
    },
    {
      "threshold": 0.95,
      "n": 23,
      "coverage": 0.046,
      "risk": 0.4347826086956522
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 50,
      "risk": 0.43999999999999995
    },
    {
      "coverage": 0.2,
      "n": 100,
      "risk": 0.5
    },
    {
      "coverage": 0.3,
      "n": 150,
      "risk": 0.5266666666666666
    },
    {
      "coverage": 0.4,
      "n": 200,
      "risk": 0.505
    },
    {
      "coverage": 0.5,
      "n": 250,
      "risk": 0.5
    },
    {
      "coverage": 0.6,
      "n": 300,
      "risk": 0.5
    },
    {
      "coverage": 0.7,
      "n": 350,
      "risk": 0.49428571428571433
    },
    {
      "coverage": 0.8,
      "n": 400,
      "risk": 0.49250000000000005
    },
    {
      "coverage": 0.9,
      "n": 450,
      "risk": 0.5
    },
    {
      "coverage": 1.0,
      "n": 500,
      "risk": 0.506
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}