{
  "run_id": "random__helpsteer2__20261006T043915Z",
  "backend": "random",
  "model_version": "s1bench-random-1",
  "suite_id": "helpsteer2",
  "suite_version": "1.0.0",
  "suite_title": "HelpSteer2 helpfulness",
  "primitive": "score",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 0.0,
  "latency_p50_ms": 0.012580499969772063,
  "latency_p95_ms": 0.015667750074044307,
  "latency_mean_ms": 0.013906846674368959,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "paraphrase": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": 0.28
    }
  },
  "warnings": [],
  "license": "CC-BY-4.0",
  "n": 300,
  "n_valid": 300,
  "schema_validity": 1.0,
  "accuracy": 0.20666666666666667,
  "accuracy_ci95": [
    0.16,
    0.25008333333333327
  ],
  "macro_f1": 0.2035690279690548,
  "macro_f1_ci95": [
    0.15806981826738747,
    0.2471716088940168
  ],
  "ece": 0.1499488323232201,
  "ece_ci95": [
    0.10963313561754248,
    0.20034374268715013
  ],
  "brier": 0.8683815430378521,
  "brier_ci95": [
    0.8410226104354683,
    0.8956438599031006
  ],
  "brier_max": 2.0,
  "score_mae": 1.2973528795784655,
  "s1_score": 58.426153398118466,
  "s1_score_ci95": [
    55.67885169284461,
    60.85908793480785
  ],
  "components": {
    "accuracy": 0.20666666666666667,
    "macro_f1": 0.2035690279690548,
    "calibration": 0.8500511676767799,
    "brier_quality": 0.565809228481074,
    "quality": 0.4060879056874066,
    "schema_validity": 1.0,
    "robustness": 1.0,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 88,
      "confidence": 0.2726155659528237,
      "accuracy": 0.25
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 139,
      "confidence": 0.3475072608605629,
      "accuracy": 0.2158273381294964
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 52,
      "confidence": 0.43685511100031466,
      "accuracy": 0.17307692307692307
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 17,
      "confidence": 0.5459786597659856,
      "accuracy": 0.058823529411764705
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 3,
      "confidence": 0.6414974414040877,
      "accuracy": 0.0
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 1,
      "confidence": 0.7683753212489143,
      "accuracy": 0.0
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 0,
      "confidence": null,
      "accuracy": null
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 21,
      "coverage": 0.07,
      "risk": 0.9523809523809523
    },
    {
      "threshold": 0.6,
      "n": 4,
      "coverage": 0.013333333333333334,
      "risk": 1.0
    },
    {
      "threshold": 0.7,
      "n": 1,
      "coverage": 0.0033333333333333335,
      "risk": 1.0
    },
    {
      "threshold": 0.8,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    },
    {
      "threshold": 0.9,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    },
    {
      "threshold": 0.95,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 30,
      "risk": 0.9333333333333333
    },
    {
      "coverage": 0.2,
      "n": 60,
      "risk": 0.8333333333333334
    },
    {
      "coverage": 0.3,
      "n": 90,
      "risk": 0.8555555555555556
    },
    {
      "coverage": 0.4,
      "n": 120,
      "risk": 0.8416666666666667
    },
    {
      "coverage": 0.5,
      "n": 150,
      "risk": 0.84
    },
    {
      "coverage": 0.6,
      "n": 180,
      "risk": 0.8277777777777777
    },
    {
      "coverage": 0.7,
      "n": 210,
      "risk": 0.8095238095238095
    },
    {
      "coverage": 0.8,
      "n": 240,
      "risk": 0.8041666666666667
    },
    {
      "coverage": 0.9,
      "n": 270,
      "risk": 0.8
    },
    {
      "coverage": 1.0,
      "n": 300,
      "risk": 0.7933333333333333
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}