{
  "run_id": "decider__ag_news__20261011T151717Z",
  "backend": "decider",
  "model_version": "StrandsAgents/strands-decider-2B-hobson-v19@strands-decider-0.1.0",
  "suite_id": "ag_news",
  "suite_version": "1.0.0",
  "suite_title": "AG News topic",
  "primitive": "choice",
  "protocol": "1.0.0",
  "split": "test",
  "limited": true,
  "limit": 200,
  "prepare_ms": 0.0,
  "latency_p50_ms": 2113.675608999756,
  "latency_p95_ms": 2866.3785798997874,
  "latency_mean_ms": 2231.4226066249967,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.41",
    "python": "3.12.14",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 8
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.03601243309417311,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.024390243902439025,
      "probability_drift": 0.01760872163672136,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.07317073170731707,
      "probability_drift": 0.06484114388799682,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.024390243902439025,
      "probability_drift": 0.0322066144769198,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.06733587572062705,
      "attack_success_rate": 0.24390243902439024
    }
  },
  "warnings": [],
  "license": "License unknown on the Hugging Face card. Research corpus from Zhang, Zhao, and LeCun (2015). Frozen subsample only.",
  "n": 200,
  "n_valid": 200,
  "schema_validity": 1.0,
  "accuracy": 0.9,
  "accuracy_ci95": [
    0.86,
    0.94
  ],
  "macro_f1": 0.8995739348370927,
  "macro_f1_ci95": [
    0.8576836228997338,
    0.9387927455671264
  ],
  "ece": 0.05983420779209708,
  "ece_ci95": [
    0.03599660605687632,
    0.10801199455502634
  ],
  "brier": 0.18434971013786314,
  "brier_ci95": [
    0.1259166431827344,
    0.2497462181423108
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 93.2955808649178,
  "s1_score_ci95": [
    90.71762249829973,
    95.40607758941424
  ],
  "components": {
    "accuracy": 0.9,
    "macro_f1": 0.8995739348370927,
    "calibration": 0.940165792207903,
    "brier_quality": 0.9078251449310685,
    "quality": 0.9094490646207768,
    "schema_validity": 1.0,
    "robustness": 0.975609756097561,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 1,
      "confidence": 0.3957,
      "accuracy": 1.0
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 2,
      "confidence": 0.45605,
      "accuracy": 1.0
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 9,
      "confidence": 0.5599270063559367,
      "accuracy": 0.6666666666666666
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 10,
      "confidence": 0.6573771602839716,
      "accuracy": 0.7
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 17,
      "confidence": 0.7496489757355486,
      "accuracy": 0.8235294117647058
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 14,
      "confidence": 0.8469280290779947,
      "accuracy": 0.9285714285714286
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 147,
      "confidence": 0.9761141579119648,
      "accuracy": 0.9319727891156463
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 197,
      "coverage": 0.985,
      "risk": 0.10152284263959388
    },
    {
      "threshold": 0.6,
      "n": 188,
      "coverage": 0.94,
      "risk": 0.09042553191489366
    },
    {
      "threshold": 0.7,
      "n": 178,
      "coverage": 0.89,
      "risk": 0.0786516853932584
    },
    {
      "threshold": 0.8,
      "n": 161,
      "coverage": 0.805,
      "risk": 0.06832298136645965
    },
    {
      "threshold": 0.9,
      "n": 147,
      "coverage": 0.735,
      "risk": 0.0680272108843537
    },
    {
      "threshold": 0.95,
      "n": 124,
      "coverage": 0.62,
      "risk": 0.048387096774193505
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 20,
      "risk": 0.050000000000000044
    },
    {
      "coverage": 0.2,
      "n": 40,
      "risk": 0.07499999999999996
    },
    {
      "coverage": 0.3,
      "n": 60,
      "risk": 0.050000000000000044
    },
    {
      "coverage": 0.4,
      "n": 80,
      "risk": 0.03749999999999998
    },
    {
      "coverage": 0.5,
      "n": 100,
      "risk": 0.040000000000000036
    },
    {
      "coverage": 0.6,
      "n": 120,
      "risk": 0.050000000000000044
    },
    {
      "coverage": 0.7,
      "n": 140,
      "risk": 0.050000000000000044
    },
    {
      "coverage": 0.8,
      "n": 160,
      "risk": 0.06874999999999998
    },
    {
      "coverage": 0.9,
      "n": 180,
      "risk": 0.08333333333333337
    },
    {
      "coverage": 1.0,
      "n": 200,
      "risk": 0.09999999999999998
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}