{
  "run_id": "tfidf__ag_news__20261006T043713Z",
  "backend": "tfidf",
  "model_version": "tfidf-logreg sklearn-1.9.1 solver=saga C=1.0 ngram=(1, 2) min_df=2 max_features=30000",
  "suite_id": "ag_news",
  "suite_version": "1.0.0",
  "suite_title": "AG News topic",
  "primitive": "choice",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 146.27822100010235,
  "latency_p50_ms": 0.3554764999762483,
  "latency_p95_ms": 0.42466979998607707,
  "latency_mean_ms": 0.36243244374816186,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.08,
      "probability_drift": 0.08343444329025437,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.145,
      "probability_drift": 0.11516423153094875,
      "attack_success_rate": 0.23
    }
  },
  "warnings": [],
  "license": "License unknown on the Hugging Face card. Research corpus from Zhang, Zhao, and LeCun (2015). Frozen subsample only.",
  "n": 800,
  "n_valid": 800,
  "schema_validity": 1.0,
  "accuracy": 0.8625,
  "accuracy_ci95": [
    0.83871875,
    0.88875
  ],
  "macro_f1": 0.8614151534918799,
  "macro_f1_ci95": [
    0.8366194506009923,
    0.8867490065487714
  ],
  "ece": 0.3099348637933123,
  "ece_ci95": [
    0.2875465346198249,
    0.3350682955744762
  ],
  "brier": 0.3580091402883275,
  "brier_ci95": [
    0.34011655089146037,
    0.37541673530959435
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 86.67826918542639,
  "s1_score_ci95": [
    85.82294506534443,
    87.57815368752821
  ],
  "components": {
    "accuracy": 0.8625,
    "macro_f1": 0.8614151534918799,
    "calibration": 0.6900651362066876,
    "brier_quality": 0.8209954298558363,
    "quality": 0.8193324169346627,
    "schema_validity": 1.0,
    "robustness": 0.955,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 13,
      "confidence": 0.2823173715772055,
      "accuracy": 0.38461538461538464
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 132,
      "confidence": 0.3561469795512693,
      "accuracy": 0.6515151515151515
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 189,
      "confidence": 0.44643809955454117,
      "accuracy": 0.8571428571428571
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 160,
      "confidence": 0.5504257814840301,
      "accuracy": 0.90625
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 141,
      "confidence": 0.6478933884158031,
      "accuracy": 0.950354609929078
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 123,
      "confidence": 0.7448480848166935,
      "accuracy": 0.959349593495935
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 34,
      "confidence": 0.840651992692627,
      "accuracy": 0.9411764705882353
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 8,
      "confidence": 0.9217757537743747,
      "accuracy": 1.0
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 466,
      "coverage": 0.5825,
      "risk": 0.0622317596566524
    },
    {
      "threshold": 0.6,
      "n": 306,
      "coverage": 0.3825,
      "risk": 0.04575163398692805
    },
    {
      "threshold": 0.7,
      "n": 165,
      "coverage": 0.20625,
      "risk": 0.042424242424242475
    },
    {
      "threshold": 0.8,
      "n": 42,
      "coverage": 0.0525,
      "risk": 0.04761904761904767
    },
    {
      "threshold": 0.9,
      "n": 8,
      "coverage": 0.01,
      "risk": 0.0
    },
    {
      "threshold": 0.95,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 80,
      "risk": 0.025000000000000022
    },
    {
      "coverage": 0.2,
      "n": 160,
      "risk": 0.043749999999999956
    },
    {
      "coverage": 0.3,
      "n": 240,
      "risk": 0.054166666666666696
    },
    {
      "coverage": 0.4,
      "n": 320,
      "risk": 0.046875
    },
    {
      "coverage": 0.5,
      "n": 400,
      "risk": 0.050000000000000044
    },
    {
      "coverage": 0.6,
      "n": 480,
      "risk": 0.0625
    },
    {
      "coverage": 0.7,
      "n": 560,
      "risk": 0.0714285714285714
    },
    {
      "coverage": 0.8,
      "n": 640,
      "risk": 0.08125000000000004
    },
    {
      "coverage": 0.9,
      "n": 720,
      "risk": 0.1069444444444444
    },
    {
      "coverage": 1.0,
      "n": 800,
      "risk": 0.13749999999999996
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}