{
  "run_id": "tfidf__banking77__20261006T043714Z",
  "backend": "tfidf",
  "model_version": "tfidf-logreg sklearn-1.9.1 solver=saga C=1.0 ngram=(1, 2) min_df=2 max_features=30000",
  "suite_id": "banking77",
  "suite_version": "1.0.0",
  "suite_title": "Banking77 intent",
  "primitive": "choice",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 597.7090449998741,
  "latency_p50_ms": 0.429528000040591,
  "latency_p95_ms": 0.4925226000182191,
  "latency_mean_ms": 0.44887893376728966,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.135,
      "probability_drift": 0.36163686647229965,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.06,
      "probability_drift": 0.2966539305925714,
      "attack_success_rate": 0.01
    }
  },
  "warnings": [
    "agent_gate: lowered min_df to 1 because the train split has 9 rows"
  ],
  "license": "CC-BY-4.0",
  "n": 3080,
  "n_valid": 3080,
  "schema_validity": 1.0,
  "accuracy": 0.8493506493506493,
  "accuracy_ci95": [
    0.8366883116883117,
    0.8623376623376623
  ],
  "macro_f1": 0.8481986438374006,
  "macro_f1_ci95": [
    0.8340588214904499,
    0.8594183910926018
  ],
  "ece": 0.46202442353303247,
  "ece_ci95": [
    0.4497575522678591,
    0.4739571037929543
  ],
  "brier": 0.47439655962455407,
  "brier_ci95": [
    0.46467594517076577,
    0.48452090683183996
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 83.27038529081834,
  "s1_score_ci95": [
    82.78856642928504,
    83.68199666931987
  ],
  "components": {
    "accuracy": 0.8493506493506493,
    "macro_f1": 0.8481986438374006,
    "calibration": 0.5379755764669676,
    "brier_quality": 0.762801720187723,
    "quality": 0.7693626470116905,
    "schema_validity": 1.0,
    "robustness": 0.961,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 324,
      "confidence": 0.07200750952042832,
      "accuracy": 0.45987654320987653
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 532,
      "confidence": 0.14984656690093756,
      "accuracy": 0.7368421052631579
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 475,
      "confidence": 0.2488418878762042,
      "accuracy": 0.8463157894736842
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 410,
      "confidence": 0.35176277560769836,
      "accuracy": 0.9048780487804878
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 355,
      "confidence": 0.4467645239515596,
      "accuracy": 0.9295774647887324
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 316,
      "confidence": 0.5451718704682786,
      "accuracy": 0.9778481012658228
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 265,
      "confidence": 0.6487724269800687,
      "accuracy": 0.9924528301886792
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 213,
      "confidence": 0.752482054513203,
      "accuracy": 0.9906103286384976
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 144,
      "confidence": 0.8458564328505849,
      "accuracy": 0.9930555555555556
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 46,
      "confidence": 0.926324346515103,
      "accuracy": 1.0
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 984,
      "coverage": 0.3194805194805195,
      "risk": 0.012195121951219523
    },
    {
      "threshold": 0.6,
      "n": 668,
      "coverage": 0.21688311688311687,
      "risk": 0.007485029940119792
    },
    {
      "threshold": 0.7,
      "n": 403,
      "coverage": 0.13084415584415585,
      "risk": 0.007444168734491274
    },
    {
      "threshold": 0.8,
      "n": 190,
      "coverage": 0.06168831168831169,
      "risk": 0.0052631578947368585
    },
    {
      "threshold": 0.9,
      "n": 46,
      "coverage": 0.014935064935064935,
      "risk": 0.0
    },
    {
      "threshold": 0.95,
      "n": 10,
      "coverage": 0.003246753246753247,
      "risk": 0.0
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 308,
      "risk": 0.00649350649350644
    },
    {
      "coverage": 0.2,
      "n": 616,
      "risk": 0.008116883116883078
    },
    {
      "coverage": 0.3,
      "n": 924,
      "risk": 0.011904761904761862
    },
    {
      "coverage": 0.4,
      "n": 1232,
      "risk": 0.025974025974025983
    },
    {
      "coverage": 0.5,
      "n": 1540,
      "risk": 0.038961038961038974
    },
    {
      "coverage": 0.6,
      "n": 1848,
      "risk": 0.0492424242424242
    },
    {
      "coverage": 0.7,
      "n": 2156,
      "risk": 0.0644712430426716
    },
    {
      "coverage": 0.8,
      "n": 2464,
      "risk": 0.08441558441558439
    },
    {
      "coverage": 0.9,
      "n": 2772,
      "risk": 0.10569985569985574
    },
    {
      "coverage": 1.0,
      "n": 3080,
      "risk": 0.1506493506493507
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}