{
  "run_id": "tfidf__sms_spam__20261006T043721Z",
  "backend": "tfidf",
  "model_version": "tfidf-logreg sklearn-1.9.1 solver=saga C=1.0 ngram=(1, 2) min_df=2 max_features=30000",
  "suite_id": "sms_spam",
  "suite_version": "1.0.0",
  "suite_title": "SMS spam",
  "primitive": "noul",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 104.2075279999608,
  "latency_p50_ms": 0.3499499999861655,
  "latency_p95_ms": 0.4046240000548096,
  "latency_mean_ms": 0.3557629772685099,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.02,
      "probability_drift": 0.03379511520263382,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 200,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.042583162200806,
      "attack_success_rate": 0.08
    }
  },
  "warnings": [
    "agent_gate: lowered min_df to 1 because the train split has 9 rows",
    "passage_relevance: lowered min_df to 1 because the train split has 8 rows"
  ],
  "license": "The Hugging Face card lists the license as unknown. This is the public SMS Spam Collection (Almeida, Hidalgo, Yamakami), distributed for spam-filtering research.",
  "n": 836,
  "n_valid": 836,
  "schema_validity": 1.0,
  "accuracy": 0.972488038277512,
  "accuracy_ci95": [
    0.9605263157894737,
    0.9832535885167464
  ],
  "macro_f1": 0.9355212989634717,
  "macro_f1_ci95": [
    0.9058837978077762,
    0.9611162790697674
  ],
  "ece": 0.06788708746573317,
  "ece_ci95": [
    0.06286386907728185,
    0.07883542984811473
  ],
  "brier": 0.050566600727240105,
  "brier_ci95": [
    0.040594389805747186,
    0.06190964360577258
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 96.57442706485057,
  "s1_score_ci95": [
    95.62754594682792,
    97.33425176753948
  ],
  "components": {
    "accuracy": 0.972488038277512,
    "macro_f1": 0.9355212989634717,
    "calibration": 0.9321129125342669,
    "brier_quality": 0.97471669963638,
    "quality": 0.9519203866407224,
    "schema_validity": 1.0,
    "robustness": 0.996,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 29,
      "confidence": 0.5541700924590145,
      "accuracy": 0.5517241379310345
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 25,
      "confidence": 0.6491544132888426,
      "accuracy": 0.76
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 45,
      "confidence": 0.7532936444044439,
      "accuracy": 0.9555555555555556
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 97,
      "confidence": 0.8617958050031563,
      "accuracy": 1.0
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 640,
      "confidence": 0.9478063439753613,
      "accuracy": 0.996875
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 836,
      "coverage": 1.0,
      "risk": 0.027511961722488043
    },
    {
      "threshold": 0.6,
      "n": 807,
      "coverage": 0.965311004784689,
      "risk": 0.012391573729863659
    },
    {
      "threshold": 0.7,
      "n": 782,
      "coverage": 0.9354066985645934,
      "risk": 0.005115089514066473
    },
    {
      "threshold": 0.8,
      "n": 737,
      "coverage": 0.881578947368421,
      "risk": 0.0027137042062415073
    },
    {
      "threshold": 0.9,
      "n": 640,
      "coverage": 0.7655502392344498,
      "risk": 0.0031250000000000444
    },
    {
      "threshold": 0.95,
      "n": 340,
      "coverage": 0.40669856459330145,
      "risk": 0.002941176470588225
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 84,
      "risk": 0.0
    },
    {
      "coverage": 0.2,
      "n": 167,
      "risk": 0.0
    },
    {
      "coverage": 0.3,
      "n": 251,
      "risk": 0.0
    },
    {
      "coverage": 0.4,
      "n": 334,
      "risk": 0.0029940119760478723
    },
    {
      "coverage": 0.5,
      "n": 418,
      "risk": 0.0023923444976076125
    },
    {
      "coverage": 0.6,
      "n": 502,
      "risk": 0.0019920318725099584
    },
    {
      "coverage": 0.7,
      "n": 585,
      "risk": 0.0017094017094017033
    },
    {
      "coverage": 0.8,
      "n": 669,
      "risk": 0.002989536621823663
    },
    {
      "coverage": 0.9,
      "n": 752,
      "risk": 0.0026595744680850686
    },
    {
      "coverage": 1.0,
      "n": 836,
      "risk": 0.027511961722488043
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}