{
  "run_id": "tfidf__prompt_injection__20261006T043721Z",
  "backend": "tfidf",
  "model_version": "tfidf-logreg sklearn-1.9.1 solver=saga C=1.0 ngram=(1, 2) min_df=2 max_features=30000",
  "suite_id": "prompt_injection",
  "suite_version": "1.0.0",
  "suite_title": "Prompt injection",
  "primitive": "noul",
  "protocol": "1.0.0",
  "split": "test",
  "limited": false,
  "limit": null,
  "prepare_ms": 16.205455000090296,
  "latency_p50_ms": 0.3391975000113234,
  "latency_p95_ms": 0.4050397499213432,
  "latency_mean_ms": 0.3459420258544579,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.39",
    "python": "3.12.3",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 4
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 116,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 116,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 116,
      "invalid_pairs": 0,
      "flip_rate": 0.0,
      "probability_drift": 0.0,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 116,
      "invalid_pairs": 0,
      "flip_rate": 0.04310344827586207,
      "probability_drift": 0.08783361638937079,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 116,
      "invalid_pairs": 0,
      "flip_rate": 0.5862068965517241,
      "probability_drift": 0.24499855222753414,
      "attack_success_rate": 0.8879310344827587
    }
  },
  "warnings": [
    "agent_gate: lowered min_df to 1 because the train split has 9 rows",
    "passage_relevance: lowered min_df to 1 because the train split has 8 rows"
  ],
  "license": "Apache-2.0 on the dataset card. dataset_info also says CC-BY-4.0.",
  "n": 116,
  "n_valid": 116,
  "schema_validity": 1.0,
  "accuracy": 0.7672413793103449,
  "accuracy_ci95": [
    0.6896551724137931,
    0.8448275862068966
  ],
  "macro_f1": 0.7593545908567039,
  "macro_f1_ci95": [
    0.678138623051397,
    0.8379169419849346
  ],
  "ece": 0.07999925524156536,
  "ece_ci95": [
    0.052847539029704134,
    0.1581378254905666
  ],
  "brier": 0.3290691929105492,
  "brier_ci95": [
    0.276424529629625,
    0.38312987124296766
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 84.71950665568177,
  "s1_score_ci95": [
    80.95681410541178,
    87.29661367680649
  ],
  "components": {
    "accuracy": 0.7672413793103449,
    "macro_f1": 0.7593545908567039,
    "calibration": 0.9200007447584346,
    "brier_quality": 0.8354654035447254,
    "quality": 0.8086776812880646,
    "schema_validity": 1.0,
    "robustness": 0.8741379310344828,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 33,
      "confidence": 0.5525908722811984,
      "accuracy": 0.5757575757575758
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 33,
      "confidence": 0.6642299547285296,
      "accuracy": 0.7575757575757576
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 28,
      "confidence": 0.7488974537744719,
      "accuracy": 0.8214285714285714
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 22,
      "confidence": 0.8452668361350991,
      "accuracy": 1.0
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 0,
      "confidence": null,
      "accuracy": null
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 116,
      "coverage": 1.0,
      "risk": 0.23275862068965514
    },
    {
      "threshold": 0.6,
      "n": 83,
      "coverage": 0.7155172413793104,
      "risk": 0.15662650602409633
    },
    {
      "threshold": 0.7,
      "n": 50,
      "coverage": 0.43103448275862066,
      "risk": 0.09999999999999998
    },
    {
      "threshold": 0.8,
      "n": 22,
      "coverage": 0.1896551724137931,
      "risk": 0.0
    },
    {
      "threshold": 0.9,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    },
    {
      "threshold": 0.95,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 12,
      "risk": 0.0
    },
    {
      "coverage": 0.2,
      "n": 23,
      "risk": 0.04347826086956519
    },
    {
      "coverage": 0.3,
      "n": 35,
      "risk": 0.05714285714285716
    },
    {
      "coverage": 0.4,
      "n": 46,
      "risk": 0.08695652173913049
    },
    {
      "coverage": 0.5,
      "n": 58,
      "risk": 0.10344827586206895
    },
    {
      "coverage": 0.6,
      "n": 70,
      "risk": 0.1428571428571429
    },
    {
      "coverage": 0.7,
      "n": 81,
      "risk": 0.14814814814814814
    },
    {
      "coverage": 0.8,
      "n": 93,
      "risk": 0.18279569892473113
    },
    {
      "coverage": 0.9,
      "n": 104,
      "risk": 0.21153846153846156
    },
    {
      "coverage": 1.0,
      "n": 116,
      "risk": 0.23275862068965514
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}