{
  "run_id": "decider__agent_tool_routing__20261011T154647Z",
  "backend": "decider",
  "model_version": "StrandsAgents/strands-decider-2B-hobson-v19@strands-decider-0.1.0",
  "suite_id": "agent_tool_routing",
  "suite_version": "1.0.0",
  "suite_title": "Agent tool routing (synthetic)",
  "primitive": "choice",
  "protocol": "1.0.0",
  "split": "test",
  "limited": true,
  "limit": 200,
  "prepare_ms": 0.0,
  "latency_p50_ms": 17446.90095099986,
  "latency_p95_ms": 21289.74052094954,
  "latency_mean_ms": 17906.655647875,
  "batch_size": 1,
  "concurrency": 1,
  "cost_usd_total": 0.0,
  "cost_per_1k_usd": 0.0,
  "hardware": {
    "platform": "Linux-6.12.94+-x86_64-with-glibc2.41",
    "python": "3.12.14",
    "machine": "x86_64",
    "processor": "x86_64",
    "cpu_count": 8
  },
  "robustness_probes": {
    "order_reverse": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.4878048780487805,
      "probability_drift": 0.3324033831746772,
      "attack_success_rate": null
    },
    "order_shuffle": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.43902439024390244,
      "probability_drift": 0.28550784411523455,
      "attack_success_rate": null
    },
    "paraphrase": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.2926829268292683,
      "probability_drift": 0.26532822666825495,
      "attack_success_rate": null
    },
    "irrelevant_context": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.1951219512195122,
      "probability_drift": 0.1334161787248152,
      "attack_success_rate": null
    },
    "adversarial": {
      "n": 41,
      "invalid_pairs": 0,
      "flip_rate": 0.6829268292682927,
      "probability_drift": 0.47403365768739536,
      "attack_success_rate": 0.6585365853658537
    }
  },
  "warnings": [],
  "license": "CC0-1.0",
  "n": 200,
  "n_valid": 200,
  "schema_validity": 1.0,
  "accuracy": 0.37,
  "accuracy_ci95": [
    0.299875,
    0.44
  ],
  "macro_f1": 0.09818999627823158,
  "macro_f1_ci95": [
    0.08227413311322733,
    0.10978236301250716
  ],
  "ece": 0.16556851971522063,
  "ece_ci95": [
    0.14475438681508795,
    0.24848033206562203
  ],
  "brier": 0.8039544046122188,
  "brier_ci95": [
    0.7405571921174032,
    0.8624045266289443
  ],
  "brier_max": 2.0,
  "score_mae": null,
  "s1_score": 52.642331873688775,
  "s1_score_ci95": [
    49.962678076338875,
    54.25250419949675
  ],
  "components": {
    "accuracy": 0.37,
    "macro_f1": 0.09818999627823158,
    "calibration": 0.8344314802847794,
    "brier_quality": 0.5980227976938906,
    "quality": 0.41335735429311504,
    "schema_validity": 1.0,
    "robustness": 0.5804878048780487,
    "robustness_measured": true
  },
  "composite_id": "s1-v1",
  "reliability": [
    {
      "bin": 0,
      "low": 0.0,
      "high": 0.1,
      "count": 0,
      "confidence": null,
      "accuracy": null
    },
    {
      "bin": 1,
      "low": 0.1,
      "high": 0.2,
      "count": 15,
      "confidence": 0.17338783239468034,
      "accuracy": 0.13333333333333333
    },
    {
      "bin": 2,
      "low": 0.2,
      "high": 0.3,
      "count": 44,
      "confidence": 0.2528526695065533,
      "accuracy": 0.25
    },
    {
      "bin": 3,
      "low": 0.3,
      "high": 0.4,
      "count": 33,
      "confidence": 0.3568263040016732,
      "accuracy": 0.21212121212121213
    },
    {
      "bin": 4,
      "low": 0.4,
      "high": 0.5,
      "count": 39,
      "confidence": 0.44443107303437374,
      "accuracy": 0.46153846153846156
    },
    {
      "bin": 5,
      "low": 0.5,
      "high": 0.6,
      "count": 22,
      "confidence": 0.5540637212296122,
      "accuracy": 0.8636363636363636
    },
    {
      "bin": 6,
      "low": 0.6,
      "high": 0.7,
      "count": 16,
      "confidence": 0.643324781729274,
      "accuracy": 0.6875
    },
    {
      "bin": 7,
      "low": 0.7,
      "high": 0.8,
      "count": 14,
      "confidence": 0.7347546581697976,
      "accuracy": 0.21428571428571427
    },
    {
      "bin": 8,
      "low": 0.8,
      "high": 0.9,
      "count": 7,
      "confidence": 0.8504624845609737,
      "accuracy": 0.42857142857142855
    },
    {
      "bin": 9,
      "low": 0.9,
      "high": 1.0,
      "count": 10,
      "confidence": 0.9187708583536807,
      "accuracy": 0.0
    }
  ],
  "selective": [
    {
      "threshold": 0.5,
      "n": 69,
      "coverage": 0.345,
      "risk": 0.4782608695652174
    },
    {
      "threshold": 0.6,
      "n": 47,
      "coverage": 0.235,
      "risk": 0.6382978723404256
    },
    {
      "threshold": 0.7,
      "n": 31,
      "coverage": 0.155,
      "risk": 0.8064516129032258
    },
    {
      "threshold": 0.8,
      "n": 17,
      "coverage": 0.085,
      "risk": 0.8235294117647058
    },
    {
      "threshold": 0.9,
      "n": 10,
      "coverage": 0.05,
      "risk": 1.0
    },
    {
      "threshold": 0.95,
      "n": 0,
      "coverage": 0.0,
      "risk": null
    }
  ],
  "risk_coverage": [
    {
      "coverage": 0.1,
      "n": 20,
      "risk": 0.85
    },
    {
      "coverage": 0.2,
      "n": 40,
      "risk": 0.725
    },
    {
      "coverage": 0.3,
      "n": 60,
      "risk": 0.5333333333333333
    },
    {
      "coverage": 0.4,
      "n": 80,
      "risk": 0.4625
    },
    {
      "coverage": 0.5,
      "n": 100,
      "risk": 0.49
    },
    {
      "coverage": 0.6,
      "n": 120,
      "risk": 0.5333333333333333
    },
    {
      "coverage": 0.7,
      "n": 140,
      "risk": 0.5642857142857143
    },
    {
      "coverage": 0.8,
      "n": 160,
      "risk": 0.6
    },
    {
      "coverage": 0.9,
      "n": 180,
      "risk": 0.6055555555555556
    },
    {
      "coverage": 1.0,
      "n": 200,
      "risk": 0.63
    }
  ],
  "bootstrap": {
    "n": 1000,
    "seed": 20261006,
    "interval": "percentile 2.5/97.5"
  },
  "notes": {
    "s1_score_interval": "The interval resamples test items and keeps the robustness term fixed at its point estimate. It is not a bootstrap of the probe subset."
  }
}