{
  "title": "BANKING77 routing experiment",
  "collected_on": "2026-09-22",
  "run_id": "35714478131",
  "collection_commit": "a60285ab70e471d0b473952a729c5f94f7d1f452",
  "analysis_commit": "bbc5babf40ecaec448225766666c2b11a93248aa",
  "criteria_version": "banking77-train-descriptions-v1",
  "interpretation": "Retrospective revised evaluation; test subset previously inspected. Costs at supplied tariffs, not provider invoices. Live cascade uses the original cascade policy, not the later Jev-only selection.",
  "selected_strategy": "jev-only",
  "prices": {
    "as_of": "2026-09-22",
    "source": "https://docs.typesafe.ai/models ; https://ai.google.dev/gemini-api/docs/pricing ; Standard paid text rates; excludes caching discounts and free credits.",
    "jev": {
      "model": "jev-latest",
      "input_per_million": 0.042,
      "output_per_million": 0
    },
    "llm": {
      "model": "gemini-3.1-flash-lite",
      "input_per_million": 0.25,
      "output_per_million": 1.5
    }
  },
  "dev": [
    {
      "strategy": "llm",
      "threshold": null,
      "count": 385,
      "correct": 336,
      "accuracy": 0.8727272727272727,
      "accuracy_wilson_95": [
        0.8357149974408103,
        0.9023750222562504
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.0,
      "accepted_error_rate": null,
      "fallback_rate": 0.0,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.18945474999999995,
      "total_cost_usd": 0.18945474999999995,
      "cost_per_1000_usd": 0.4920902597402596,
      "p50_ms": 572.1721479999928,
      "p95_ms": 828.3998714000062,
      "latency_basis": "observed_wall_clock",
      "savings_vs_llm": 0.0,
      "accuracy_delta_vs_llm": 0.0
    },
    {
      "strategy": "jev",
      "threshold": null,
      "count": 385,
      "correct": 341,
      "accuracy": 0.8857142857142857,
      "accuracy_wilson_95": [
        0.8500518510113155,
        0.9137555909748414
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 1.0,
      "accepted_error_rate": 0.11428571428571428,
      "fallback_rate": 0.0,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.04171108200000002,
      "total_cost_usd": 0.04171108200000002,
      "cost_per_1000_usd": 0.10834047272727276,
      "p50_ms": 106.98557899996786,
      "p95_ms": 170.69298179997065,
      "latency_basis": "observed_wall_clock",
      "savings_vs_llm": 0.7798361772402116,
      "accuracy_delta_vs_llm": 0.012987012987012991
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.5,
      "count": 385,
      "correct": 339,
      "accuracy": 0.8805194805194805,
      "accuracy_wilson_95": [
        0.8443029289642335,
        0.909217544106285
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.9584415584415584,
      "accepted_error_rate": 0.0975609756097561,
      "fallback_rate": 0.04155844155844156,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.04957408200000001,
      "total_cost_usd": 0.04957408200000001,
      "cost_per_1000_usd": 0.12876384935064938,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": 0.7383328631243079,
      "accuracy_delta_vs_llm": 0.007792207792207795
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.6,
      "count": 385,
      "correct": 340,
      "accuracy": 0.8831168831168831,
      "accuracy_wilson_95": [
        0.8471749389454775,
        0.9114890185828604
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.9272727272727272,
      "accepted_error_rate": 0.0896358543417367,
      "fallback_rate": 0.07272727272727272,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.055489832,
      "total_cost_usd": 0.055489832,
      "cost_per_1000_usd": 0.14412943376623377,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": 0.7071077288904077,
      "accuracy_delta_vs_llm": 0.010389610389610393
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.7,
      "count": 385,
      "correct": 340,
      "accuracy": 0.8831168831168831,
      "accuracy_wilson_95": [
        0.8471749389454775,
        0.9114890185828604
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.9038961038961039,
      "accepted_error_rate": 0.07758620689655173,
      "fallback_rate": 0.09610389610389611,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.059929832,
      "total_cost_usd": 0.059929832,
      "cost_per_1000_usd": 0.1556619012987013,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": 0.6836720536170245,
      "accuracy_delta_vs_llm": 0.010389610389610393
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.8,
      "count": 385,
      "correct": 340,
      "accuracy": 0.8831168831168831,
      "accuracy_wilson_95": [
        0.8471749389454775,
        0.9114890185828604
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.8545454545454545,
      "accepted_error_rate": 0.0486322188449848,
      "fallback_rate": 0.14545454545454545,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.06927258199999999,
      "total_cost_usd": 0.06927258199999999,
      "cost_per_1000_usd": 0.17992878441558438,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": 0.634358167319637,
      "accuracy_delta_vs_llm": 0.010389610389610393
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.9,
      "count": 385,
      "correct": 339,
      "accuracy": 0.8805194805194805,
      "accuracy_wilson_95": [
        0.8443029289642335,
        0.909217544106285
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.7844155844155845,
      "accepted_error_rate": 0.033112582781456956,
      "fallback_rate": 0.21558441558441557,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.08259283199999996,
      "total_cost_usd": 0.08259283199999996,
      "cost_per_1000_usd": 0.21452683636363626,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": 0.564049821923177,
      "accuracy_delta_vs_llm": 0.007792207792207795
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.95,
      "count": 385,
      "correct": 338,
      "accuracy": 0.8779220779220779,
      "accuracy_wilson_95": [
        0.8414356862448582,
        0.906941302367841
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.7584415584415585,
      "accepted_error_rate": 0.0273972602739726,
      "fallback_rate": 0.24155844155844156,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.08753458200000001,
      "total_cost_usd": 0.08753458200000001,
      "cost_per_1000_usd": 0.22736255064935068,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": 0.5379657569947439,
      "accuracy_delta_vs_llm": 0.0051948051948051965
    }
  ],
  "test": [
    {
      "strategy": "llm",
      "threshold": null,
      "count": 770,
      "correct": 665,
      "accuracy": 0.8636363636363636,
      "accuracy_wilson_95": [
        0.8375850140306735,
        0.8860774282135291
      ],
      "request_failure_rate": 0.0012987012987012987,
      "jev_acceptance_rate": 0.0,
      "accepted_error_rate": null,
      "fallback_rate": 0.0,
      "cost_coverage": 0.9987012987012988,
      "known_cost_subtotal_usd": 0.37838075000000043,
      "total_cost_usd": null,
      "cost_per_1000_usd": null,
      "p50_ms": 572.457782000015,
      "p95_ms": 773.5379201000027,
      "latency_basis": "observed_wall_clock",
      "savings_vs_llm": null,
      "accuracy_delta_vs_llm": 0.0
    },
    {
      "strategy": "jev",
      "threshold": null,
      "count": 770,
      "correct": 674,
      "accuracy": 0.8753246753246753,
      "accuracy_wilson_95": [
        0.8501116692289696,
        0.8968113515159393
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 1.0,
      "accepted_error_rate": 0.12467532467532468,
      "fallback_rate": 0.0,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.08338415400000004,
      "total_cost_usd": 0.08338415400000004,
      "cost_per_1000_usd": 0.10829110909090914,
      "p50_ms": 104.7321240000656,
      "p95_ms": 157.54240014999255,
      "latency_basis": "observed_wall_clock",
      "savings_vs_llm": null,
      "accuracy_delta_vs_llm": 0.011688311688311637
    },
    {
      "strategy": "cascade-replay",
      "threshold": 0.5,
      "count": 770,
      "correct": 677,
      "accuracy": 0.8792207792207792,
      "accuracy_wilson_95": [
        0.85430147672834,
        0.9003750701834713
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.9818181818181818,
      "accepted_error_rate": 0.11375661375661375,
      "fallback_rate": 0.01818181818181818,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.09025715400000006,
      "total_cost_usd": 0.09025715400000006,
      "cost_per_1000_usd": 0.1172170831168832,
      "p50_ms": null,
      "p95_ms": null,
      "latency_basis": "not_measured",
      "savings_vs_llm": null,
      "accuracy_delta_vs_llm": 0.01558441558441559
    }
  ],
  "live": [
    {
      "strategy": "cascade-live",
      "threshold": 0.5,
      "count": 770,
      "correct": 674,
      "accuracy": 0.8753246753246753,
      "accuracy_wilson_95": [
        0.8501116692289696,
        0.8968113515159393
      ],
      "request_failure_rate": 0.0,
      "jev_acceptance_rate": 0.9792207792207792,
      "accepted_error_rate": 0.11538461538461539,
      "fallback_rate": 0.02077922077922078,
      "cost_coverage": 1.0,
      "known_cost_subtotal_usd": 0.091258904,
      "total_cost_usd": 0.091258904,
      "cost_per_1000_usd": 0.11851805714285714,
      "p50_ms": 105.8962680001514,
      "p95_ms": 183.8574601500113,
      "latency_basis": "observed_wall_clock"
    }
  ],
  "random_controls": {
    "dev": [
      {
        "threshold": 0.5,
        "count": 385,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 16,
        "fallback_count": 16,
        "jev_correct": 341,
        "cascade_correct": 339,
        "corrections": 1,
        "regressions": 3,
        "net_correct_gain": -2,
        "random_expected_net_gain": -0.2077922077922078,
        "random_expected_accuracy": 0.8851745656940463,
        "random_net_gain_central_95": [
          -2,
          1
        ],
        "random_fraction_ge_observed": 0.9937
      },
      {
        "threshold": 0.6,
        "count": 385,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 28,
        "fallback_count": 28,
        "jev_correct": 341,
        "cascade_correct": 340,
        "corrections": 4,
        "regressions": 5,
        "net_correct_gain": -1,
        "random_expected_net_gain": -0.36363636363636365,
        "random_expected_accuracy": 0.8847697756788666,
        "random_net_gain_central_95": [
          -3,
          2
        ],
        "random_fraction_ge_observed": 0.8547
      },
      {
        "threshold": 0.7,
        "count": 385,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 37,
        "fallback_count": 37,
        "jev_correct": 341,
        "cascade_correct": 340,
        "corrections": 5,
        "regressions": 6,
        "net_correct_gain": -1,
        "random_expected_net_gain": -0.4805194805194805,
        "random_expected_accuracy": 0.8844661831674818,
        "random_net_gain_central_95": [
          -3,
          2
        ],
        "random_fraction_ge_observed": 0.802
      },
      {
        "threshold": 0.8,
        "count": 385,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 56,
        "fallback_count": 56,
        "jev_correct": 341,
        "cascade_correct": 340,
        "corrections": 6,
        "regressions": 7,
        "net_correct_gain": -1,
        "random_expected_net_gain": -0.7272727272727273,
        "random_expected_accuracy": 0.8838252656434474,
        "random_net_gain_central_95": [
          -4,
          2
        ],
        "random_fraction_ge_observed": 0.7053
      },
      {
        "threshold": 0.9,
        "count": 385,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 83,
        "fallback_count": 83,
        "jev_correct": 341,
        "cascade_correct": 339,
        "corrections": 7,
        "regressions": 9,
        "net_correct_gain": -2,
        "random_expected_net_gain": -1.077922077922078,
        "random_expected_accuracy": 0.8829144881092933,
        "random_net_gain_central_95": [
          -5,
          2
        ],
        "random_fraction_ge_observed": 0.7918
      },
      {
        "threshold": 0.95,
        "count": 385,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 93,
        "fallback_count": 93,
        "jev_correct": 341,
        "cascade_correct": 338,
        "corrections": 7,
        "regressions": 10,
        "net_correct_gain": -3,
        "random_expected_net_gain": -1.2077922077922079,
        "random_expected_accuracy": 0.8825771630966437,
        "random_net_gain_central_95": [
          -5,
          2
        ],
        "random_fraction_ge_observed": 0.8927
      }
    ],
    "test": [
      {
        "threshold": 0.5,
        "count": 770,
        "matched_on": "fallback_call_count_not_token_cost",
        "seed": 42,
        "trials": 10000,
        "forced_fallback_count": 0,
        "randomized_fallback_count": 14,
        "fallback_count": 14,
        "jev_correct": 674,
        "cascade_correct": 677,
        "corrections": 4,
        "regressions": 1,
        "net_correct_gain": 3,
        "random_expected_net_gain": -0.16363636363636364,
        "random_expected_accuracy": 0.875112160566706,
        "random_net_gain_central_95": [
          -2,
          2
        ],
        "random_fraction_ge_observed": 0.0023
      }
    ]
  }
}