Skip to content
1        10        20        30        40        50        60        70        80        90        1001//! Exact-match accuracy on `fastino/fast-decisions`.2//!3//! The model card reports **60.2%** average exact match over the 17 domains of4//! `fastino/fast-decisions`; this harness reproduces that number with the Rust5//! engine, which is the strongest available check that the port is faithful.6//!7//! It is ignored by default because it needs the dataset next to the8//! checkpoint:9//!10//! ```sh11//! curl -sL -o storage/fastino/fast-decisions/support_intent.jsonl \12//!   https://huggingface.co/datasets/fastino/fast-decisions/resolve/main/support_intent.jsonl13//! cargo test --release --test benchmark -- --ignored --nocapture14//! ```15//!16//! Set `BENCHMARK_DIR` to read the JSONL files from somewhere else.1718#![allow(clippy::cast_precision_loss, clippy::format_push_string)]1920use std::path::PathBuf;2122use local_decisions::config::Config;23use local_decisions::decisions::{self, AppState};24use local_decisions::engine::Engine;25use serde_json::{Value, json};26use std::sync::Arc;2728const DOMAINS: [&str; 17] = [29    "agent_handoff",30    "banking_intent",31    "benefits_request",32    "clinic_request",33    "document_type",34    "email_triage",35    "news_topic",36    "paper_field",37    "product_feedback",38    "restaurant_review",39    "review_sentiment",40    "screen_tags",41    "sports_recap",42    "support_intent",43    "support_topic",44    "ticket_route",45    "travel_request",46];4748/// The accuracy the model card reports for this checkpoint.49const PUBLISHED: f64 = 60.2;5051fn benchmark_dir() -> PathBuf {52    match std::env::var("BENCHMARK_DIR") {53        Ok(dir) => PathBuf::from(dir),54        Err(_) => Config::from_env()55            .expect("config")56            .model_dir57            .parent()58            .map_or_else(59                || PathBuf::from("."),60                |parent| parent.join("fastino/fast-decisions"),61            ),62    }63}6465#[test]66#[ignore = "needs the fast-decisions dataset"]67fn exact_match_accuracy_matches_the_model_card() {68    let config = Config::from_env().expect("config");69    let engine = Arc::new(70        actix_web::rt::System::new()71            .block_on(Engine::load(&config))72            .expect("engine"),73    );74    let state = AppState {75        engine,76        gate: std::sync::Mutex::new(()),77    };78    let directory = benchmark_dir();7980    let mut total_correct = 0usize;81    let mut total_rows = 0usize;82    let mut per_domain = Vec::new();8384    for domain in DOMAINS {85        let path = directory.join(format!("{domain}.jsonl"));86        let Ok(raw) = std::fs::read_to_string(&path) else {87            eprintln!("skipping {domain}: {} not found", path.display());88            continue;89        };9091        let mut correct = 0usize;92        let mut rows = 0usize;93        for line in raw.lines().filter(|line| !line.trim().is_empty()) {94            let row: Value = serde_json::from_str(line).expect("jsonl row");95            let classifications = row["output"]["classifications"].as_array().expect("heads");9697            let mut tasks = serde_json::Map::new();98            let mut expected: Vec<(String, Vec<String>)> = Vec::new();99            for head in classifications {100                let task = head["task"].as_str().expect("task").to_owned();101                let labels: Vec<&str> = head["labels"]102                    .as_array()103                    .expect("labels")104                    .iter()105                    .filter_map(Value::as_str)106                    .collect();107                let multi_label = head["multi_label"].as_bool().unwrap_or(false);108                let threshold = head["cls_threshold"].as_f64().unwrap_or(0.5);109                let gold: Vec<String> = head["true_label"]110                    .as_array()111                    .expect("true_label")112                    .iter()113                    .filter_map(|value| value.as_str().map(ToString::to_string))114                    .collect();115116                tasks.insert(117                    task.clone(),118                    json!({119                        "labels": labels,120                        "multi_label": multi_label,121                        "cls_threshold": threshold122                    }),123                );124                expected.push((task, gold));125            }126127            let answer = decisions::classify(128                &state,129                row["input"].as_str().expect("input"),130                &tasks,131                false,132                None,133            )134            .expect("classification");135136            let mut hits = 0usize;137            for (task, gold) in &expected {138                if predicted(&answer[task]) == gold.iter().map(String::as_str).collect::<Vec<_>>() {139                    hits += 1;140                }141            }142            if hits == expected.len() {143                correct += 1;144            }145            rows += 1;146        }147148        let accuracy = 100.0 * correct as f64 / rows.max(1) as f64;149        per_domain.push((domain, correct, rows, accuracy));150        total_correct += correct;151        total_rows += rows;152    }153154    let mut report = String::new();155    for (domain, correct, rows, accuracy) in &per_domain {156        report.push_str(&format!(157            "{domain:<20} {accuracy:>5.1}%  ({correct}/{rows})\n"158        ));159    }160    let average = 100.0 * total_correct as f64 / total_rows.max(1) as f64;161    report.push_str(&format!(162        "\naverage over {total_rows} rows: {average:.1}% (model card: {PUBLISHED:.1}%)\n"163    ));164    println!("{report}");165166    // The published average is a benchmark, not a guarantee: allow a couple of167    // points of drift, but a broken port lands far below this band.168    assert!(169        (average - PUBLISHED).abs() < 5.0,170        "average exact match {average:.1}% is not close to the published {PUBLISHED:.1}%"171    );172}173174/// Read the labels out of a formatted answer, which is a string or a list.175fn predicted(value: &Value) -> Vec<&str> {176    match value {177        Value::String(label) => vec![label.as_str()],178        Value::Array(items) => items.iter().filter_map(Value::as_str).collect(),179        _ => Vec::new(),180    }181}