LINE1 10 20 30 40 50 60 70 80 90 100123456789101112131415161718#![allow(clippy::cast_precision_loss, clippy::format_push_string)]1920use std::path::PathBuf;2122use local_decisions::config::Config;23use local_decisions::decisions::{self, AppState};24use local_decisions::engine::Engine;25use serde_json::{Value, json};26use std::sync::Arc;2728const DOMAINS: [&str; 17] = [29 "agent_handoff",30 "banking_intent",31 "benefits_request",32 "clinic_request",33 "document_type",34 "email_triage",35 "news_topic",36 "paper_field",37 "product_feedback",38 "restaurant_review",39 "review_sentiment",40 "screen_tags",41 "sports_recap",42 "support_intent",43 "support_topic",44 "ticket_route",45 "travel_request",46];474849const PUBLISHED: f64 = 60.2;5051fn benchmark_dir() -> PathBuf {52 match std::env::var("BENCHMARK_DIR") {53 Ok(dir) => PathBuf::from(dir),54 Err(_) => Config::from_env()55 .expect("config")56 .model_dir57 .parent()58 .map_or_else(59 || PathBuf::from("."),60 |parent| parent.join("fastino/fast-decisions"),61 ),62 }63}6465#[test]66#[ignore = "needs the fast-decisions dataset"]67fn exact_match_accuracy_matches_the_model_card() {68 let config = Config::from_env().expect("config");69 let engine = Arc::new(70 actix_web::rt::System::new()71 .block_on(Engine::load(&config))72 .expect("engine"),73 );74 let state = AppState {75 engine,76 gate: std::sync::Mutex::new(()),77 };78 let directory = benchmark_dir();7980 let mut total_correct = 0usize;81 let mut total_rows = 0usize;82 let mut per_domain = Vec::new();8384 for domain in DOMAINS {85 let path = directory.join(format!("{domain}.jsonl"));86 let Ok(raw) = std::fs::read_to_string(&path) else {87 eprintln!("skipping {domain}: {} not found", path.display());88 continue;89 };9091 let mut correct = 0usize;92 let mut rows = 0usize;93 for line in raw.lines().filter(|line| !line.trim().is_empty()) {94 let row: Value = serde_json::from_str(line).expect("jsonl row");95 let classifications = row["output"]["classifications"].as_array().expect("heads");9697 let mut tasks = serde_json::Map::new();98 let mut expected: Vec<(String, Vec<String>)> = Vec::new();99 for head in classifications {100 let task = head["task"].as_str().expect("task").to_owned();101 let labels: Vec<&str> = head["labels"]102 .as_array()103 .expect("labels")104 .iter()105 .filter_map(Value::as_str)106 .collect();107 let multi_label = head["multi_label"].as_bool().unwrap_or(false);108 let threshold = head["cls_threshold"].as_f64().unwrap_or(0.5);109 let gold: Vec<String> = head["true_label"]110 .as_array()111 .expect("true_label")112 .iter()113 .filter_map(|value| value.as_str().map(ToString::to_string))114 .collect();115116 tasks.insert(117 task.clone(),118 json!({119 "labels": labels,120 "multi_label": multi_label,121 "cls_threshold": threshold122 }),123 );124 expected.push((task, gold));125 }126127 let answer = decisions::classify(128 &state,129 row["input"].as_str().expect("input"),130 &tasks,131 false,132 None,133 )134 .expect("classification");135136 let mut hits = 0usize;137 for (task, gold) in &expected {138 if predicted(&answer[task]) == gold.iter().map(String::as_str).collect::<Vec<_>>() {139 hits += 1;140 }141 }142 if hits == expected.len() {143 correct += 1;144 }145 rows += 1;146 }147148 let accuracy = 100.0 * correct as f64 / rows.max(1) as f64;149 per_domain.push((domain, correct, rows, accuracy));150 total_correct += correct;151 total_rows += rows;152 }153154 let mut report = String::new();155 for (domain, correct, rows, accuracy) in &per_domain {156 report.push_str(&format!(157 "{domain:<20} {accuracy:>5.1}% ({correct}/{rows})\n"158 ));159 }160 let average = 100.0 * total_correct as f64 / total_rows.max(1) as f64;161 report.push_str(&format!(162 "\naverage over {total_rows} rows: {average:.1}% (model card: {PUBLISHED:.1}%)\n"163 ));164 println!("{report}");165166 167 168 assert!(169 (average - PUBLISHED).abs() < 5.0,170 "average exact match {average:.1}% is not close to the published {PUBLISHED:.1}%"171 );172}173174175fn predicted(value: &Value) -> Vec<&str> {176 match value {177 Value::String(label) => vec![label.as_str()],178 Value::Array(items) => items.iter().filter_map(Value::as_str).collect(),179 _ => Vec::new(),180 }181}