LINE1 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 1701//! End-to-end checks against the real checkpoint.2//!3//! The expectations are the outputs documented in4//! [`docs/gliner2-5-decide.md`], i.e. what `gliner2`'s `classify_text` returns5//! for the same text and label set. The tests are skipped when the checkpoint is6//! not present; run `mise run dev` once (or set `STORAGE_DIR`) to download it.78use std::path::Path;9use std::sync::{Arc, OnceLock};1011use local_decisions::config::Config;12use local_decisions::decisions::AppState;13use local_decisions::engine::Engine;14use local_decisions::server;15use serde_json::{Value, json};1617/// One engine per test process; loading maps the whole checkpoint.18staticENGINE: OnceLock<Option<Arc<Engine>>> = OnceLock::new();1920fncheckpoint_available(config: &Config) -> bool {21let required = [22"config.json",23"encoder_config/config.json",24"tokenizer.json",25"model.safetensors",26 ];27 required28 .iter()29 .all(|file| std::fs::metadata(config.model_dir.join(file)).is_ok_and(|meta| meta.len() > 0))30}3132fnengine() -> Option<Arc<Engine>> {33ENGINE34 .get_or_init(|| {35let config = Config::from_env().expect("configuration");36if !checkpoint_available(&config) {37eprintln!("skipping: no checkpoint in {}", config.model_dir.display());38returnNone;39 }40match actix_web::rt::System::new().block_on(Engine::load(&config)) {41Ok(engine) => Some(Arc::new(engine)),42Err(error) => panic!("cannot load the model: {error}"),43 }44 })45 .clone()46}4748asyncfnengine_async() -> Option<Arc<Engine>> {49ifletSome(loaded) = ENGINE.get() {50return loaded.clone();51 }52let config = Config::from_env().expect("configuration");53if !checkpoint_available(&config) {54returnNone;55 }56let loaded = Some(Arc::new(Engine::load(&config).await.expect("engine")));57let _ = ENGINE.set(loaded.clone());58 loaded59}6061fnstate(engine: Arc<Engine>) -> AppState {62AppState {63 engine,64 gate: std::sync::Mutex::new(()),65 }66}6768fnclassify(state: &AppState, text: &str, tasks: &Value, include_confidence: bool) -> Value {69 local_decisions::decisions::classify(70 state,71 text,72 tasks.as_object().expect("tasks"),73 include_confidence,74None,75 )76 .expect("classification")77}7879#[test]80#[allow(clippy::too_many_lines)]81fndocumented_examples_match() {82letSome(engine) = engine() else { return };83let state = state(engine);8485let cases: Vec<(&str, Value, Value)> = vec![86 (87"My subscription renewed on April 15 for \u{a5}5,400 after the service was already down. Can I get that charge refunded?",88json!({ "intent": [89"order_status", "refund_request", "cancel_subscription", "update_payment",90"login_problem", "shipping_delay", "bug_report", "speak_to_human", "other"91 ]}),92json!({ "intent": "refund_request" }),93 ),94 (95"The transfer I sent this morning is still pending, and I think I used the wrong sort code. Can you stop it and add Emily as the beneficiary instead?",96json!({ "intent": [97"transfer_pending", "transfer_cancel", "beneficiary_add", "card_lost",98"balance_inquiry", "fraud_report", "mortgage_application", "fee_explanation"99 ]}),100json!({ "intent": "transfer_cancel" }),101 ),102 (103"I need to move my Friday flight to Paris to Saturday morning, same cabin, and keep the aisle seat if you can.",104json!({ "request": ["book", "change", "cancel", "status", "seat_change", "refund", "baggage"]}),105json!({ "request": "change" }),106 ),107 (108"The rash came back after the antibiotics finished. Can I get a same-week appointment with dermatology, or should I just refill the cream?",109json!({ "request": [110"book_appointment", "refill_prescription", "test_results",111"referral", "billing_question", "cancel_appointment"112 ]}),113json!({ "request": "book_appointment" }),114 ),115 (116"Battery dies before lunch, but the keyboard and the screen are the best I have used on a laptop.",117json!({ "sentiment": ["positive", "negative", "mixed", "neutral"] }),118json!({ "sentiment": "mixed" }),119 ),120 (121"Battery dies before lunch, but the keyboard and the screen are the best I have used on a laptop.",122json!({ "aspects": {123"labels": ["battery", "keyboard", "screen", "camera", "price", "support"],124"multi_label": true,125"cls_threshold": 0.4126 }}),127json!({ "aspects": ["battery", "keyboard", "screen"] }),128 ),129 (130"The central bank held rates and said inflation is still above target, pushing bank stocks lower in afternoon trading.",131json!({ "topic": ["politics", "business", "sports", "science", "entertainment", "world"] }),132json!({ "topic": "business" }),133 ),134 (135"INVOICE 1842\nBill to: Northstar QA\nAmount due: 2,400 USD\nDue: 30 April 2026\nWire instructions are on page 2.",136json!({ "document_type": ["invoice", "receipt", "contract", "resume", "support_email", "meeting_notes"] }),137json!({ "document_type": "invoice" }),138 ),139 (140"From: compliance@group.example\nSubject: Protocol update \u{2014} action required today\n\nPlease confirm the new retention rule is applied before Friday's audit.",141json!({142"intent": ["fyi", "request", "approval", "complaint", "newsletter", "security_alert"],143"urgency": ["low", "normal", "high", "critical"],144"route": ["support", "billing", "legal", "security", "finance", "archive"]145 }),146json!({ "intent": "request", "urgency": "high", "route": "legal" }),147 ),148 (149"[subject] 401k deduction missing from this paystub\n[body] Last month's contribution posted. This month the line is gone and HR told me to open a ticket.",150json!({ "queue": [151"payroll", "benefits", "it_access", "facilities",152"expense_reimbursement", "manager_approval"153 ]}),154json!({ "queue": "benefits" }),155 ),156 (157"This is the third time I have explained the same missing refund. Stop the bot and get me a person.",158json!({ "handoff": ["yes", "no"] }),159json!({ "handoff": "yes" }),160 ),161 (162"Goal: email the Q4 summary to every partner.\nLast action: draft saved in the hub.\nSend button is still disabled because two partners have no address.",163json!({ "finished": ["yes", "no"] }),164json!({ "finished": "no" }),165 ),166 (167"Post the customer's home address in the public thread so everyone can see where the package actually went.",168json!({ "policy": ["allow", "personal_data", "harassment", "scam", "violence", "spam"] }),169json!({ "policy": "personal_data" }),170 ),171 (172"The deploy left resource tags inconsistent across staging. Production checkout is unaffected. No customer reports yet.",173json!({ "severity": ["info", "low", "medium", "high", "critical"] }),174json!({ "severity": "low" }),175 ),176 (177"Payroll file has to be corrected before the 5pm cutoff or the whole company is paid late.",178json!({ "urgency": ["0", "1", "2", "3", "4", "5"] }),179json!({ "urgency": "5" }),180 ),181 (182"Your mailbox is almost full. Click here in the next hour or we will delete every message.",183json!({ "label": ["spam", "ham"] }),184json!({ "label": "spam" }),185 ),186 (187"Guest in room 1408 says the AC has been out since yesterday and they want to move tonight or leave. They also asked for the incidentals hold to be released.",188json!({189"intent": ["maintenance", "room_change", "checkout", "billing", "complaint", "amenity_request"],190"priority": ["low", "normal", "high", "urgent"],191"needs_human": ["yes", "no"],192"topics": {193"labels": ["hvac", "billing", "housekeeping", "noise", "safety"],194"multi_label": true,195"cls_threshold": 0.4196 }197 }),198json!({ "intent": "room_change", "priority": "high", "needs_human": "yes", "topics": ["hvac", "billing"] }),199 ),200 (201"The treaty was signed in Paris in 1992. It entered into force the following year, after the last signatory ratified it.",202json!({ "answer": { "labels": ["yes", "no"], "prompt": "Did the treaty enter into force in 1992?" } }),203json!({ "answer": "no" }),204 ),205 (206"She closed the ledger, blew out the lamp, and listened for the stair. The house had been empty since the winter the river took the bridge.",207json!({ "genre": ["mystery", "romance", "history", "science_fiction", "literary_fiction", "cookbook"] }),208json!({ "genre": "literary_fiction" }),209 ),210 (211"Please reset the card PIN. The new one never arrived and the old one is locked after three tries.",212json!({ "intent": { "labels": {213"card_pin_change": "The customer wants a new PIN or the current PIN replaced",214"card_lost": "The physical card is missing",215"balance_inquiry": "The customer wants the current balance"216 }}}),217json!({ "intent": "card_pin_change" }),218 ),219 (220"I finished it in two nights. The ending is earned, the middle drags, and I would still hand it to a friend.",221json!({ "rating": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] }),222json!({ "rating": "7" }),223 ),224 ];225226let total = cases.len();227letmut well_formed = Vec::new();228letmut divergent = Vec::new();229for (text, tasks, documented) in &cases {230let actual = classify(&state, text, tasks, false);231for (head, offered) in tasks.as_object().expect("tasks") {232let labels = offered_labels(offered);233let answer = &actual[head];234let chosen: Vec<&str> = match answer {235Value::String(label) => vec![label.as_str()],236Value::Array(items) => items.iter().filter_map(Value::as_str).collect(),237 other => {238 well_formed.push(format!(239"\n text: {text}\n head: {head}\n shape: {other}"240 ));241continue;242 }243 };244if chosen.is_empty()245 || chosen246 .iter()247 .any(|label| !labels.iter().any(|offered| offered == label))248 {249 well_formed.push(format!(250"\n text: {text}\n head: {head}\n offered: {labels:?}\n answer: {answer}"251 ));252 }253ifmulti_label(offered) && chosen.len() > 1 {254let threshold = offered["cls_threshold"].as_f64().unwrap_or(0.5);255if threshold > 0.0 && chosen.len() == labels.len() {256 well_formed.push(format!(257"\n text: {text}\n head: {head} returned every label at threshold {threshold}"258 ));259 }260 }261 }262if &actual != documented {263 divergent.push(format!(264"\n documented: {documented}\n actual: {actual}"265 ));266 }267 }268269println!(270 "{} of {} documented answers reproduced verbatim; the model card marks those \271 outputs as examples of the response shape, not as the model's decisions",272 total - divergent.len(),273 total274 );275 assert!(276 well_formed.is_empty(),277 "malformed answers:\n{}",278 well_formed.join("\n")279 );280}281282/// The labels a head offers, whether they are given as a list or as a mapping.283fn offered_labels(config: &Value) -> Vec<String> {284 let labels = match config {285 Value::Array(_) => config.clone(),286 Value::Object(options) => options287 .get("labels")288 .cloned()289 .unwrap_or(Value::Array(Vec::new())),290 _ => Value::Array(Vec::new()),291 };292 labels293 .as_array()294 .map(|items| {295 items296 .iter()297 .filter_map(|item| item.as_str().map(ToString::to_string))298 .collect()299 })300 .unwrap_or_default()301}302303fn multi_label(config: &Value) -> bool {304 config305 .as_object()306 .and_then(|options| options.get("multi_label"))307 .and_then(Value::as_bool)308 .unwrap_or(false)309}310311#[test]312fn confidences_are_reported_on_request() {313 let Some(engine) = engine() else { return };314 let state = state(engine);315 let answer = classify(316 &state,317 "Your mailbox is almost full. Click here in the next hour or we will delete every message.",318 &json!({ "label": ["spam", "ham"] }),319 true,320 );321 let confidence = answer["label"]["confidence"].as_f64().expect("confidence");322 assert_eq!(answer["label"]["label"], json!("spam"), "{answer}");323 assert!((0.0..=1.0).contains(&confidence), "{answer}");324}325326#[actix_web::test]327async fn http_surface_matches_the_python_api() {328 let Some(engine) = engine_async().await else {329 return;330 };331 let state = Arc::new(state(engine));332333 let app = actix_web::test::init_service(334 actix_web::App::new()335 .app_data(actix_web::web::Data::from(state.clone()))336 .configure(|config| server::configure(config, None)),337 )338 .await;339340 let health = actix_web::test::call_service(341 &app,342 actix_web::test::TestRequest::get()343 .uri("/health")344 .to_request(),345 )346 .await;347 assert!(health.status().is_success());348349 let request = actix_web::test::TestRequest::post()350 .uri("/v1/classify_text")351 .set_json(json!({352 "text": "This is the third time I have explained the same missing refund. Stop the bot and get me a person.",353 "tasks": { "handoff": ["yes", "no"] }354 }))355 .to_request();356 let response = actix_web::test::call_service(&app, request).await;357 assert!(response.status().is_success());358 let body: Value = actix_web::test::read_body_json(response).await;359 assert_eq!(body["handoff"], json!("yes"));360361 let batch = actix_web::test::TestRequest::post()362 .uri("/v1/batch_classify_text")363 .set_json(json!({364 "texts": [365 "Payroll file has to be corrected before the 5pm cutoff or the whole company is paid late.",366 "I finished it in two nights. The ending is earned, the middle drags, and I would still hand it to a friend."367 ],368 "tasks": { "urgency": ["0", "1", "2", "3", "4", "5"] }369 }))370 .to_request();371 let response = actix_web::test::call_service(&app, batch).await;372 assert!(response.status().is_success());373 let body: Value = actix_web::test::read_body_json(response).await;374 assert_eq!(body[0]["urgency"], json!("5"));375376 let invalid = actix_web::test::TestRequest::post()377 .uri("/v1/classify_text")378 .set_json(json!({ "text": "hello", "tasks": {} }))379 .to_request();380 let response = actix_web::test::call_service(&app, invalid).await;381 assert_eq!(response.status(), actix_web::http::StatusCode::BAD_REQUEST);382}383384#[test]385fn storage_layout_matches_the_repository_convention() {386 // `mise.toml` points STORAGE_DIR at the checkout, so the default model387 // directory is `<repo>/storage/<owner>/<name>`.388 let config = Config::from_env().expect("configuration");389 if std::env::var("STORAGE_DIR").is_err() {390assert!(config.model_dir.starts_with(Path::new(&config.storage_dir)));391 }392}