Skip to content
1        10        20        30        40        50        60        70        80        90        100       110       120       130       140       150       160       1701//! End-to-end checks against the real checkpoint.2//!3//! The expectations are the outputs documented in4//! [`docs/gliner2-5-decide.md`], i.e. what `gliner2`'s `classify_text` returns5//! for the same text and label set. The tests are skipped when the checkpoint is6//! not present; run `mise run dev` once (or set `STORAGE_DIR`) to download it.78use std::path::Path;9use std::sync::{Arc, OnceLock};1011use local_decisions::config::Config;12use local_decisions::decisions::AppState;13use local_decisions::engine::Engine;14use local_decisions::server;15use serde_json::{Value, json};1617/// One engine per test process; loading maps the whole checkpoint.18static ENGINE: OnceLock<Option<Arc<Engine>>> = OnceLock::new();1920fn checkpoint_available(config: &Config) -> bool {21    let required = [22        "config.json",23        "encoder_config/config.json",24        "tokenizer.json",25        "model.safetensors",26    ];27    required28        .iter()29        .all(|file| std::fs::metadata(config.model_dir.join(file)).is_ok_and(|meta| meta.len() > 0))30}3132fn engine() -> Option<Arc<Engine>> {33    ENGINE34        .get_or_init(|| {35            let config = Config::from_env().expect("configuration");36            if !checkpoint_available(&config) {37                eprintln!("skipping: no checkpoint in {}", config.model_dir.display());38                return None;39            }40            match actix_web::rt::System::new().block_on(Engine::load(&config)) {41                Ok(engine) => Some(Arc::new(engine)),42                Err(error) => panic!("cannot load the model: {error}"),43            }44        })45        .clone()46}4748async fn engine_async() -> Option<Arc<Engine>> {49    if let Some(loaded) = ENGINE.get() {50        return loaded.clone();51    }52    let config = Config::from_env().expect("configuration");53    if !checkpoint_available(&config) {54        return None;55    }56    let loaded = Some(Arc::new(Engine::load(&config).await.expect("engine")));57    let _ = ENGINE.set(loaded.clone());58    loaded59}6061fn state(engine: Arc<Engine>) -> AppState {62    AppState {63        engine,64        gate: std::sync::Mutex::new(()),65    }66}6768fn classify(state: &AppState, text: &str, tasks: &Value, include_confidence: bool) -> Value {69    local_decisions::decisions::classify(70        state,71        text,72        tasks.as_object().expect("tasks"),73        include_confidence,74        None,75    )76    .expect("classification")77}7879#[test]80#[allow(clippy::too_many_lines)]81fn documented_examples_match() {82    let Some(engine) = engine() else { return };83    let state = state(engine);8485    let cases: Vec<(&str, Value, Value)> = vec![86        (87            "My subscription renewed on April 15 for \u{a5}5,400 after the service was already down. Can I get that charge refunded?",88            json!({ "intent": [89                "order_status", "refund_request", "cancel_subscription", "update_payment",90                "login_problem", "shipping_delay", "bug_report", "speak_to_human", "other"91            ]}),92            json!({ "intent": "refund_request" }),93        ),94        (95            "The transfer I sent this morning is still pending, and I think I used the wrong sort code. Can you stop it and add Emily as the beneficiary instead?",96            json!({ "intent": [97                "transfer_pending", "transfer_cancel", "beneficiary_add", "card_lost",98                "balance_inquiry", "fraud_report", "mortgage_application", "fee_explanation"99            ]}),100            json!({ "intent": "transfer_cancel" }),101        ),102        (103            "I need to move my Friday flight to Paris to Saturday morning, same cabin, and keep the aisle seat if you can.",104            json!({ "request": ["book", "change", "cancel", "status", "seat_change", "refund", "baggage"]}),105            json!({ "request": "change" }),106        ),107        (108            "The rash came back after the antibiotics finished. Can I get a same-week appointment with dermatology, or should I just refill the cream?",109            json!({ "request": [110                "book_appointment", "refill_prescription", "test_results",111                "referral", "billing_question", "cancel_appointment"112            ]}),113            json!({ "request": "book_appointment" }),114        ),115        (116            "Battery dies before lunch, but the keyboard and the screen are the best I have used on a laptop.",117            json!({ "sentiment": ["positive", "negative", "mixed", "neutral"] }),118            json!({ "sentiment": "mixed" }),119        ),120        (121            "Battery dies before lunch, but the keyboard and the screen are the best I have used on a laptop.",122            json!({ "aspects": {123                "labels": ["battery", "keyboard", "screen", "camera", "price", "support"],124                "multi_label": true,125                "cls_threshold": 0.4126            }}),127            json!({ "aspects": ["battery", "keyboard", "screen"] }),128        ),129        (130            "The central bank held rates and said inflation is still above target, pushing bank stocks lower in afternoon trading.",131            json!({ "topic": ["politics", "business", "sports", "science", "entertainment", "world"] }),132            json!({ "topic": "business" }),133        ),134        (135            "INVOICE 1842\nBill to: Northstar QA\nAmount due: 2,400 USD\nDue: 30 April 2026\nWire instructions are on page 2.",136            json!({ "document_type": ["invoice", "receipt", "contract", "resume", "support_email", "meeting_notes"] }),137            json!({ "document_type": "invoice" }),138        ),139        (140            "From: compliance@group.example\nSubject: Protocol update \u{2014} action required today\n\nPlease confirm the new retention rule is applied before Friday's audit.",141            json!({142                "intent": ["fyi", "request", "approval", "complaint", "newsletter", "security_alert"],143                "urgency": ["low", "normal", "high", "critical"],144                "route": ["support", "billing", "legal", "security", "finance", "archive"]145            }),146            json!({ "intent": "request", "urgency": "high", "route": "legal" }),147        ),148        (149            "[subject] 401k deduction missing from this paystub\n[body] Last month's contribution posted. This month the line is gone and HR told me to open a ticket.",150            json!({ "queue": [151                "payroll", "benefits", "it_access", "facilities",152                "expense_reimbursement", "manager_approval"153            ]}),154            json!({ "queue": "benefits" }),155        ),156        (157            "This is the third time I have explained the same missing refund. Stop the bot and get me a person.",158            json!({ "handoff": ["yes", "no"] }),159            json!({ "handoff": "yes" }),160        ),161        (162            "Goal: email the Q4 summary to every partner.\nLast action: draft saved in the hub.\nSend button is still disabled because two partners have no address.",163            json!({ "finished": ["yes", "no"] }),164            json!({ "finished": "no" }),165        ),166        (167            "Post the customer's home address in the public thread so everyone can see where the package actually went.",168            json!({ "policy": ["allow", "personal_data", "harassment", "scam", "violence", "spam"] }),169            json!({ "policy": "personal_data" }),170        ),171        (172            "The deploy left resource tags inconsistent across staging. Production checkout is unaffected. No customer reports yet.",173            json!({ "severity": ["info", "low", "medium", "high", "critical"] }),174            json!({ "severity": "low" }),175        ),176        (177            "Payroll file has to be corrected before the 5pm cutoff or the whole company is paid late.",178            json!({ "urgency": ["0", "1", "2", "3", "4", "5"] }),179            json!({ "urgency": "5" }),180        ),181        (182            "Your mailbox is almost full. Click here in the next hour or we will delete every message.",183            json!({ "label": ["spam", "ham"] }),184            json!({ "label": "spam" }),185        ),186        (187            "Guest in room 1408 says the AC has been out since yesterday and they want to move tonight or leave. They also asked for the incidentals hold to be released.",188            json!({189                "intent": ["maintenance", "room_change", "checkout", "billing", "complaint", "amenity_request"],190                "priority": ["low", "normal", "high", "urgent"],191                "needs_human": ["yes", "no"],192                "topics": {193                    "labels": ["hvac", "billing", "housekeeping", "noise", "safety"],194                    "multi_label": true,195                    "cls_threshold": 0.4196                }197            }),198            json!({ "intent": "room_change", "priority": "high", "needs_human": "yes", "topics": ["hvac", "billing"] }),199        ),200        (201            "The treaty was signed in Paris in 1992. It entered into force the following year, after the last signatory ratified it.",202            json!({ "answer": { "labels": ["yes", "no"], "prompt": "Did the treaty enter into force in 1992?" } }),203            json!({ "answer": "no" }),204        ),205        (206            "She closed the ledger, blew out the lamp, and listened for the stair. The house had been empty since the winter the river took the bridge.",207            json!({ "genre": ["mystery", "romance", "history", "science_fiction", "literary_fiction", "cookbook"] }),208            json!({ "genre": "literary_fiction" }),209        ),210        (211            "Please reset the card PIN. The new one never arrived and the old one is locked after three tries.",212            json!({ "intent": { "labels": {213                "card_pin_change": "The customer wants a new PIN or the current PIN replaced",214                "card_lost": "The physical card is missing",215                "balance_inquiry": "The customer wants the current balance"216            }}}),217            json!({ "intent": "card_pin_change" }),218        ),219        (220            "I finished it in two nights. The ending is earned, the middle drags, and I would still hand it to a friend.",221            json!({ "rating": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] }),222            json!({ "rating": "7" }),223        ),224    ];225226    let total = cases.len();227    let mut well_formed = Vec::new();228    let mut divergent = Vec::new();229    for (text, tasks, documented) in &cases {230        let actual = classify(&state, text, tasks, false);231        for (head, offered) in tasks.as_object().expect("tasks") {232            let labels = offered_labels(offered);233            let answer = &actual[head];234            let chosen: Vec<&str> = match answer {235                Value::String(label) => vec![label.as_str()],236                Value::Array(items) => items.iter().filter_map(Value::as_str).collect(),237                other => {238                    well_formed.push(format!(239                        "\n  text:  {text}\n  head:  {head}\n  shape: {other}"240                    ));241                    continue;242                }243            };244            if chosen.is_empty()245                || chosen246                    .iter()247                    .any(|label| !labels.iter().any(|offered| offered == label))248            {249                well_formed.push(format!(250                    "\n  text:    {text}\n  head:    {head}\n  offered: {labels:?}\n  answer:  {answer}"251                ));252            }253            if multi_label(offered) && chosen.len() > 1 {254                let threshold = offered["cls_threshold"].as_f64().unwrap_or(0.5);255                if threshold > 0.0 && chosen.len() == labels.len() {256                    well_formed.push(format!(257                        "\n  text: {text}\n  head: {head} returned every label at threshold {threshold}"258                    ));259                }260            }261        }262        if &actual != documented {263            divergent.push(format!(264                "\n  documented: {documented}\n  actual:     {actual}"265            ));266        }267    }268269    println!(270        "{} of {} documented answers reproduced verbatim; the model card marks those \271         outputs as examples of the response shape, not as the model's decisions",272        total - divergent.len(),273        total274    );275    assert!(276        well_formed.is_empty(),277        "malformed answers:\n{}",278        well_formed.join("\n")279    );280}281282/// The labels a head offers, whether they are given as a list or as a mapping.283fn offered_labels(config: &Value) -> Vec<String> {284    let labels = match config {285        Value::Array(_) => config.clone(),286        Value::Object(options) => options287            .get("labels")288            .cloned()289            .unwrap_or(Value::Array(Vec::new())),290        _ => Value::Array(Vec::new()),291    };292    labels293        .as_array()294        .map(|items| {295            items296                .iter()297                .filter_map(|item| item.as_str().map(ToString::to_string))298                .collect()299        })300        .unwrap_or_default()301}302303fn multi_label(config: &Value) -> bool {304    config305        .as_object()306        .and_then(|options| options.get("multi_label"))307        .and_then(Value::as_bool)308        .unwrap_or(false)309}310311#[test]312fn confidences_are_reported_on_request() {313    let Some(engine) = engine() else { return };314    let state = state(engine);315    let answer = classify(316        &state,317        "Your mailbox is almost full. Click here in the next hour or we will delete every message.",318        &json!({ "label": ["spam", "ham"] }),319        true,320    );321    let confidence = answer["label"]["confidence"].as_f64().expect("confidence");322    assert_eq!(answer["label"]["label"], json!("spam"), "{answer}");323    assert!((0.0..=1.0).contains(&confidence), "{answer}");324}325326#[actix_web::test]327async fn http_surface_matches_the_python_api() {328    let Some(engine) = engine_async().await else {329        return;330    };331    let state = Arc::new(state(engine));332333    let app = actix_web::test::init_service(334        actix_web::App::new()335            .app_data(actix_web::web::Data::from(state.clone()))336            .configure(|config| server::configure(config, None)),337    )338    .await;339340    let health = actix_web::test::call_service(341        &app,342        actix_web::test::TestRequest::get()343            .uri("/health")344            .to_request(),345    )346    .await;347    assert!(health.status().is_success());348349    let request = actix_web::test::TestRequest::post()350        .uri("/v1/classify_text")351        .set_json(json!({352            "text": "This is the third time I have explained the same missing refund. Stop the bot and get me a person.",353            "tasks": { "handoff": ["yes", "no"] }354        }))355        .to_request();356    let response = actix_web::test::call_service(&app, request).await;357    assert!(response.status().is_success());358    let body: Value = actix_web::test::read_body_json(response).await;359    assert_eq!(body["handoff"], json!("yes"));360361    let batch = actix_web::test::TestRequest::post()362        .uri("/v1/batch_classify_text")363        .set_json(json!({364            "texts": [365                "Payroll file has to be corrected before the 5pm cutoff or the whole company is paid late.",366                "I finished it in two nights. The ending is earned, the middle drags, and I would still hand it to a friend."367            ],368            "tasks": { "urgency": ["0", "1", "2", "3", "4", "5"] }369        }))370        .to_request();371    let response = actix_web::test::call_service(&app, batch).await;372    assert!(response.status().is_success());373    let body: Value = actix_web::test::read_body_json(response).await;374    assert_eq!(body[0]["urgency"], json!("5"));375376    let invalid = actix_web::test::TestRequest::post()377        .uri("/v1/classify_text")378        .set_json(json!({ "text": "hello", "tasks": {} }))379        .to_request();380    let response = actix_web::test::call_service(&app, invalid).await;381    assert_eq!(response.status(), actix_web::http::StatusCode::BAD_REQUEST);382}383384#[test]385fn storage_layout_matches_the_repository_convention() {386    // `mise.toml` points STORAGE_DIR at the checkout, so the default model387    // directory is `<repo>/storage/<owner>/<name>`.388    let config = Config::from_env().expect("configuration");389    if std::env::var("STORAGE_DIR").is_err() {390        assert!(config.model_dir.starts_with(Path::new(&config.storage_dir)));391    }392}