Skip to content
1        10        20        30        40        50        60        70        80        901//! The inference engine: the `fastino/GLiNER2.5-Decide` checkpoint, running2//! locally in this process.3//!4//! The engine owns the pieces that mirror `gliner2`: the word splitter, the5//! checkpoint tokenizer, a DeBERTa-v3 encoder, `GLiNER`'s classification head6//! and its span head. Nothing leaves the machine and no Python runtime is7//! involved.89pub mod deberta;10pub mod download;11pub mod extract;12pub mod schema;13pub mod span;14pub mod tokenizer;1516use std::path::Path;1718use candle_core::{Device, Tensor};19use candle_nn::VarBuilder;2021use crate::config::Config;2223use self::deberta::{ClassifierHead, Encoder};24use self::schema::{25    ClassificationTask, Prepared, Query, Structure, classify_scores, normalize_text, prepare,26};27use self::span::SpanHead;28use self::tokenizer::{Tokenizer, WordSplitter};2930/// Answers for every head of one request.31pub type TaskAnswer = Vec<Vec<(String, f64)>>;3233/// How much detail an extraction answer carries.34#[derive(Debug, Clone, Copy)]35pub struct ExtractOptions {36    pub threshold: f64,37    pub include_confidence: bool,38    pub include_spans: bool,39}4041impl Default for ExtractOptions {42    fn default() -> Self {43        Self {44            threshold: 0.5,45            include_confidence: false,46            include_spans: false,47        }48    }49}5051/// A loaded checkpoint, ready to classify text and to extract values from it.52pub struct Engine {53    tokenizer: Tokenizer,54    splitter: WordSplitter,55    encoder: Encoder,56    head: ClassifierHead,57    span: SpanHead,58    max_width: usize,59}6061impl Engine {62    /// Download whatever is missing, then load the checkpoint.63    ///64    /// # Errors65    /// Returns a message when a file cannot be fetched or the weights cannot be66    /// mapped into a usable encoder.67    pub async fn load(config: &Config) -> Result<Self, String> {68        download::ensure(69            &config.model_dir,70            &config.model_repo,71            &config.hf_endpoint,72            config.hf_token.as_deref(),73        )74        .await?;7576        let tokenizer = Tokenizer::from_file(&config.model_dir.join("tokenizer.json"))?;77        let splitter = WordSplitter::new()?;78        let encoder_config =79            deberta::read_config(&config.model_dir.join("encoder_config/config.json"))?;80        let (encoder, head) =81            deberta::load(&encoder_config, &weights(&config.model_dir), &Device::Cpu)82                .map_err(|error| format!("cannot load the checkpoint: {error}"))?;8384        let device = Device::Cpu;85        let weights_path = weights(&config.model_dir);86        let max_width = max_width(&config.model_dir)?;87        // The span heads live in the same file; a second mapping keeps the88        // classification path from holding on to tensors it never reads.89        let span_vb = unsafe {90            VarBuilder::from_mmaped_safetensors(&[&weights_path], candle_core::DType::F32, &device)91        }92        .map_err(|error| format!("cannot open {}: {error}", weights_path.display()))?;93        let span = SpanHead::load(&span_vb, encoder_config.hidden_size, max_width, &device)94            .map_err(|error| format!("cannot load the span head: {error}"))?;9596        Ok(Self {97            tokenizer,98            splitter,99            encoder,100            head,101            span,102            max_width,103        })104    }105106    /// An engine that answers nothing, for tests that never reach inference.107    ///108    /// The HTTP auth tests build an `AppState` without a checkpoint: their109    /// requests are rejected by the key check or the JSON extractor before110    /// the model is touched, so every field can hold an empty placeholder.111    #[cfg(test)]112    pub(crate) fn for_tests() -> Self {113        Self {114            tokenizer: Tokenizer::empty(),115            splitter: WordSplitter::empty(),116            encoder: Encoder::empty(),117            head: ClassifierHead::empty(),118            span: SpanHead::empty(),119            max_width: 0,120        }121    }122123    /// Classify `text` against every head, mirroring `model.classify_text`.124    ///125    /// # Errors126    /// Returns a message when the text cannot be tokenized or the forward pass127    /// fails.128    pub fn classify(129        &self,130        text: &str,131        tasks: &[ClassificationTask],132        max_len: Option<usize>,133    ) -> Result<TaskAnswer, String> {134        let queries: Vec<Query> = tasks.iter().map(Query::classification).collect();135        let prepared = self.prepare(text, &queries, max_len)?;136        let hidden = self.encode(&prepared)?;137        let logits = self.label_logits(&hidden, &prepared)?;138        Self::split_logits(&logits, tasks)139    }140141    /// Extract one value per property, mirroring `model.extract_json`.142    ///143    /// # Errors144    /// Returns a message when the text cannot be tokenized or scoring fails.145    pub fn extract(146        &self,147        text: &str,148        structures: &[Structure],149        options: ExtractOptions,150        max_len: Option<usize>,151    ) -> Result<Vec<Vec<serde_json::Value>>, String> {152        if structures.is_empty() {153            return Err("at least one structure is required".to_owned());154        }155        let queries: Vec<Query> = structures.iter().map(Query::structure).collect();156        let normalized = normalize_text(text);157        let prepared = prepare(158            &self.tokenizer,159            &self.splitter,160            &normalized,161            &queries,162            max_len,163        )?;164        let hidden = self.encode(&prepared)?;165166        let mut results = Vec::with_capacity(structures.len());167        for (index, structure) in structures.iter().enumerate() {168            results.push(self.extract_one(169                structure,170                &hidden,171                &prepared,172                index,173                &normalized,174                options,175            )?);176        }177        Ok(results)178    }179180    /// One structure's instances, decoded from its own span scores.181    fn extract_one(182        &self,183        structure: &Structure,184        hidden: &Tensor,185        prepared: &Prepared,186        index: usize,187        text: &str,188        options: ExtractOptions,189    ) -> Result<Vec<serde_json::Value>, String> {190        let markers = prepared191            .marker_positions192            .get(index)193            .ok_or_else(|| format!("structure '{}' was not encoded", structure.name))?;194        let device = hidden.device();195        let gather = |positions: &[usize]| -> Result<Tensor, String> {196            let indices: Vec<u32> = positions197                .iter()198                .map(|position| u32::try_from(*position).map_err(|error| error.to_string()))199                .collect::<Result<_, _>>()?;200            Tensor::new(indices, device)201                .and_then(|index| hidden.index_select(&index, 0))202                .map_err(|error| format!("cannot gather marker states: {error}"))203        };204205        let query = gather(&markers[..1])?;206        let fields = gather(markers[1..].to_vec().as_slice())?;207        let words = gather(&prepared.word_positions)?;208209        let scores = self210            .span211            .score(&words, &query, &fields)212            .map_err(|error| format!("span scoring failed: {error}"))?;213        let dimensions = scores214            .dims4()215            .map_err(|error| format!("bad score shape: {error}"))?;216        let (instances, field_count, word_count, width_count) = dimensions;217        let flat = scores218            .flatten_all()219            .and_then(|scores| scores.to_vec1::<f32>())220            .map_err(|error| format!("cannot read scores: {error}"))?;221222        Ok(extract::decode(223            structure,224            &flat,225            extract::ScoreShape {226                instances,227                fields: field_count,228                words: word_count,229                widths: width_count,230            },231            &prepared.word_offsets,232            text,233            extract::Detail {234                threshold: options.threshold,235                confidence: options.include_confidence,236                spans: options.include_spans,237            },238        ))239    }240241    fn encode(&self, prepared: &Prepared) -> Result<Tensor, String> {242        self.encoder243            .forward(&prepared.input_ids)244            .map_err(|error| format!("encoder failed: {error}"))245    }246247    fn prepare(248        &self,249        text: &str,250        queries: &[Query],251        max_len: Option<usize>,252    ) -> Result<Prepared, String> {253        prepare(254            &self.tokenizer,255            &self.splitter,256            &normalize_text(text),257            queries,258            max_len,259        )260    }261262    /// Score the child marker of every query in one pass; `[P]` is dropped.263    fn label_logits(&self, hidden: &Tensor, prepared: &Prepared) -> Result<Vec<f32>, String> {264        let mut positions: Vec<u32> = Vec::new();265        for markers in &prepared.marker_positions {266            for position in markers.iter().skip(1) {267                positions.push(u32::try_from(*position).map_err(|error| error.to_string())?);268            }269        }270        let index = Tensor::new(positions, hidden.device())271            .map_err(|error| format!("cannot index markers: {error}"))?;272        let embeddings = hidden273            .index_select(&index, 0)274            .map_err(|error| format!("cannot gather marker states: {error}"))?;275        self.head276            .forward(&embeddings)277            .map_err(|error| format!("classifier failed: {error}"))278    }279280    fn split_logits(logits: &[f32], tasks: &[ClassificationTask]) -> Result<TaskAnswer, String> {281        let mut offset = 0;282        let mut answers = Vec::with_capacity(tasks.len());283        for task in tasks {284            let count = task.labels.len();285            let slice = logits.get(offset..offset + count).ok_or_else(|| {286                format!(287                    "model returned {} logits for {} tasks",288                    logits.len(),289                    tasks.len()290                )291            })?;292            answers.push(classify_scores(task, slice));293            offset += count;294        }295        Ok(answers)296    }297298    /// Widest span the checkpoint's span marker can represent.299    pub fn max_width(&self) -> usize {300        self.max_width301    }302}303304fn weights(model_dir: &Path) -> std::path::PathBuf {305    model_dir.join("model.safetensors")306}307308/// `max_width` lives in the extractor's own `config.json`.309fn max_width(model_dir: &Path) -> Result<usize, String> {310    let raw = std::fs::read_to_string(model_dir.join("config.json"))311        .map_err(|error| format!("cannot read the model config: {error}"))?;312    let config: serde_json::Value = serde_json::from_str(&raw)313        .map_err(|error| format!("cannot parse the model config: {error}"))?;314    let max_width = config315        .get("max_width")316        .or_else(|| config.pointer("/span_head/max_width"))317        .and_then(serde_json::Value::as_u64)318        .unwrap_or(8);319    usize::try_from(max_width).map_err(|error| format!("max_width is too large: {error}"))320}