Skip to content
1        10        20        30        40        50        60        70        80        901//! Schema encoding, a direct port of `gliner2.processor.SchemaTransformer`.2//!3//! For every schema the Python side builds the token sequence4//! `(" [P] <prompt> ( [C|L] name [C|L] name ... ) )`, joins the schemas with5//! `[SEP_STRUCT]`, appends `[SEP_TEXT]` and finally the lower-cased words of the6//! text. Classification heads use the `[L]` child marker, JSON structures the7//! `[C]` one; only the `[P]` and child positions are read back out of the8//! encoder, and `[P]` is dropped before scoring, leaving one logit per label or9//! one span query per field.1011use std::fmt::Write;1213use crate::engine::tokenizer::{Tokenizer, WordSplitter};1415const SEP_STRUCT: &str = "[SEP_STRUCT]";16const SEP_TEXT: &str = "[SEP_TEXT]";17const P_TOKEN: &str = "[P]";18const L_TOKEN: &str = "[L]";19const C_TOKEN: &str = "[C]";20const DESC_TOKEN: &str = "[DESCRIPTION]";21const EXAMPLE_TOKEN: &str = "[EXAMPLE]";22const OUTPUT_TOKEN: &str = "[OUTPUT]";2324/// One classification head, as `gliner2`'s `ClassificationSchema.single` /25/// `.multi` describes it.26#[derive(Debug, Clone)]27pub struct ClassificationTask {28    /// Head name; the key the answer is reported under.29    pub name: String,30    /// Candidate labels, in the order they are presented to the model.31    pub labels: Vec<String>,32    /// Optional per-label descriptions, rendered into the prompt.33    pub label_descriptions: Vec<(String, String)>,34    /// Optional instruction, e.g. the question in a yes/no passage task.35    pub prompt: Option<String>,36    /// Optional few-shot `(input, output)` pairs.37    pub examples: Vec<(String, String)>,38    /// Score every label above [`Self::cls_threshold`] instead of taking the best.39    pub multi_label: bool,40    /// Sigmoid cutoff for multi-label heads.41    pub cls_threshold: f64,42    /// `Some("sigmoid" | "softmax")` overrides the default activation.43    pub class_act: Option<String>,44}4546/// One property of a [`Structure`], as `extract_json`'s field spec describes it.47#[derive(Debug, Clone)]48pub struct Field {49    pub name: String,50    /// `"str"` for a single value, `"list"` for every match.51    pub dtype: String,52    pub description: Option<String>,53    /// Overrides the request-wide threshold for this field.54    pub threshold: Option<f64>,55}5657/// A named record: the JSON-structure schema of `extract_json`.58#[derive(Debug, Clone)]59pub struct Structure {60    pub name: String,61    pub prompt: Option<String>,62    pub fields: Vec<Field>,63}6465/// One schema: a classification head or a JSON structure. Both are encoded the66/// same way; they differ only in the child marker and in how answers are read.67#[derive(Debug, Clone)]68pub(crate) struct Query {69    /// Prompt name: the head or structure name.70    pub(crate) name: String,71    /// Instruction appended to the prompt, as `prompt` does for classification.72    pub(crate) prompt: Option<String>,73    /// `[L]` for classification, `[C]` for structures.74    pub(crate) child: &'static str,75    /// Candidate labels or field names, in schema order.76    pub(crate) fields: Vec<String>,77    /// Descriptions rendered into the prompt, filtered to known fields.78    pub(crate) descriptions: Vec<(String, String)>,79    /// Few-shot `(input, output)` pairs.80    pub(crate) examples: Vec<(String, String)>,81}8283impl Query {84    /// A classification head, whose labels are its fields.85    pub(crate) fn classification(task: &ClassificationTask) -> Self {86        Self {87            name: task.name.clone(),88            prompt: task.prompt.clone(),89            child: L_TOKEN,90            fields: task.labels.clone(),91            descriptions: task.label_descriptions.clone(),92            examples: task.examples.clone(),93        }94    }9596    /// A JSON structure, whose fields are its properties.97    pub(crate) fn structure(structure: &Structure) -> Self {98        Self {99            name: structure.name.clone(),100            prompt: structure.prompt.clone(),101            child: C_TOKEN,102            fields: structure103                .fields104                .iter()105                .map(|field| field.name.clone())106                .collect(),107            descriptions: structure108                .fields109                .iter()110                .filter_map(|field| field.description.clone().map(|d| (field.name.clone(), d)))111                .collect(),112            examples: Vec::new(),113        }114    }115}116117/// A tokenized sample: the encoder input plus where the markers landed.118#[derive(Debug, Clone)]119pub(crate) struct Prepared {120    pub(crate) input_ids: Vec<u32>,121    /// Per query, the sub-word position of `[P]` followed by one child per field.122    pub(crate) marker_positions: Vec<Vec<usize>>,123    /// Sub-word position of the first sub-word of every text word.124    pub(crate) word_positions: Vec<usize>,125    /// Character offsets of every text word, into the normalized text.126    pub(crate) word_offsets: Vec<(usize, usize)>,127}128129/// Build the schema token sequence for one query.130pub(crate) fn schema_tokens(query: &Query) -> Vec<String> {131    let mut prompt = match &query.prompt {132        Some(prompt) => format!("{}: {prompt}", query.name),133        None => query.name.clone(),134    };135    for (field, description) in &query.descriptions {136        if query.fields.contains(field) {137            // Writing into the prompt keeps the token order `gliner2` produces.138            write!(prompt, " {DESC_TOKEN} {field}: {description}").expect("write to a String");139        }140    }141    for (input, output) in &query.examples {142        if query.fields.contains(output) {143            write!(prompt, " {EXAMPLE_TOKEN} {input} {OUTPUT_TOKEN} {output}")144                .expect("write to a String");145        }146    }147148    let mut tokens = vec!["(".to_owned(), P_TOKEN.to_owned(), prompt, "(".to_owned()];149    for field in &query.fields {150        tokens.push(query.child.to_owned());151        tokens.push(field.clone());152    }153    tokens.push(")".to_owned());154    tokens.push(")".to_owned());155    tokens156}157158/// Indices inside `schema_tokens` that carry a routing marker.159fn is_marker(index: usize, len: usize) -> bool {160    index == 1 || (index >= 4 && index < len - 2 && (index - 4).is_multiple_of(2))161}162163/// Tokenize `text` plus `queries` into encoder input, mirroring the collator.164pub(crate) fn prepare(165    tokenizer: &Tokenizer,166    splitter: &WordSplitter,167    text: &str,168    queries: &[Query],169    max_len: Option<usize>,170) -> Result<Prepared, String> {171    let mut input_ids: Vec<u32> = Vec::new();172    let mut marker_positions: Vec<Vec<usize>> = Vec::with_capacity(queries.len());173174    for (index, query) in queries.iter().enumerate() {175        let tokens = schema_tokens(query);176        let mut markers = Vec::with_capacity(tokens.len());177        for (position, token) in tokens.iter().enumerate() {178            let start = input_ids.len();179            input_ids.extend_from_slice(&tokenizer.encode(token)?);180            if is_marker(position, tokens.len()) {181                markers.push(start);182            }183        }184        marker_positions.push(markers);185        if index + 1 < queries.len() {186            input_ids.extend_from_slice(&tokenizer.encode(SEP_STRUCT)?);187        }188    }189190    input_ids.extend_from_slice(&tokenizer.encode(SEP_TEXT)?);191192    // `token_pooling` is `first`, so one embedding per text word: the encoder193    // state at that word's first sub-word.194    let mut word_positions = Vec::new();195    let mut word_offsets = Vec::new();196    for (word, start, end) in splitter197        .split(text)198        .into_iter()199        .take(max_len.unwrap_or(usize::MAX))200    {201        word_positions.push(input_ids.len());202        word_offsets.push((start, end));203        input_ids.extend_from_slice(&tokenizer.encode(&word)?);204    }205206    Ok(Prepared {207        input_ids,208        marker_positions,209        word_positions,210        word_offsets,211    })212}213214/// `gliner2` appends a full stop when the text does not end in punctuation.215#[must_use]216pub fn normalize_text(text: &str) -> String {217    let text = text.trim();218    if text.is_empty() {219        return ".".to_owned();220    }221    if text.ends_with(['.', '!', '?']) {222        text.to_owned()223    } else {224        format!("{text}.")225    }226}227228/// Turn per-label logits into the head's answer.229///230/// Single-label heads softmax and take the best label; multi-label heads apply a231/// sigmoid per label and keep everything at or above `cls_threshold`, falling232/// back to the best label when nothing clears the bar. `class_act` overrides the233/// activation, exactly like `gliner2`'s `"auto" | "sigmoid" | "softmax"`.234pub fn classify_scores(task: &ClassificationTask, logits: &[f32]) -> Vec<(String, f64)> {235    let activation = task.class_act.as_deref().unwrap_or("auto");236    let probs: Vec<f64> = match activation {237        "sigmoid" => logits.iter().copied().map(sigmoid).collect(),238        "softmax" => softmax(logits),239        _ if task.multi_label => logits.iter().copied().map(sigmoid).collect(),240        _ => softmax(logits),241    };242243    let scored: Vec<(String, f64)> = task244        .labels245        .iter()246        .cloned()247        .zip(probs.iter().copied())248        .collect();249    let best = scored250        .iter()251        .max_by(|left, right| left.1.total_cmp(&right.1))252        .cloned()253        .unwrap_or_default();254255    if task.multi_label {256        let chosen: Vec<(String, f64)> = scored257            .into_iter()258            .filter(|(_, score)| *score >= task.cls_threshold)259            .collect();260        if chosen.is_empty() {261            vec![best]262        } else {263            chosen264        }265    } else {266        vec![best]267    }268}269270fn sigmoid(value: f32) -> f64 {271    1.0 / (1.0 + (-f64::from(value)).exp())272}273274fn softmax(values: &[f32]) -> Vec<f64> {275    let max = values276        .iter()277        .copied()278        .map(f64::from)279        .fold(f64::NEG_INFINITY, f64::max);280    let exponentials: Vec<f64> = values281        .iter()282        .map(|value| (f64::from(*value) - max).exp())283        .collect();284    let total: f64 = exponentials.iter().sum();285    exponentials286        .into_iter()287        .map(|value| value / total)288        .collect()289}290291#[cfg(test)]292mod tests {293    use super::{ClassificationTask, Query, classify_scores, normalize_text, schema_tokens};294295    fn task(name: &str, labels: &[&str]) -> ClassificationTask {296        ClassificationTask {297            name: name.to_owned(),298            labels: labels.iter().map(ToString::to_string).collect(),299            label_descriptions: Vec::new(),300            prompt: None,301            examples: Vec::new(),302            multi_label: false,303            cls_threshold: 0.5,304            class_act: None,305        }306    }307308    #[test]309    fn schema_tokens_match_the_python_shape() {310        let tokens = schema_tokens(&Query::classification(&task("intent", &["spam", "ham"])));311        assert_eq!(312            tokens,313            [314                "(", "[P]", "intent", "(", "[L]", "spam", "[L]", "ham", ")", ")"315            ]316        );317    }318319    #[test]320    fn descriptions_and_prompts_extend_the_task_text() {321        let mut head = task("answer", &["yes", "no"]);322        head.prompt = Some("Did it work?".to_owned());323        head.label_descriptions = vec![("yes".to_owned(), "It worked".to_owned())];324        head.label_descriptions325            .push(("missing".to_owned(), "dropped".to_owned()));326        assert_eq!(327            schema_tokens(&Query::classification(&head)),328            [329                "(",330                "[P]",331                "answer: Did it work? [DESCRIPTION] yes: It worked",332                "(",333                "[L]",334                "yes",335                "[L]",336                "no",337                ")",338                ")"339            ]340        );341    }342343    #[test]344    fn text_gains_terminating_punctuation() {345        assert_eq!(normalize_text("hello"), "hello.");346        assert_eq!(normalize_text("  "), ".");347        assert_eq!(normalize_text("really?"), "really?");348    }349350    #[test]351    fn single_label_head_takes_the_best_score() {352        let head = task("sentiment", &["positive", "negative"]);353        let picked = classify_scores(&head, &[0.1, 4.0]);354        assert_eq!(picked.len(), 1);355        assert_eq!(picked[0].0, "negative");356        assert!((picked[0].1 - 0.98).abs() < 0.01);357    }358359    #[test]360    fn multi_label_head_keeps_every_score_above_the_threshold() {361        let mut head = task("aspects", &["battery", "keyboard", "screen"]);362        head.multi_label = true;363        head.cls_threshold = 0.4;364        // Sigmoid of 0.0 / 2.2 / -2.0 is 0.50 / 0.90 / 0.12.365        let picked = classify_scores(&head, &[0.0, 2.2, -2.0]);366        let labels: Vec<&str> = picked.iter().map(|(label, _)| label.as_str()).collect();367        assert_eq!(labels, ["battery", "keyboard"]);368    }369370    #[test]371    fn multi_label_head_falls_back_to_the_best_score() {372        let mut head = task("aspects", &["battery", "keyboard"]);373        head.multi_label = true;374        head.cls_threshold = 0.9;375        let picked = classify_scores(&head, &[0.0, 0.5]);376        assert_eq!(picked.len(), 1);377        assert_eq!(picked[0].0, "keyboard");378    }379380    #[test]381    fn class_act_only_changes_the_activation() {382        let mut head = task("intent", &["spam", "ham"]);383        head.class_act = Some("sigmoid".to_owned());384        // `gliner2` still takes the argmax for a single-label head, but the score385        // it reports is the sigmoid rather than the softmax.386        let picked = classify_scores(&head, &[2.0, 0.0]);387        assert_eq!(picked[0].0, "spam");388        assert!((picked[0].1 - 0.88).abs() < 0.01);389    }390}