LINE1 10 20 30 40 50 60 70 80 901//! Schema encoding, a direct port of `gliner2.processor.SchemaTransformer`.2//!3//! For every schema the Python side builds the token sequence4//! `(" [P] <prompt> ( [C|L] name [C|L] name ... ) )`, joins the schemas with5//! `[SEP_STRUCT]`, appends `[SEP_TEXT]` and finally the lower-cased words of the6//! text. Classification heads use the `[L]` child marker, JSON structures the7//! `[C]` one; only the `[P]` and child positions are read back out of the8//! encoder, and `[P]` is dropped before scoring, leaving one logit per label or9//! one span query per field.1011use std::fmt::Write;1213usecrate::engine::tokenizer::{Tokenizer, WordSplitter};1415constSEP_STRUCT: &str = "[SEP_STRUCT]";16constSEP_TEXT: &str = "[SEP_TEXT]";17constP_TOKEN: &str = "[P]";18constL_TOKEN: &str = "[L]";19constC_TOKEN: &str = "[C]";20constDESC_TOKEN: &str = "[DESCRIPTION]";21constEXAMPLE_TOKEN: &str = "[EXAMPLE]";22constOUTPUT_TOKEN: &str = "[OUTPUT]";2324/// One classification head, as `gliner2`'s `ClassificationSchema.single` /25/// `.multi` describes it.26#[derive(Debug, Clone)]27pubstructClassificationTask {28/// Head name; the key the answer is reported under.29pub name: String,30/// Candidate labels, in the order they are presented to the model.31pub labels: Vec<String>,32/// Optional per-label descriptions, rendered into the prompt.33pub label_descriptions: Vec<(String, String)>,34/// Optional instruction, e.g. the question in a yes/no passage task.35pub prompt: Option<String>,36/// Optional few-shot `(input, output)` pairs.37pub examples: Vec<(String, String)>,38/// Score every label above [`Self::cls_threshold`] instead of taking the best.39pub multi_label: bool,40/// Sigmoid cutoff for multi-label heads.41pub cls_threshold: f64,42/// `Some("sigmoid" | "softmax")` overrides the default activation.43pub class_act: Option<String>,44}4546/// One property of a [`Structure`], as `extract_json`'s field spec describes it.47#[derive(Debug, Clone)]48pubstructField {49pub name: String,50/// `"str"` for a single value, `"list"` for every match.51pub dtype: String,52pub description: Option<String>,53/// Overrides the request-wide threshold for this field.54pub threshold: Option<f64>,55}5657/// A named record: the JSON-structure schema of `extract_json`.58#[derive(Debug, Clone)]59pubstructStructure {60pub name: String,61pub prompt: Option<String>,62pub fields: Vec<Field>,63}6465/// One schema: a classification head or a JSON structure. Both are encoded the66/// same way; they differ only in the child marker and in how answers are read.67#[derive(Debug, Clone)]68pub(crate) structQuery {69/// Prompt name: the head or structure name.70pub(crate) name: String,71/// Instruction appended to the prompt, as `prompt` does for classification.72pub(crate) prompt: Option<String>,73/// `[L]` for classification, `[C]` for structures.74pub(crate) child: &'staticstr,75/// Candidate labels or field names, in schema order.76pub(crate) fields: Vec<String>,77/// Descriptions rendered into the prompt, filtered to known fields.78pub(crate) descriptions: Vec<(String, String)>,79/// Few-shot `(input, output)` pairs.80pub(crate) examples: Vec<(String, String)>,81}8283implQuery {84/// A classification head, whose labels are its fields.85pub(crate) fnclassification(task: &ClassificationTask) -> Self {86Self {87 name: task.name.clone(),88 prompt: task.prompt.clone(),89 child: L_TOKEN,90 fields: task.labels.clone(),91 descriptions: task.label_descriptions.clone(),92 examples: task.examples.clone(),93 }94 }9596/// A JSON structure, whose fields are its properties.97pub(crate) fnstructure(structure: &Structure) -> Self {98Self {99 name: structure.name.clone(),100 prompt: structure.prompt.clone(),101 child: C_TOKEN,102 fields: structure103 .fields104 .iter()105 .map(|field| field.name.clone())106 .collect(),107 descriptions: structure108 .fields109 .iter()110 .filter_map(|field| field.description.clone().map(|d| (field.name.clone(), d)))111 .collect(),112 examples: Vec::new(),113 }114 }115}116117/// A tokenized sample: the encoder input plus where the markers landed.118#[derive(Debug, Clone)]119pub(crate) structPrepared {120pub(crate) input_ids: Vec<u32>,121/// Per query, the sub-word position of `[P]` followed by one child per field.122pub(crate) marker_positions: Vec<Vec<usize>>,123/// Sub-word position of the first sub-word of every text word.124pub(crate) word_positions: Vec<usize>,125/// Character offsets of every text word, into the normalized text.126pub(crate) word_offsets: Vec<(usize, usize)>,127}128129/// Build the schema token sequence for one query.130pub(crate) fnschema_tokens(query: &Query) -> Vec<String> {131letmut prompt = match &query.prompt {132Some(prompt) => format!("{}: {prompt}", query.name),133None => query.name.clone(),134 };135for (field, description) in &query.descriptions {136if query.fields.contains(field) {137// Writing into the prompt keeps the token order `gliner2` produces.138write!(prompt, " {DESC_TOKEN} {field}: {description}").expect("write to a String");139 }140 }141for (input, output) in &query.examples {142if query.fields.contains(output) {143write!(prompt, " {EXAMPLE_TOKEN} {input} {OUTPUT_TOKEN} {output}")144 .expect("write to a String");145 }146 }147148letmut tokens = vec!["(".to_owned(), P_TOKEN.to_owned(), prompt, "(".to_owned()];149for field in &query.fields {150 tokens.push(query.child.to_owned());151 tokens.push(field.clone());152 }153 tokens.push(")".to_owned());154 tokens.push(")".to_owned());155 tokens156}157158/// Indices inside `schema_tokens` that carry a routing marker.159fnis_marker(index: usize, len: usize) -> bool {160 index == 1 || (index >= 4 && index < len - 2 && (index - 4).is_multiple_of(2))161}162163/// Tokenize `text` plus `queries` into encoder input, mirroring the collator.164pub(crate) fnprepare(165 tokenizer: &Tokenizer,166 splitter: &WordSplitter,167 text: &str,168 queries: &[Query],169 max_len: Option<usize>,170) -> Result<Prepared, String> {171letmut input_ids: Vec<u32> = Vec::new();172letmut marker_positions: Vec<Vec<usize>> = Vec::with_capacity(queries.len());173174for (index, query) in queries.iter().enumerate() {175let tokens = schema_tokens(query);176letmut markers = Vec::with_capacity(tokens.len());177for (position, token) in tokens.iter().enumerate() {178let start = input_ids.len();179 input_ids.extend_from_slice(&tokenizer.encode(token)?);180ifis_marker(position, tokens.len()) {181 markers.push(start);182 }183 }184 marker_positions.push(markers);185if index + 1 < queries.len() {186 input_ids.extend_from_slice(&tokenizer.encode(SEP_STRUCT)?);187 }188 }189190 input_ids.extend_from_slice(&tokenizer.encode(SEP_TEXT)?);191192// `token_pooling` is `first`, so one embedding per text word: the encoder193// state at that word's first sub-word.194letmut word_positions = Vec::new();195letmut word_offsets = Vec::new();196for (word, start, end) in splitter197 .split(text)198 .into_iter()199 .take(max_len.unwrap_or(usize::MAX))200 {201 word_positions.push(input_ids.len());202 word_offsets.push((start, end));203 input_ids.extend_from_slice(&tokenizer.encode(&word)?);204 }205206Ok(Prepared {207 input_ids,208 marker_positions,209 word_positions,210 word_offsets,211 })212}213214/// `gliner2` appends a full stop when the text does not end in punctuation.215#[must_use]216pubfnnormalize_text(text: &str) -> String {217let text = text.trim();218if text.is_empty() {219return".".to_owned();220 }221if text.ends_with(['.', '!', '?']) {222 text.to_owned()223 } else {224format!("{text}.")225 }226}227228/// Turn per-label logits into the head's answer.229///230/// Single-label heads softmax and take the best label; multi-label heads apply a231/// sigmoid per label and keep everything at or above `cls_threshold`, falling232/// back to the best label when nothing clears the bar. `class_act` overrides the233/// activation, exactly like `gliner2`'s `"auto" | "sigmoid" | "softmax"`.234pubfnclassify_scores(task: &ClassificationTask, logits: &[f32]) -> Vec<(String, f64)> {235let activation = task.class_act.as_deref().unwrap_or("auto");236let probs: Vec<f64> = match activation {237"sigmoid" => logits.iter().copied().map(sigmoid).collect(),238"softmax" => softmax(logits),239 _ if task.multi_label => logits.iter().copied().map(sigmoid).collect(),240 _ => softmax(logits),241 };242243let scored: Vec<(String, f64)> = task244 .labels245 .iter()246 .cloned()247 .zip(probs.iter().copied())248 .collect();249let best = scored250 .iter()251 .max_by(|left, right| left.1.total_cmp(&right.1))252 .cloned()253 .unwrap_or_default();254255if task.multi_label {256let chosen: Vec<(String, f64)> = scored257 .into_iter()258 .filter(|(_, score)| *score >= task.cls_threshold)259 .collect();260if chosen.is_empty() {261vec![best]262 } else {263 chosen264 }265 } else {266vec![best]267 }268}269270fnsigmoid(value: f32) -> f64 {2711.0 / (1.0 + (-f64::from(value)).exp())272}273274fnsoftmax(values: &[f32]) -> Vec<f64> {275let max = values276 .iter()277 .copied()278 .map(f64::from)279 .fold(f64::NEG_INFINITY, f64::max);280let exponentials: Vec<f64> = values281 .iter()282 .map(|value| (f64::from(*value) - max).exp())283 .collect();284let total: f64 = exponentials.iter().sum();285 exponentials286 .into_iter()287 .map(|value| value / total)288 .collect()289}290291#[cfg(test)]292mod tests {293usesuper::{ClassificationTask, Query, classify_scores, normalize_text, schema_tokens};294295fntask(name: &str, labels: &[&str]) -> ClassificationTask {296ClassificationTask {297 name: name.to_owned(),298 labels: labels.iter().map(ToString::to_string).collect(),299 label_descriptions: Vec::new(),300 prompt: None,301 examples: Vec::new(),302 multi_label: false,303 cls_threshold: 0.5,304 class_act: None,305 }306 }307308#[test]309fnschema_tokens_match_the_python_shape() {310let tokens = schema_tokens(&Query::classification(&task("intent", &["spam", "ham"])));311assert_eq!(312 tokens,313 [314"(", "[P]", "intent", "(", "[L]", "spam", "[L]", "ham", ")", ")"315 ]316 );317 }318319#[test]320fndescriptions_and_prompts_extend_the_task_text() {321letmut head = task("answer", &["yes", "no"]);322 head.prompt = Some("Did it work?".to_owned());323 head.label_descriptions = vec![("yes".to_owned(), "It worked".to_owned())];324 head.label_descriptions325 .push(("missing".to_owned(), "dropped".to_owned()));326assert_eq!(327schema_tokens(&Query::classification(&head)),328 [329"(",330"[P]",331"answer: Did it work? [DESCRIPTION] yes: It worked",332"(",333"[L]",334"yes",335"[L]",336"no",337")",338")"339 ]340 );341 }342343#[test]344fntext_gains_terminating_punctuation() {345assert_eq!(normalize_text("hello"), "hello.");346assert_eq!(normalize_text(" "), ".");347assert_eq!(normalize_text("really?"), "really?");348 }349350#[test]351fnsingle_label_head_takes_the_best_score() {352let head = task("sentiment", &["positive", "negative"]);353let picked = classify_scores(&head, &[0.1, 4.0]);354assert_eq!(picked.len(), 1);355assert_eq!(picked[0].0, "negative");356assert!((picked[0].1 - 0.98).abs() < 0.01);357 }358359#[test]360fnmulti_label_head_keeps_every_score_above_the_threshold() {361letmut head = task("aspects", &["battery", "keyboard", "screen"]);362 head.multi_label = true;363 head.cls_threshold = 0.4;364// Sigmoid of 0.0 / 2.2 / -2.0 is 0.50 / 0.90 / 0.12.365let picked = classify_scores(&head, &[0.0, 2.2, -2.0]);366let labels: Vec<&str> = picked.iter().map(|(label, _)| label.as_str()).collect();367assert_eq!(labels, ["battery", "keyboard"]);368 }369370#[test]371fnmulti_label_head_falls_back_to_the_best_score() {372letmut head = task("aspects", &["battery", "keyboard"]);373 head.multi_label = true;374 head.cls_threshold = 0.9;375let picked = classify_scores(&head, &[0.0, 0.5]);376assert_eq!(picked.len(), 1);377assert_eq!(picked[0].0, "keyboard");378 }379380#[test]381fnclass_act_only_changes_the_activation() {382letmut head = task("intent", &["spam", "ham"]);383 head.class_act = Some("sigmoid".to_owned());384// `gliner2` still takes the argmax for a single-label head, but the score385// it reports is the sigmoid rather than the softmax.386let picked = classify_scores(&head, &[2.0, 0.0]);387assert_eq!(picked[0].0, "spam");388assert!((picked[0].1 - 0.88).abs() < 0.01);389 }390}