Skip to content
1        10        20        30        40        50        60        70        80        901//! Text handling that mirrors `gliner2`.2//!3//! The model never sees a raw string: `gliner2` splits the input into words4//! with a regular expression, lower-cases every word, and then encodes each word5//! on its own with the checkpoint's `SentencePiece` (Unigram) tokenizer. Both6//! halves are reproduced here so the token stream is byte-identical to the7//! Python implementation.89use std::collections::HashMap;10use std::path::Path;11use std::sync::Mutex;1213use regex::Regex;1415/// `gliner2.processing.word_splitter.WhitespaceTokenSplitter`.16const SPLIT_PATTERN: &str = concat!(17    r"(?:https?://[^\s]+|www\.[^\s]+)",18    r"|[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}",19    r"|@[a-z0-9_]+",20    r"|\w+(?:[-_]\w+)*",21    r"|\S",22);2324/// Splits text into `(lowercased word, start, end)` triples.25pub struct WordSplitter {26    pattern: Regex,27}2829impl WordSplitter {30    pub(crate) fn new() -> Result<Self, String> {31        let pattern = Regex::new(SPLIT_PATTERN)32            .map_err(|error| format!("cannot compile the word splitter: {error}"))?;33        Ok(Self { pattern })34    }3536    /// A splitter that matches nothing, for tests that never split text.37    #[cfg(test)]38    pub(crate) fn empty() -> Self {39        Self {40            pattern: Regex::new(r"$^").expect("an empty pattern"),41        }42    }4344    /// Split `text` into words, lower-casing each one and recording its offsets.45    ///46    /// Only the token text matters for classification; the offsets are kept so47    /// the splitter stays a drop-in for `gliner2`'s callable splitter.48    pub(crate) fn split(&self, text: &str) -> Vec<(String, usize, usize)> {49        self.pattern50            .find_iter(text)51            .map(|found| (found.as_str().to_lowercase(), found.start(), found.end()))52            .collect()53    }54}5556/// The checkpoint tokenizer, loaded straight from `tokenizer.json`.57///58/// Words repeat heavily inside a schema and across requests, so encoded words59/// are memoized the same way `gliner2` memoizes `tokenizer.tokenize`.60pub struct Tokenizer {61    inner: tokenizers::Tokenizer,62    cache: Mutex<HashMap<String, Vec<u32>>>,63}6465impl Tokenizer {66    pub(crate) fn from_file(path: &Path) -> Result<Self, String> {67        let inner = tokenizers::Tokenizer::from_file(path)68            .map_err(|error| format!("cannot load {}: {error}", path.display()))?;69        Ok(Self {70            inner,71            cache: Mutex::new(HashMap::new()),72        })73    }7475    /// A tokenizer that encodes nothing, for tests that never tokenize.76    #[cfg(test)]77    pub(crate) fn empty() -> Self {78        let inner = tokenizers::Tokenizer::new(79            tokenizers::models::wordlevel::WordLevel::builder()80                .build()81                .expect("an empty model"),82        );83        Self {84            inner,85            cache: Mutex::new(HashMap::new()),86        }87    }8889    /// Encode a single word or schema token, without `[CLS]`/`[SEP]`.90    pub(crate) fn encode(&self, word: &str) -> Result<Vec<u32>, String> {91        if let Ok(cache) = self.cache.lock()92            && let Some(ids) = cache.get(word)93        {94            return Ok(ids.clone());95        }96        let ids = self97            .inner98            .encode(word, false)99            .map_err(|error| format!("cannot tokenize {word:?}: {error}"))?100            .get_ids()101            .to_vec();102        if let Ok(mut cache) = self.cache.lock() {103            cache.insert(word.to_owned(), ids.clone());104        }105        Ok(ids)106    }107}108109#[cfg(test)]110mod tests {111    use std::path::Path;112113    use super::{Tokenizer, WordSplitter};114115    fn splitter() -> WordSplitter {116        WordSplitter::new().expect("splitter")117    }118119    #[test]120    fn splits_words_keeps_punctuation_and_offsets() {121        let words = splitter().split("Hello, brave new world!");122        let text: Vec<&str> = words.iter().map(|(word, _, _)| word.as_str()).collect();123        assert_eq!(text, ["hello", ",", "brave", "new", "world", "!"]);124        assert_eq!(words[0].1, 0);125        assert_eq!(words[0].2, 5);126    }127128    #[test]129    fn keeps_urls_and_emails_intact() {130        let words = splitter().split("mail ops@example.com or see https://example.com/a?b=1 now");131        let text: Vec<&str> = words.iter().map(|(word, _, _)| word.as_str()).collect();132        assert_eq!(133            text,134            [135                "mail",136                "ops@example.com",137                "or",138                "see",139                "https://example.com/a?b=1",140                "now"141            ]142        );143    }144145    #[test]146    fn tokenizer_is_optional_without_a_file() {147        // Guards the constructor signature; a real load is covered by the148        // integration test that runs against the downloaded checkpoint.149        assert!(Tokenizer::from_file(Path::new("/nonexistent/tokenizer.json")).is_err());150    }151}