LINE1 10 20 30 40 50 60 70 80 90123456789pub mod deberta;10pub mod download;11pub mod extract;12pub mod schema;13pub mod span;14pub mod tokenizer;1516use std::path::Path;1718use candle_core::{Device, Tensor};19use candle_nn::VarBuilder;2021use crate::config::Config;2223use self::deberta::{ClassifierHead, Encoder};24use self::schema::{25 ClassificationTask, Prepared, Query, Structure, classify_scores, normalize_text, prepare,26};27use self::span::SpanHead;28use self::tokenizer::{Tokenizer, WordSplitter};293031pub type TaskAnswer = Vec<Vec<(String, f64)>>;323334#[derive(Debug, Clone, Copy)]35pub struct ExtractOptions {36 pub threshold: f64,37 pub include_confidence: bool,38 pub include_spans: bool,39}4041impl Default for ExtractOptions {42 fn default() -> Self {43 Self {44 threshold: 0.5,45 include_confidence: false,46 include_spans: false,47 }48 }49}505152pub struct Engine {53 tokenizer: Tokenizer,54 splitter: WordSplitter,55 encoder: Encoder,56 head: ClassifierHead,57 span: SpanHead,58 max_width: usize,59}6061impl Engine {62 63 64 65 66 67 pub async fn load(config: &Config) -> Result<Self, String> {68 download::ensure(69 &config.model_dir,70 &config.model_repo,71 &config.hf_endpoint,72 config.hf_token.as_deref(),73 )74 .await?;7576 let tokenizer = Tokenizer::from_file(&config.model_dir.join("tokenizer.json"))?;77 let splitter = WordSplitter::new()?;78 let encoder_config =79 deberta::read_config(&config.model_dir.join("encoder_config/config.json"))?;80 let (encoder, head) =81 deberta::load(&encoder_config, &weights(&config.model_dir), &Device::Cpu)82 .map_err(|error| format!("cannot load the checkpoint: {error}"))?;8384 let device = Device::Cpu;85 let weights_path = weights(&config.model_dir);86 let max_width = max_width(&config.model_dir)?;87 88 89 let span_vb = unsafe {90 VarBuilder::from_mmaped_safetensors(&[&weights_path], candle_core::DType::F32, &device)91 }92 .map_err(|error| format!("cannot open {}: {error}", weights_path.display()))?;93 let span = SpanHead::load(&span_vb, encoder_config.hidden_size, max_width, &device)94 .map_err(|error| format!("cannot load the span head: {error}"))?;9596 Ok(Self {97 tokenizer,98 splitter,99 encoder,100 head,101 span,102 max_width,103 })104 }105106 107 108 109 110 111 #[cfg(test)]112 pub(crate) fn for_tests() -> Self {113 Self {114 tokenizer: Tokenizer::empty(),115 splitter: WordSplitter::empty(),116 encoder: Encoder::empty(),117 head: ClassifierHead::empty(),118 span: SpanHead::empty(),119 max_width: 0,120 }121 }122123 124 125 126 127 128 pub fn classify(129 &self,130 text: &str,131 tasks: &[ClassificationTask],132 max_len: Option<usize>,133 ) -> Result<TaskAnswer, String> {134 let queries: Vec<Query> = tasks.iter().map(Query::classification).collect();135 let prepared = self.prepare(text, &queries, max_len)?;136 let hidden = self.encode(&prepared)?;137 let logits = self.label_logits(&hidden, &prepared)?;138 Self::split_logits(&logits, tasks)139 }140141 142 143 144 145 pub fn extract(146 &self,147 text: &str,148 structures: &[Structure],149 options: ExtractOptions,150 max_len: Option<usize>,151 ) -> Result<Vec<Vec<serde_json::Value>>, String> {152 if structures.is_empty() {153 return Err("at least one structure is required".to_owned());154 }155 let queries: Vec<Query> = structures.iter().map(Query::structure).collect();156 let normalized = normalize_text(text);157 let prepared = prepare(158 &self.tokenizer,159 &self.splitter,160 &normalized,161 &queries,162 max_len,163 )?;164 let hidden = self.encode(&prepared)?;165166 let mut results = Vec::with_capacity(structures.len());167 for (index, structure) in structures.iter().enumerate() {168 results.push(self.extract_one(169 structure,170 &hidden,171 &prepared,172 index,173 &normalized,174 options,175 )?);176 }177 Ok(results)178 }179180 181 fn extract_one(182 &self,183 structure: &Structure,184 hidden: &Tensor,185 prepared: &Prepared,186 index: usize,187 text: &str,188 options: ExtractOptions,189 ) -> Result<Vec<serde_json::Value>, String> {190 let markers = prepared191 .marker_positions192 .get(index)193 .ok_or_else(|| format!("structure '{}' was not encoded", structure.name))?;194 let device = hidden.device();195 let gather = |positions: &[usize]| -> Result<Tensor, String> {196 let indices: Vec<u32> = positions197 .iter()198 .map(|position| u32::try_from(*position).map_err(|error| error.to_string()))199 .collect::<Result<_, _>>()?;200 Tensor::new(indices, device)201 .and_then(|index| hidden.index_select(&index, 0))202 .map_err(|error| format!("cannot gather marker states: {error}"))203 };204205 let query = gather(&markers[..1])?;206 let fields = gather(markers[1..].to_vec().as_slice())?;207 let words = gather(&prepared.word_positions)?;208209 let scores = self210 .span211 .score(&words, &query, &fields)212 .map_err(|error| format!("span scoring failed: {error}"))?;213 let dimensions = scores214 .dims4()215 .map_err(|error| format!("bad score shape: {error}"))?;216 let (instances, field_count, word_count, width_count) = dimensions;217 let flat = scores218 .flatten_all()219 .and_then(|scores| scores.to_vec1::<f32>())220 .map_err(|error| format!("cannot read scores: {error}"))?;221222 Ok(extract::decode(223 structure,224 &flat,225 extract::ScoreShape {226 instances,227 fields: field_count,228 words: word_count,229 widths: width_count,230 },231 &prepared.word_offsets,232 text,233 extract::Detail {234 threshold: options.threshold,235 confidence: options.include_confidence,236 spans: options.include_spans,237 },238 ))239 }240241 fn encode(&self, prepared: &Prepared) -> Result<Tensor, String> {242 self.encoder243 .forward(&prepared.input_ids)244 .map_err(|error| format!("encoder failed: {error}"))245 }246247 fn prepare(248 &self,249 text: &str,250 queries: &[Query],251 max_len: Option<usize>,252 ) -> Result<Prepared, String> {253 prepare(254 &self.tokenizer,255 &self.splitter,256 &normalize_text(text),257 queries,258 max_len,259 )260 }261262 263 fn label_logits(&self, hidden: &Tensor, prepared: &Prepared) -> Result<Vec<f32>, String> {264 let mut positions: Vec<u32> = Vec::new();265 for markers in &prepared.marker_positions {266 for position in markers.iter().skip(1) {267 positions.push(u32::try_from(*position).map_err(|error| error.to_string())?);268 }269 }270 let index = Tensor::new(positions, hidden.device())271 .map_err(|error| format!("cannot index markers: {error}"))?;272 let embeddings = hidden273 .index_select(&index, 0)274 .map_err(|error| format!("cannot gather marker states: {error}"))?;275 self.head276 .forward(&embeddings)277 .map_err(|error| format!("classifier failed: {error}"))278 }279280 fn split_logits(logits: &[f32], tasks: &[ClassificationTask]) -> Result<TaskAnswer, String> {281 let mut offset = 0;282 let mut answers = Vec::with_capacity(tasks.len());283 for task in tasks {284 let count = task.labels.len();285 let slice = logits.get(offset..offset + count).ok_or_else(|| {286 format!(287 "model returned {} logits for {} tasks",288 logits.len(),289 tasks.len()290 )291 })?;292 answers.push(classify_scores(task, slice));293 offset += count;294 }295 Ok(answers)296 }297298 299 pub fn max_width(&self) -> usize {300 self.max_width301 }302}303304fn weights(model_dir: &Path) -> std::path::PathBuf {305 model_dir.join("model.safetensors")306}307308309fn max_width(model_dir: &Path) -> Result<usize, String> {310 let raw = std::fs::read_to_string(model_dir.join("config.json"))311 .map_err(|error| format!("cannot read the model config: {error}"))?;312 let config: serde_json::Value = serde_json::from_str(&raw)313 .map_err(|error| format!("cannot parse the model config: {error}"))?;314 let max_width = config315 .get("max_width")316 .or_else(|| config.pointer("/span_head/max_width"))317 .and_then(serde_json::Value::as_u64)318 .unwrap_or(8);319 usize::try_from(max_width).map_err(|error| format!("max_width is too large: {error}"))320}