LINE1 10 20 30 40 50 60 70 80 90123456789101112use serde_json::{Map, Value};1314use crate::engine::schema::Structure;151617#[derive(Debug, Clone, PartialEq)]18pub(crate) struct Span {19 pub(crate) text: String,20 pub(crate) confidence: f64,21 pub(crate) start: usize,22 pub(crate) end: usize,23}242526#[derive(Debug, Clone, Copy)]27pub(crate) struct ScoreShape {28 pub(crate) instances: usize,29 pub(crate) fields: usize,30 pub(crate) words: usize,31 pub(crate) widths: usize,32}333435#[derive(Debug, Clone, Copy)]36pub(crate) struct Detail {37 pub(crate) threshold: f64,38 pub(crate) confidence: bool,39 pub(crate) spans: bool,40}41424344454647pub(crate) fn decode(48 structure: &Structure,49 scores: &[f32],50 shape: ScoreShape,51 word_offsets: &[(usize, usize)],52 text: &str,53 detail: Detail,54) -> Vec<Value> {55 let ScoreShape {56 instances,57 fields,58 words,59 widths,60 } = shape;61 let mut out = Vec::new();62 for instance in 0..instances {63 let mut record = Map::new();64 let mut filled = false;6566 for (field_index, field) in structure.fields.iter().enumerate() {67 let plane = (instance * fields + field_index) * words * widths;68 let spans = field_spans(69 scores,70 plane,71 words,72 widths,73 word_offsets,74 text,75 field.threshold.unwrap_or(detail.threshold),76 );77 let single = field.dtype.as_str() != "list";7879 if single {80 let value = match spans.first() {81 Some(span) => {82 filled = true;83 span_value(span, detail.confidence, detail.spans)84 }85 None => Value::Null,86 };87 record.insert(field.name.clone(), value);88 } else {89 let values: Vec<Value> = spans90 .iter()91 .map(|span| span_value(span, detail.confidence, detail.spans))92 .collect();93 filled = filled || !values.is_empty();94 record.insert(field.name.clone(), Value::Array(values));95 }96 }9798 if filled {99 out.push(Value::Object(record));100 }101 }102 out103}104105fn span_value(span: &Span, include_confidence: bool, include_spans: bool) -> Value {106 if include_confidence && include_spans {107 return serde_json::json!({108 "text": span.text,109 "confidence": span.confidence,110 "start": span.start,111 "end": span.end,112 });113 }114 if include_confidence {115 return serde_json::json!({ "text": span.text, "confidence": span.confidence });116 }117 if include_spans {118 return serde_json::json!({ "text": span.text, "start": span.start, "end": span.end });119 }120 Value::String(span.text.clone())121}122123124125126fn field_spans(127 scores: &[f32],128 plane: usize,129 words: usize,130 widths: usize,131 word_offsets: &[(usize, usize)],132 text: &str,133 threshold: f64,134) -> Vec<Span> {135 let mut spans = Vec::new();136 for start in 0..words {137 for width in 0..widths {138 let end = start + width + 1;139 if end > words {140 continue;141 }142 let Some(&score) = scores.get(plane + start * widths + width) else {143 continue;144 };145 let score = f64::from(score);146 if score < threshold {147 continue;148 }149 let Some((char_start, char_end)) = span_offsets(start, end, word_offsets) else {150 continue;151 };152 let span_text = text153 .get(char_start..char_end)154 .unwrap_or_default()155 .trim()156 .to_owned();157 if !span_text.is_empty() {158 spans.push(Span {159 text: span_text,160 confidence: score,161 start: char_start,162 end: char_end,163 });164 }165 }166 }167168 169 spans.sort_by(|left, right| {170 right171 .confidence172 .total_cmp(&left.confidence)173 .then_with(|| left.start.cmp(&right.start))174 .then_with(|| left.end.cmp(&right.end))175 });176 let mut kept: Vec<Span> = Vec::new();177 for span in spans {178 if kept179 .iter()180 .any(|existing| span.start < existing.end && existing.start < span.end)181 {182 continue;183 }184 kept.push(span);185 }186 kept187}188189190fn span_offsets(start: usize, end: usize, offsets: &[(usize, usize)]) -> Option<(usize, usize)> {191 let (char_start, _) = offsets.get(start)?;192 let (_, char_end) = offsets.get(end.checked_sub(1)?)?;193 Some((*char_start, *char_end))194}195196#[cfg(test)]197mod tests {198 use super::{Detail, ScoreShape, Structure, decode, span_offsets};199 use crate::engine::schema::Field;200 use serde_json::Value;201202 const TEXT: &str = "the tool costs 2000 and you need to pay taxes of 210.";203204 205 const WORDS: [&str; 13] = [206 "the", "tool", "costs", "2000", "and", "you", "need", "to", "pay", "taxes", "of", "210",207 ".",208 ];209210 211 fn word_offsets() -> Vec<(usize, usize)> {212 let mut offsets = Vec::with_capacity(WORDS.len());213 let mut cursor = 0;214 for word in WORDS {215 let start = TEXT[cursor..].find(word).expect("word") + cursor;216 offsets.push((start, start + word.len()));217 cursor = start + word.len();218 }219 offsets220 }221222 fn structure(fields: &[(&str, &str)]) -> Structure {223 Structure {224 name: "record".to_owned(),225 prompt: None,226 fields: fields227 .iter()228 .map(|(name, dtype)| Field {229 name: (*name).to_owned(),230 dtype: (*dtype).to_owned(),231 description: None,232 threshold: None,233 })234 .collect(),235 }236 }237238 239 240 241 fn scores(fields: usize, hits: &[(usize, usize, f32)]) -> Vec<f32> {242 const WIDTHS: usize = 2;243 let plane = WORDS.len() * WIDTHS;244 let mut scores = vec![0.0f32; fields * plane];245 for (field, word, score) in hits {246 scores[field * plane + word * WIDTHS] = *score;247 }248 scores249 }250251 fn shape(fields: usize) -> ScoreShape {252 ScoreShape {253 instances: 1,254 fields,255 words: WORDS.len(),256 widths: 2,257 }258 }259260 fn detail(threshold: f64, confidence: bool, spans: bool) -> Detail {261 Detail {262 threshold,263 confidence,264 spans,265 }266 }267268 269 fn close(actual: &Value, expected: f64) -> bool {270 actual271 .as_f64()272 .is_some_and(|value| (value - expected).abs() < 1e-6)273 }274275 #[test]276 fn single_values_are_reported_as_an_object() {277 278 let values = decode(279 &structure(&[("taxes", "str"), ("costs", "str")]),280 &scores(2, &[(0, 11, 0.8), (1, 3, 0.9)]),281 shape(2),282 &word_offsets(),283 TEXT,284 detail(0.5, false, false),285 );286 assert_eq!(287 values,288 vec![serde_json::json!({ "taxes": "210", "costs": "2000" })]289 );290 }291292 #[test]293 fn list_fields_keep_every_span() {294 let values = decode(295 &structure(&[("costs", "list")]),296 &scores(1, &[(0, 3, 0.9), (0, 11, 0.7)]),297 shape(1),298 &word_offsets(),299 TEXT,300 detail(0.5, true, false),301 );302 let spans = values[0]["costs"].as_array().expect("array");303 assert_eq!(spans.len(), 2, "{spans:?}");304 assert_eq!(spans[0]["text"], serde_json::json!("2000"));305 assert!(close(&spans[0]["confidence"], 0.9), "{spans:?}");306 assert_eq!(spans[1]["text"], serde_json::json!("210"));307 assert!(close(&spans[1]["confidence"], 0.7), "{spans:?}");308 }309310 #[test]311 fn overlapping_candidates_are_suppressed() {312 313 const WIDTHS: usize = 2;314 let mut hits = scores(1, &[(0, 3, 0.7)]);315 hits[3 * WIDTHS + 1] = 0.9;316 let values = decode(317 &structure(&[("costs", "str")]),318 &hits,319 shape(1),320 &word_offsets(),321 TEXT,322 detail(0.5, false, false),323 );324 assert_eq!(values[0]["costs"], serde_json::json!("2000 and"));325 }326327 #[test]328 fn an_instance_with_no_hits_is_dropped() {329 let values = decode(330 &structure(&[("taxes", "str")]),331 &scores(1, &[]),332 shape(1),333 &word_offsets(),334 TEXT,335 detail(0.5, false, false),336 );337 assert!(values.is_empty(), "{values:?}");338 }339340 #[test]341 fn offsets_survive_the_round_trip() {342 let offsets = word_offsets();343 let (start, end) = span_offsets(3, 4, &offsets).expect("offsets");344 assert_eq!(&TEXT[start..end], "2000");345 }346}