LINE1 10 20 30 40 50 60 70 80 90123456789use candle_core::{Error, Result as CandleResult, Tensor};10use candle_nn::{Linear, Module, VarBuilder};111213141516fn mlp(17 vb: &VarBuilder,18 name: &str,19 in_features: usize,20 intermediate: usize,21 out_features: usize,22) -> CandleResult<(Linear, Linear)> {23 let first_weight = vb.get((intermediate, in_features), &format!("{name}.0.weight"))?;24 let first_bias = vb.get(intermediate, &format!("{name}.0.bias"))?;25 let second_weight = vb.get((out_features, intermediate), &format!("{name}.2.weight"))?;26 let second_bias = vb.get(out_features, &format!("{name}.2.bias"))?;27 Ok((28 Linear::new(first_weight, Some(first_bias)),29 Linear::new(second_weight, Some(second_bias)),30 ))31}3233fn gate_error() -> Error {34 Error::Msg("the GRU gates are not split three ways".to_owned())35}363738fn sigmoid(x: &Tensor) -> CandleResult<Tensor> {39 x.neg()?.exp()?.affine(1.0, 1.0)?.recip()40}414243fn linear(input: &Tensor, weight: &Tensor, bias: &Tensor) -> CandleResult<Tensor> {44 input.matmul(&weight.t()?)?.broadcast_add(bias)45}464748struct Projection {49 first: Linear,50 second: Linear,51}5253impl Projection {54 fn forward(&self, input: &Tensor) -> CandleResult<Tensor> {55 self.second.forward(&self.first.forward(input)?.relu()?)56 }57}5859606162fn projection_layer(63 vb: &VarBuilder,64 name: &str,65 in_features: usize,66 hidden: usize,67) -> CandleResult<Projection> {68 let first_weight = vb.get((hidden * 4, in_features), &format!("{name}.0.weight"))?;69 let first_bias = vb.get(hidden * 4, &format!("{name}.0.bias"))?;70 let second_weight = vb.get((hidden, hidden * 4), &format!("{name}.3.weight"))?;71 let second_bias = vb.get(hidden, &format!("{name}.3.bias"))?;72 Ok(Projection {73 first: Linear::new(first_weight, Some(first_bias)),74 second: Linear::new(second_weight, Some(second_bias)),75 })76}777879struct SpanMarker {80 start: Projection,81 end: Projection,82 out: Projection,83 max_width: usize,84}8586impl SpanMarker {87 fn load(vb: &VarBuilder, hidden: usize, max_width: usize) -> CandleResult<Self> {88 let prefix = "span_rep.span_rep_layer";89 Ok(Self {90 start: projection_layer(vb, &format!("{prefix}.project_start"), hidden, hidden)?,91 end: projection_layer(vb, &format!("{prefix}.project_end"), hidden, hidden)?,92 out: projection_layer(vb, &format!("{prefix}.out_project"), hidden * 2, hidden)?,93 max_width,94 })95 }9697 98 #[cfg(test)]99 fn empty() -> Self {100 Self {101 start: Projection {102 first: Linear::new(103 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)104 .expect("empty"),105 None,106 ),107 second: Linear::new(108 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)109 .expect("empty"),110 None,111 ),112 },113 end: Projection {114 first: Linear::new(115 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)116 .expect("empty"),117 None,118 ),119 second: Linear::new(120 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)121 .expect("empty"),122 None,123 ),124 },125 out: Projection {126 first: Linear::new(127 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)128 .expect("empty"),129 None,130 ),131 second: Linear::new(132 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)133 .expect("empty"),134 None,135 ),136 },137 max_width: 0,138 }139 }140141 142 fn forward(&self, token_states: &Tensor) -> CandleResult<Tensor> {143 let text_len = token_states.dim(0)?;144 let hidden = token_states.dim(1)?;145 let width = self.max_width;146 let device = token_states.device();147148 let start_rep = self.start.forward(token_states)?;149 let end_rep = self.end.forward(token_states)?;150151 152 153 154 let to_index = |value: usize| u32::try_from(value).unwrap_or(u32::MAX);155 let starts: Vec<u32> = (0..text_len)156 .flat_map(|start| vec![to_index(start); width])157 .collect();158 let ends: Vec<u32> = (0..text_len)159 .flat_map(|start| {160 (0..width).map(move |offset| to_index((start + offset).min(text_len - 1)))161 })162 .collect();163164 let start_spans = start_rep.index_select(&Tensor::new(starts, device)?, 0)?;165 let end_spans = end_rep.index_select(&Tensor::new(ends, device)?, 0)?;166 let joined = Tensor::cat(&[&start_spans, &end_spans], 1)?.relu()?;167 self.out168 .forward(&joined)?169 .reshape((text_len, width, hidden))170 }171}172173174struct CountEmbed {175 pos_embedding: Tensor,176 gru_input: Tensor,177 gru_hidden: Tensor,178 bias_input: Tensor,179 bias_hidden: Tensor,180 projector: Projection,181 max_count: usize,182}183184impl CountEmbed {185 fn load(vb: &VarBuilder, hidden: usize) -> CandleResult<Self> {186 let (first, second) = mlp(vb, "count_embed.projector", hidden * 2, hidden * 4, hidden)?;187 Ok(Self {188 pos_embedding: vb.get((20, hidden), "count_embed.pos_embedding.weight")?,189 gru_input: vb.get((hidden * 3, hidden), "count_embed.gru.weight_ih_l0")?,190 gru_hidden: vb.get((hidden * 3, hidden), "count_embed.gru.weight_hh_l0")?,191 bias_input: vb.get(hidden * 3, "count_embed.gru.bias_ih_l0")?,192 bias_hidden: vb.get(hidden * 3, "count_embed.gru.bias_hh_l0")?,193 projector: Projection { first, second },194 max_count: 20,195 })196 }197198 199 #[cfg(test)]200 fn empty() -> Self {201 Self {202 pos_embedding: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)203 .expect("an empty tensor"),204 gru_input: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)205 .expect("an empty tensor"),206 gru_hidden: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)207 .expect("an empty tensor"),208 bias_input: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)209 .expect("an empty tensor"),210 bias_hidden: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)211 .expect("an empty tensor"),212 projector: Projection {213 first: Linear::new(214 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)215 .expect("empty"),216 None,217 ),218 second: Linear::new(219 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)220 .expect("empty"),221 None,222 ),223 },224 max_count: 0,225 }226 }227228 229 fn forward(&self, fields: &Tensor, count: usize) -> CandleResult<Tensor> {230 let count = count.min(self.max_count).max(1);231 let field_count = fields.dim(0)?;232 let hidden = fields.dim(1)?;233 let device = fields.device();234235 let positions = self.pos_embedding.narrow(0, 0, count)?.to_device(device)?;236 let sequence = positions237 .unsqueeze(1)?238 .broadcast_as((count, field_count, hidden))?;239240 let mut state = fields.clone();241 let mut steps = Vec::with_capacity(count);242 for step in 0..count {243 let input = sequence.narrow(0, step, 1)?.squeeze(0)?;244 let input_gate = linear(&input, &self.gru_input, &self.bias_input)?;245 let hidden_gate = linear(&state, &self.gru_hidden, &self.bias_hidden)?;246 let mut input_gate = input_gate.chunk(3, 1)?.into_iter();247 let mut hidden_gate = hidden_gate.chunk(3, 1)?.into_iter();248 let input_reset = input_gate.next().ok_or_else(gate_error)?;249 let input_update = input_gate.next().ok_or_else(gate_error)?;250 let input_new = input_gate.next().ok_or_else(gate_error)?;251 let hidden_reset = hidden_gate.next().ok_or_else(gate_error)?;252 let hidden_update = hidden_gate.next().ok_or_else(gate_error)?;253 let hidden_new = hidden_gate.next().ok_or_else(gate_error)?;254 255 let reset = sigmoid(&input_reset.broadcast_add(&hidden_reset)?)?;256 let update = sigmoid(&input_update.broadcast_add(&hidden_update)?)?;257 let candidate = input_new258 .broadcast_add(&reset.broadcast_mul(&hidden_new)?)?259 .tanh()?;260 261 let closed = update.affine(-1.0, 1.0)?;262 state = closed263 .broadcast_mul(&candidate)?264 .broadcast_add(&update.broadcast_mul(&state)?)?;265 steps.push(state.clone());266 }267268 let sequence = Tensor::stack(&steps, 0)?;269 let fields = fields.unsqueeze(0)?.broadcast_as(sequence.dims())?;270 self.projector271 .forward(&Tensor::cat(&[&sequence, &fields], 2)?)272 }273}274275276pub(crate) struct SpanHead {277 count: Projection,278 embed: CountEmbed,279 marker: SpanMarker,280}281282impl SpanHead {283 284 pub(crate) fn load(285 vb: &VarBuilder,286 hidden: usize,287 max_width: usize,288 _device: &candle_core::Device,289 ) -> CandleResult<Self> {290 let (count_first, count_second) = mlp(vb, "count_pred", hidden, hidden * 2, 20)?;291 Ok(Self {292 count: Projection {293 first: count_first,294 second: count_second,295 },296 embed: CountEmbed::load(vb, hidden)?,297 marker: SpanMarker::load(vb, hidden, max_width)?,298 })299 }300301 302 #[cfg(test)]303 pub(crate) fn empty() -> Self {304 Self {305 count: Projection {306 first: Linear::new(307 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)308 .expect("empty"),309 None,310 ),311 second: Linear::new(312 Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)313 .expect("empty"),314 None,315 ),316 },317 embed: CountEmbed::empty(),318 marker: SpanMarker::empty(),319 }320 }321322 323 pub(crate) fn predicted_count(&self, query: &Tensor) -> CandleResult<usize> {324 let scores = self.count.forward(query)?.flatten_all()?.to_vec1::<f32>()?;325 let mut best = 0;326 for (index, score) in scores.iter().enumerate() {327 if *score > scores[best] {328 best = index;329 }330 }331 Ok(best)332 }333334 335 336 337 pub(crate) fn score(338 &self,339 token_states: &Tensor,340 query: &Tensor,341 fields: &Tensor,342 ) -> CandleResult<Tensor> {343 let count = self344 .predicted_count(query)?345 .min(self.embed.max_count)346 .max(1);347 let span_rep = self.marker.forward(token_states)?;348 let queries = self.embed.forward(fields, count)?;349 let (_, field_count, hidden) = (queries.dim(0)?, queries.dim(1)?, queries.dim(2)?);350 let text_len = token_states.dim(0)?;351 let width = self.marker.max_width;352353 354 355 356 357 let queries = queries.reshape((count * field_count, hidden))?;358 let span_rep = span_rep.reshape((text_len * width, hidden))?.t()?;359 sigmoid(360 &queries361 .matmul(&span_rep)?362 .reshape((count, field_count, text_len, width))?,363 )364 }365}