Skip to content
1        10        20        30        40        50        60        70        80        901//! The span half of the checkpoint: count head, count embedding, and the span2//! marker that scores candidate spans.3//!4//! Ported from `gliner2.layers` (`CountLSTM`, `SpanMarkerV0`, `create_mlp`) and5//! `gliner2.inference.runtime._extract_structures`. Dropout is a no-op at6//! inference, and spans are scored per (instance, field) exactly as the Python7//! runtime does with `torch.einsum("lkd,bpd->bplk", ...)`.89use candle_core::{Error, Result as CandleResult, Tensor};10use candle_nn::{Linear, Module, VarBuilder};1112/// `create_mlp(input, [intermediate], output, dropout=0, activation="relu")`:13/// `Linear -> ReLU -> Linear`. `dropout=0` omits the dropout module, so the two14/// linear layers sit at indices 0 and 2 — that is `count_pred`,15/// `count_embed.projector` and `classifier`.16fn mlp(17    vb: &VarBuilder,18    name: &str,19    in_features: usize,20    intermediate: usize,21    out_features: usize,22) -> CandleResult<(Linear, Linear)> {23    let first_weight = vb.get((intermediate, in_features), &format!("{name}.0.weight"))?;24    let first_bias = vb.get(intermediate, &format!("{name}.0.bias"))?;25    let second_weight = vb.get((out_features, intermediate), &format!("{name}.2.weight"))?;26    let second_bias = vb.get(out_features, &format!("{name}.2.bias"))?;27    Ok((28        Linear::new(first_weight, Some(first_bias)),29        Linear::new(second_weight, Some(second_bias)),30    ))31}3233fn gate_error() -> Error {34    Error::Msg("the GRU gates are not split three ways".to_owned())35}3637/// `torch.sigmoid`, which candle has no helper for.38fn sigmoid(x: &Tensor) -> CandleResult<Tensor> {39    x.neg()?.exp()?.affine(1.0, 1.0)?.recip()40}4142/// `nn.Linear` on the last axis, matching `F.linear`.43fn linear(input: &Tensor, weight: &Tensor, bias: &Tensor) -> CandleResult<Tensor> {44    input.matmul(&weight.t()?)?.broadcast_add(bias)45}4647/// A projection with a `ReLU` in the middle, the shape `create_projection_layer` builds.48struct Projection {49    first: Linear,50    second: Linear,51}5253impl Projection {54    fn forward(&self, input: &Tensor) -> CandleResult<Tensor> {55        self.second.forward(&self.first.forward(input)?.relu()?)56    }57}5859/// `create_projection_layer(input, dropout, out_dim)`: `Linear -> ReLU ->60/// Dropout -> Linear`. Unlike `create_mlp` this one always inserts the dropout61/// module, so the second linear sits at index 3 — that is the span head.62fn projection_layer(63    vb: &VarBuilder,64    name: &str,65    in_features: usize,66    hidden: usize,67) -> CandleResult<Projection> {68    let first_weight = vb.get((hidden * 4, in_features), &format!("{name}.0.weight"))?;69    let first_bias = vb.get(hidden * 4, &format!("{name}.0.bias"))?;70    let second_weight = vb.get((hidden, hidden * 4), &format!("{name}.3.weight"))?;71    let second_bias = vb.get(hidden, &format!("{name}.3.bias"))?;72    Ok(Projection {73        first: Linear::new(first_weight, Some(first_bias)),74        second: Linear::new(second_weight, Some(second_bias)),75    })76}7778/// `self.span_rep`: `SpanMarkerV0`, projecting every start/end pair of a span.79struct SpanMarker {80    start: Projection,81    end: Projection,82    out: Projection,83    max_width: usize,84}8586impl SpanMarker {87    fn load(vb: &VarBuilder, hidden: usize, max_width: usize) -> CandleResult<Self> {88        let prefix = "span_rep.span_rep_layer";89        Ok(Self {90            start: projection_layer(vb, &format!("{prefix}.project_start"), hidden, hidden)?,91            end: projection_layer(vb, &format!("{prefix}.project_end"), hidden, hidden)?,92            out: projection_layer(vb, &format!("{prefix}.out_project"), hidden * 2, hidden)?,93            max_width,94        })95    }9697    /// A marker of empty weights, for tests that never run it.98    #[cfg(test)]99    fn empty() -> Self {100        Self {101            start: Projection {102                first: Linear::new(103                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)104                        .expect("empty"),105                    None,106                ),107                second: Linear::new(108                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)109                        .expect("empty"),110                    None,111                ),112            },113            end: Projection {114                first: Linear::new(115                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)116                        .expect("empty"),117                    None,118                ),119                second: Linear::new(120                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)121                        .expect("empty"),122                    None,123                ),124            },125            out: Projection {126                first: Linear::new(127                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)128                        .expect("empty"),129                    None,130                ),131                second: Linear::new(132                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)133                        .expect("empty"),134                    None,135                ),136            },137            max_width: 0,138        }139    }140141    /// Span representations for every `(start, width)` pair: `(text, width, hidden)`.142    fn forward(&self, token_states: &Tensor) -> CandleResult<Tensor> {143        let text_len = token_states.dim(0)?;144        let hidden = token_states.dim(1)?;145        let width = self.max_width;146        let device = token_states.device();147148        let start_rep = self.start.forward(token_states)?;149        let end_rep = self.end.forward(token_states)?;150151        // Span `(start, width)` reads `start_rep[start]` and `end_rep[start + width]`.152        // Ends past the end of the text are clamped; those spans are never scored153        // above the threshold because decoding checks `start + width < text_len`.154        let to_index = |value: usize| u32::try_from(value).unwrap_or(u32::MAX);155        let starts: Vec<u32> = (0..text_len)156            .flat_map(|start| vec![to_index(start); width])157            .collect();158        let ends: Vec<u32> = (0..text_len)159            .flat_map(|start| {160                (0..width).map(move |offset| to_index((start + offset).min(text_len - 1)))161            })162            .collect();163164        let start_spans = start_rep.index_select(&Tensor::new(starts, device)?, 0)?;165        let end_spans = end_rep.index_select(&Tensor::new(ends, device)?, 0)?;166        let joined = Tensor::cat(&[&start_spans, &end_spans], 1)?.relu()?;167        self.out168            .forward(&joined)?169            .reshape((text_len, width, hidden))170    }171}172173/// `self.count_embed`: `CountLSTM` — one learned query per predicted instance.174struct CountEmbed {175    pos_embedding: Tensor,176    gru_input: Tensor,177    gru_hidden: Tensor,178    bias_input: Tensor,179    bias_hidden: Tensor,180    projector: Projection,181    max_count: usize,182}183184impl CountEmbed {185    fn load(vb: &VarBuilder, hidden: usize) -> CandleResult<Self> {186        let (first, second) = mlp(vb, "count_embed.projector", hidden * 2, hidden * 4, hidden)?;187        Ok(Self {188            pos_embedding: vb.get((20, hidden), "count_embed.pos_embedding.weight")?,189            gru_input: vb.get((hidden * 3, hidden), "count_embed.gru.weight_ih_l0")?,190            gru_hidden: vb.get((hidden * 3, hidden), "count_embed.gru.weight_hh_l0")?,191            bias_input: vb.get(hidden * 3, "count_embed.gru.bias_ih_l0")?,192            bias_hidden: vb.get(hidden * 3, "count_embed.gru.bias_hh_l0")?,193            projector: Projection { first, second },194            max_count: 20,195        })196    }197198    /// An embedding of empty weights, for tests that never run it.199    #[cfg(test)]200    fn empty() -> Self {201        Self {202            pos_embedding: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)203                .expect("an empty tensor"),204            gru_input: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)205                .expect("an empty tensor"),206            gru_hidden: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)207                .expect("an empty tensor"),208            bias_input: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)209                .expect("an empty tensor"),210            bias_hidden: Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)211                .expect("an empty tensor"),212            projector: Projection {213                first: Linear::new(214                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)215                        .expect("empty"),216                    None,217                ),218                second: Linear::new(219                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)220                        .expect("empty"),221                    None,222                ),223            },224            max_count: 0,225        }226    }227228    /// `(count, fields, hidden)` queries, seeded by `CountLSTM`'s GRU.229    fn forward(&self, fields: &Tensor, count: usize) -> CandleResult<Tensor> {230        let count = count.min(self.max_count).max(1);231        let field_count = fields.dim(0)?;232        let hidden = fields.dim(1)?;233        let device = fields.device();234235        let positions = self.pos_embedding.narrow(0, 0, count)?.to_device(device)?;236        let sequence = positions237            .unsqueeze(1)?238            .broadcast_as((count, field_count, hidden))?;239240        let mut state = fields.clone();241        let mut steps = Vec::with_capacity(count);242        for step in 0..count {243            let input = sequence.narrow(0, step, 1)?.squeeze(0)?;244            let input_gate = linear(&input, &self.gru_input, &self.bias_input)?;245            let hidden_gate = linear(&state, &self.gru_hidden, &self.bias_hidden)?;246            let mut input_gate = input_gate.chunk(3, 1)?.into_iter();247            let mut hidden_gate = hidden_gate.chunk(3, 1)?.into_iter();248            let input_reset = input_gate.next().ok_or_else(gate_error)?;249            let input_update = input_gate.next().ok_or_else(gate_error)?;250            let input_new = input_gate.next().ok_or_else(gate_error)?;251            let hidden_reset = hidden_gate.next().ok_or_else(gate_error)?;252            let hidden_update = hidden_gate.next().ok_or_else(gate_error)?;253            let hidden_new = hidden_gate.next().ok_or_else(gate_error)?;254            // GRU: h = (1 - z) * n + z * h, with n = tanh(i_n + r * h_n).255            let reset = sigmoid(&input_reset.broadcast_add(&hidden_reset)?)?;256            let update = sigmoid(&input_update.broadcast_add(&hidden_update)?)?;257            let candidate = input_new258                .broadcast_add(&reset.broadcast_mul(&hidden_new)?)?259                .tanh()?;260            // h = (1 - z) * n + z * h261            let closed = update.affine(-1.0, 1.0)?;262            state = closed263                .broadcast_mul(&candidate)?264                .broadcast_add(&update.broadcast_mul(&state)?)?;265            steps.push(state.clone());266        }267268        let sequence = Tensor::stack(&steps, 0)?;269        let fields = fields.unsqueeze(0)?.broadcast_as(sequence.dims())?;270        self.projector271            .forward(&Tensor::cat(&[&sequence, &fields], 2)?)272    }273}274275/// The three heads that turn encoder states into scored spans.276pub(crate) struct SpanHead {277    count: Projection,278    embed: CountEmbed,279    marker: SpanMarker,280}281282impl SpanHead {283    /// Read `count_pred.*`, `count_embed.*` and `span_rep.*` from the checkpoint.284    pub(crate) fn load(285        vb: &VarBuilder,286        hidden: usize,287        max_width: usize,288        _device: &candle_core::Device,289    ) -> CandleResult<Self> {290        let (count_first, count_second) = mlp(vb, "count_pred", hidden, hidden * 2, 20)?;291        Ok(Self {292            count: Projection {293                first: count_first,294                second: count_second,295            },296            embed: CountEmbed::load(vb, hidden)?,297            marker: SpanMarker::load(vb, hidden, max_width)?,298        })299    }300301    /// A head of empty weights, for tests that never run it.302    #[cfg(test)]303    pub(crate) fn empty() -> Self {304        Self {305            count: Projection {306                first: Linear::new(307                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)308                        .expect("empty"),309                    None,310                ),311                second: Linear::new(312                    Tensor::zeros(0, candle_core::DType::F32, &candle_core::Device::Cpu)313                        .expect("empty"),314                    None,315                ),316            },317            embed: CountEmbed::empty(),318            marker: SpanMarker::empty(),319        }320    }321322    /// Number of instances the model expects in this text.323    pub(crate) fn predicted_count(&self, query: &Tensor) -> CandleResult<usize> {324        let scores = self.count.forward(query)?.flatten_all()?.to_vec1::<f32>()?;325        let mut best = 0;326        for (index, score) in scores.iter().enumerate() {327            if *score > scores[best] {328                best = index;329            }330        }331        Ok(best)332    }333334    /// Score every span of every field of every instance.335    ///336    /// Returns `(instances, fields, text_len, width)` sigmoid scores.337    pub(crate) fn score(338        &self,339        token_states: &Tensor,340        query: &Tensor,341        fields: &Tensor,342    ) -> CandleResult<Tensor> {343        let count = self344            .predicted_count(query)?345            .min(self.embed.max_count)346            .max(1);347        let span_rep = self.marker.forward(token_states)?;348        let queries = self.embed.forward(fields, count)?;349        let (_, field_count, hidden) = (queries.dim(0)?, queries.dim(1)?, queries.dim(2)?);350        let text_len = token_states.dim(0)?;351        let width = self.marker.max_width;352353        // `torch.einsum("lkd,bpd->bplk")`: one row per (instance, field) against354        // every (start, width) span representation. The transpose matters —355        // reshaping `(text, width, hidden)` to `(hidden, text * width)` would356        // reinterpret memory instead of moving the axis.357        let queries = queries.reshape((count * field_count, hidden))?;358        let span_rep = span_rep.reshape((text_len * width, hidden))?.t()?;359        sigmoid(360            &queries361                .matmul(&span_rep)?362                .reshape((count, field_count, text_len, width))?,363        )364    }365}