Skip to content
1        10        20        30        40        50        60        70        80        90        1001//! A faithful DeBERTa-v3 encoder, ported from `transformers`' `deberta_v2`2//! modeling code so `fastino/GLiNER2.5-Decide` runs without Python.3//!4//! The checkpoint stores a `deberta-v2` model (`legacy` DeBERTa-v3): no5//! position or token-type embeddings, disentangled attention with relative6//! position bias (`c2p` + `p2c`), shared attention key/query projections and a7//! layer norm on the relative embeddings. Dropout is a no-op at inference.89use std::path::Path;1011use candle_core::{DType, Device, Result as CandleResult, Tensor};12use candle_nn::{LayerNorm, Linear, Module, VarBuilder};13use serde::Deserialize;1415/// Subset of `encoder_config/config.json` this port understands.16#[derive(Debug, Clone, Deserialize)]17pub struct EncoderConfig {18    pub(crate) vocab_size: usize,19    pub(crate) hidden_size: usize,20    pub(crate) num_hidden_layers: usize,21    pub(crate) num_attention_heads: usize,22    pub(crate) intermediate_size: usize,23    pub(crate) hidden_act: String,24    pub(crate) layer_norm_eps: f64,25    pub(crate) max_position_embeddings: usize,26    pub(crate) relative_attention: bool,27    #[serde(default)]28    pub(crate) share_att_key: bool,29    #[serde(default = "default_buckets")]30    pub(crate) position_buckets: isize,31    #[serde(default = "default_relative_positions")]32    pub(crate) max_relative_positions: isize,33    #[serde(default)]34    pub(crate) pos_att_type: Vec<String>,35    #[serde(default)]36    pub(crate) norm_rel_ebd: Option<String>,37    #[serde(default = "default_true")]38    pub(crate) position_biased_input: bool,39    #[serde(default)]40    pub(crate) type_vocab_size: usize,41    #[serde(default)]42    pub(crate) embedding_size: Option<usize>,43}4445const fn default_buckets() -> isize {46    -147}4849const fn default_relative_positions() -> isize {50    -151}5253const fn default_true() -> bool {54    true55}5657impl EncoderConfig {58    /// Reject anything that would silently produce wrong numbers.59    fn check(&self) -> Result<(), String> {60        let unsupported = self.position_biased_input61            || self.type_vocab_size > 062            || self63                .embedding_size64                .is_some_and(|size| size != self.hidden_size)65            || !self.relative_attention66            || !self.share_att_key67            || self.position_buckets <= 068            || self.hidden_act != "gelu"69            || !self.pos_att_type.iter().any(|kind| kind == "c2p")70            || !self.pos_att_type.iter().any(|kind| kind == "p2c");71        if unsupported {72            return Err(73                "unsupported encoder configuration: this port implements DeBERTa-v3 \74                 (relative attention with c2p and p2c, shared attention keys, gelu, no \75                 position or token-type embeddings)"76                    .to_owned(),77            );78        }79        if !self.hidden_size.is_multiple_of(self.num_attention_heads) {80            return Err("hidden_size is not a multiple of num_attention_heads".to_owned());81        }82        Ok(())83    }8485    fn max_relative_positions(&self) -> i64 {86        if self.max_relative_positions < 1 {87            i64::try_from(self.max_position_embeddings).unwrap_or(512)88        } else {89            i64::try_from(self.max_relative_positions).unwrap_or(512)90        }91    }92}9394struct LayerWeights {95    query: Linear,96    key: Linear,97    value: Linear,98    attention_output: Linear,99    attention_norm: LayerNorm,100    intermediate: Linear,101    output: Linear,102    output_norm: LayerNorm,103}104105/// Relative-position gather indices, shared by every layer of one sequence.106struct RelativePositions {107    content_to_position: Tensor,108    position_to_content: Tensor,109}110111/// `GLiNER`'s classification head: `Linear -> ReLU -> Linear`.112pub struct ClassifierHead {113    first: Linear,114    second: Linear,115}116117impl ClassifierHead {118    /// One logit per row of `embeddings`.119    ///120    /// # Errors121    /// Returns a candle error when the tensor shapes disagree.122    pub fn forward(&self, embeddings: &Tensor) -> CandleResult<Vec<f32>> {123        let hidden = self.first.forward(embeddings)?.relu()?;124        self.second125            .forward(&hidden)?126            .flatten_all()?127            .to_vec1::<f32>()128    }129130    /// A head of empty weights, for tests that never run it.131    #[cfg(test)]132    pub(crate) fn empty() -> Self {133        Self {134            first: Linear::new(135                Tensor::zeros(0, DType::F32, &Device::Cpu).expect("empty"),136                None,137            ),138            second: Linear::new(139                Tensor::zeros(0, DType::F32, &Device::Cpu).expect("empty"),140                None,141            ),142        }143    }144}145146/// The DeBERTa-v3 encoder.147pub struct Encoder {148    device: Device,149    config: EncoderConfig,150    heads: usize,151    head_size: usize,152    /// `sqrt(head_size * scale_factor)`, applied to the content and relative terms.153    #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]154    scale: f32,155    att_span: i64,156    word_embeddings: Tensor,157    embeddings_norm: LayerNorm,158    rel_embeddings: Tensor,159    rel_norm: Option<LayerNorm>,160    layers: Vec<LayerWeights>,161}162163/// Read `encoder_config/config.json`.164///165/// # Errors166/// Returns a message when the file is missing, unparsable, or describes an167/// encoder this port does not implement.168pub fn read_config(config_path: &Path) -> Result<EncoderConfig, String> {169    let raw = std::fs::read_to_string(config_path)170        .map_err(|error| format!("cannot read {}: {error}", config_path.display()))?;171    let config: EncoderConfig = serde_json::from_str(&raw)172        .map_err(|error| format!("cannot parse {}: {error}", config_path.display()))?;173    config.check()?;174    Ok(config)175}176177/// Memory-map the checkpoint and build the encoder plus the classification head.178///179/// # Errors180/// Returns a candle error when a weight is missing or has an unexpected shape.181pub fn load(182    config: &EncoderConfig,183    weights_path: &Path,184    device: &Device,185) -> CandleResult<(Encoder, ClassifierHead)> {186    let hidden = config.hidden_size;187    let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[weights_path], DType::F32, device) }?;188189    let word_embeddings = vb.get(190        (config.vocab_size, hidden),191        "encoder.embeddings.word_embeddings.weight",192    )?;193    let embeddings_norm = layer_norm(&vb, "encoder.embeddings.LayerNorm", hidden, config)?;194    let rel_embeddings = vb.get(195        (196            usize::try_from(config.position_buckets).unwrap_or(0) * 2,197            hidden,198        ),199        "encoder.encoder.rel_embeddings.weight",200    )?;201    let rel_norm = if config202        .norm_rel_ebd203        .as_deref()204        .is_some_and(|value| value.to_lowercase().contains("layer_norm"))205    {206        Some(layer_norm(207            &vb,208            "encoder.encoder.LayerNorm",209            hidden,210            config,211        )?)212    } else {213        None214    };215216    let mut layers = Vec::with_capacity(config.num_hidden_layers);217    for index in 0..config.num_hidden_layers {218        layers.push(LayerWeights::load(&vb, config, index)?);219    }220221    let head = ClassifierHead {222        first: linear(&vb, hidden * 2, hidden, "classifier.0")?,223        second: linear(&vb, 1, hidden * 2, "classifier.2")?,224    };225226    let heads = config.num_attention_heads;227    let head_size = hidden / heads;228    let mut scale_factor = 1.0;229    for kind in &config.pos_att_type {230        if kind == "c2p" || kind == "p2c" {231            scale_factor += 1.0;232        }233    }234235    let encoder = Encoder {236        device: device.clone(),237        config: config.clone(),238        heads,239        head_size,240        #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]241        scale: (head_size as f32 * scale_factor as f32).sqrt(),242        att_span: config.position_buckets as i64,243        word_embeddings,244        embeddings_norm,245        rel_embeddings,246        rel_norm,247        layers,248    };249    Ok((encoder, head))250}251252impl LayerWeights {253    /// Read one `encoder.encoder.layer.{index}` block out of the checkpoint.254    fn load(vb: &VarBuilder, config: &EncoderConfig, index: usize) -> CandleResult<Self> {255        let hidden = config.hidden_size;256        let prefix = format!("encoder.encoder.layer.{index}");257        Ok(Self {258            query: linear(259                vb,260                hidden,261                hidden,262                &format!("{prefix}.attention.self.query_proj"),263            )?,264            key: linear(265                vb,266                hidden,267                hidden,268                &format!("{prefix}.attention.self.key_proj"),269            )?,270            value: linear(271                vb,272                hidden,273                hidden,274                &format!("{prefix}.attention.self.value_proj"),275            )?,276            attention_output: linear(277                vb,278                hidden,279                hidden,280                &format!("{prefix}.attention.output.dense"),281            )?,282            attention_norm: layer_norm(283                vb,284                &format!("{prefix}.attention.output.LayerNorm"),285                hidden,286                config,287            )?,288            intermediate: linear(289                vb,290                config.intermediate_size,291                hidden,292                &format!("{prefix}.intermediate.dense"),293            )?,294            output: linear(295                vb,296                hidden,297                config.intermediate_size,298                &format!("{prefix}.output.dense"),299            )?,300            output_norm: layer_norm(vb, &format!("{prefix}.output.LayerNorm"), hidden, config)?,301        })302    }303}304305/// Read an element-wise affine `LayerNorm` by its checkpoint path.306fn layer_norm(307    vb: &VarBuilder,308    path: &str,309    size: usize,310    config: &EncoderConfig,311) -> CandleResult<LayerNorm> {312    Ok(LayerNorm::new(313        vb.get(size, &format!("{path}.weight"))?,314        vb.get(size, &format!("{path}.bias"))?,315        config.layer_norm_eps,316    ))317}318319fn linear(vb: &VarBuilder, out: usize, in_features: usize, name: &str) -> CandleResult<Linear> {320    let weight = vb.get((out, in_features), &format!("{name}.weight"))?;321    let bias = vb.get(out, &format!("{name}.bias"))?;322    Ok(Linear::new(weight, Some(bias)))323}324325impl Encoder {326    /// Encode `input_ids` into per-token hidden states of shape `(len, hidden)`.327    ///328    /// # Errors329    /// Returns a candle error when the forward pass fails, for example on an330    /// out-of-range token id.331    pub fn forward(&self, input_ids: &[u32]) -> CandleResult<Tensor> {332        let ids = Tensor::new(input_ids.to_vec(), &self.device)?;333        let hidden = self334            .embeddings_norm335            .forward(&self.word_embeddings.embedding(&ids)?)?;336337        let rel = match &self.rel_norm {338            Some(norm) => norm.forward(&self.rel_embeddings)?,339            None => self.rel_embeddings.clone(),340        };341        let rel = rel.narrow(0, 0, span_rows(self.att_span))?;342        let positions = self.relative_positions(input_ids.len())?;343344        let mut state = hidden;345        for layer in &self.layers {346            state = self.layer_forward(layer, &state, &rel, &positions)?;347        }348        Ok(state)349    }350351    /// An encoder of empty weights, for tests that never run it.352    #[cfg(test)]353    pub(crate) fn empty() -> Self {354        let device = Device::Cpu;355        let config = EncoderConfig {356            vocab_size: 0,357            hidden_size: 0,358            num_hidden_layers: 0,359            num_attention_heads: 1,360            intermediate_size: 0,361            hidden_act: "gelu".to_owned(),362            layer_norm_eps: 1e-7,363            max_position_embeddings: 0,364            relative_attention: true,365            share_att_key: true,366            position_buckets: 256,367            max_relative_positions: -1,368            pos_att_type: ["c2p".to_owned(), "p2c".to_owned()].to_vec(),369            norm_rel_ebd: None,370            position_biased_input: false,371            type_vocab_size: 0,372            embedding_size: None,373        };374        let tensor = Tensor::zeros((0, 0), DType::F32, &device).expect("an empty tensor");375        Self {376            scale: 1.0,377            att_span: 0,378            config,379            heads: 0,380            head_size: 0,381            word_embeddings: tensor.clone(),382            embeddings_norm: LayerNorm::new(383                Tensor::zeros(0, DType::F32, &device).expect("an empty vector"),384                tensor.clone(),385                1e-7,386            ),387            rel_embeddings: tensor,388            rel_norm: None,389            layers: Vec::new(),390            device,391        }392    }393394    fn layer_forward(395        &self,396        layer: &LayerWeights,397        hidden: &Tensor,398        rel: &Tensor,399        positions: &RelativePositions,400    ) -> CandleResult<Tensor> {401        let query = self.split_heads(&layer.query.forward(hidden)?)?;402        let key = self.split_heads(&layer.key.forward(hidden)?)?;403        let value = self.split_heads(&layer.value.forward(hidden)?)?;404405        let scale = Tensor::new(self.scale, &self.device)?;406        let mut scores = query.matmul(&key.transpose(1, 2)?)?.broadcast_div(&scale)?;407408        if self.config.pos_att_type.iter().any(|kind| kind == "c2p") {409            let pos_key = self.split_heads(&layer.key.forward(rel)?)?;410            let bias = query411                .matmul(&pos_key.transpose(1, 2)?)?412                .gather(&positions.content_to_position, 2)?;413            scores = (&scores + &bias.broadcast_div(&scale)?)?;414        }415        if self.config.pos_att_type.iter().any(|kind| kind == "p2c") {416            let pos_query = self.split_heads(&layer.query.forward(rel)?)?;417            let bias = key418                .matmul(&pos_query.transpose(1, 2)?)?419                .gather(&positions.position_to_content, 2)?;420            let bias = bias.transpose(1, 2)?;421            scores = (&scores + &bias.broadcast_div(&scale)?)?;422        }423424        let context = candle_nn::ops::softmax(&scores, 2)?.matmul(&value)?;425        let context = self.merge_heads(&context)?;426427        let attention = layer.attention_output.forward(&context)?;428        let attention = layer.attention_norm.forward(&(&attention + hidden)?)?;429        let intermediate = layer.intermediate.forward(&attention)?.gelu_erf()?;430        let output = layer.output.forward(&intermediate)?;431        layer.output_norm.forward(&(&output + &attention)?)432    }433434    /// `(len, hidden)` -> `(heads, len, head_size)`.435    fn split_heads(&self, projected: &Tensor) -> CandleResult<Tensor> {436        let len = projected.dim(0)?;437        projected438            .reshape((len, self.heads, self.head_size))?439            .permute((1, 0, 2))?440            .contiguous()441    }442443    /// `(heads, len, head_size)` -> `(len, hidden)`.444    fn merge_heads(&self, context: &Tensor) -> CandleResult<Tensor> {445        let len = context.dim(1)?;446        context447            .permute((1, 0, 2))?448            .contiguous()?449            .reshape((len, self.config.hidden_size))450    }451452    /// `build_relative_position` plus the c2p/p2c gathers, for one sequence.453    #[allow(clippy::cast_possible_wrap)]454    fn relative_positions(&self, len: usize) -> CandleResult<RelativePositions> {455        let span = self.att_span;456        let buckets = self.config.position_buckets as i64;457        let max_relative = self.config.max_relative_positions();458        #[allow(clippy::cast_possible_wrap)]459        let len_i64 = len as i64;460        let total = self.heads * len * len;461462        let mut content_to_position = Vec::with_capacity(total);463        let mut position_to_content = Vec::with_capacity(total);464        for _ in 0..self.heads {465            for query in 0..len_i64 {466                for key in 0..len_i64 {467                    let bucket = log_bucket_position(query - key, buckets, max_relative);468                    content_to_position.push(clamp_index(bucket + span, span));469                    position_to_content.push(clamp_index(-bucket + span, span));470                }471            }472        }473474        let shape = (self.heads, len, len);475        Ok(RelativePositions {476            content_to_position: Tensor::new(content_to_position, &self.device)?.reshape(shape)?,477            position_to_content: Tensor::new(position_to_content, &self.device)?.reshape(shape)?,478        })479    }480}481482/// The relative embedding slice is `2 * position_buckets` rows.483fn span_rows(att_span: i64) -> usize {484    usize::try_from(att_span * 2).unwrap_or(0)485}486487fn clamp_index(value: i64, span: i64) -> u32 {488    u32::try_from(value.clamp(0, 2 * span - 1)).unwrap_or(0)489}490491/// `make_log_bucket_position` from `modeling_deberta_v2.py`.492///493/// The arithmetic runs in `f32` on purpose: `PyTorch` evaluates the logarithm on494/// `float32` tensors there, and the ceiling makes small differences visible.495#[allow(496    clippy::cast_precision_loss,497    clippy::cast_possible_truncation,498    clippy::cast_sign_loss499)]500fn log_bucket_position(relative: i64, bucket_size: i64, max_position: i64) -> i64 {501    if bucket_size <= 0 || max_position <= 0 {502        return relative;503    }504    let mid = bucket_size / 2;505    let sign = relative.signum();506    let absolute = if relative < mid && relative > -mid {507        mid - 1508    } else {509        relative.abs()510    };511    let ratio = (absolute as f32 / mid as f32).ln() / ((max_position - 1) as f32 / mid as f32).ln();512    let logged = (ratio * (mid - 1) as f32).ceil() as i64 + mid;513    if absolute <= mid {514        relative515    } else {516        logged * sign517    }518}519520#[cfg(test)]521mod tests {522    use super::log_bucket_position;523524    #[test]525    fn near_positions_keep_their_distance() {526        for distance in -127..=127 {527            assert_eq!(log_bucket_position(distance, 256, 512), distance);528        }529    }530531    #[test]532    fn far_positions_share_buckets_per_side() {533        // Buckets for 256 positions, capped at 512: the values below are the ones534        // `make_log_bucket_position` produces for a 512 maximum.535        assert_eq!(log_bucket_position(128, 256, 512), 128);536        assert_eq!(log_bucket_position(-128, 256, 512), -128);537        assert_eq!(log_bucket_position(150, 256, 512), 143);538        assert_eq!(log_bucket_position(200, 256, 512), 169);539        assert_eq!(log_bucket_position(255, 256, 512), 192);540        assert_eq!(log_bucket_position(300, 256, 512), 207);541    }542543    #[test]544    fn buckets_are_monotonic_and_symmetric() {545        let mut previous = 0;546        for distance in 129..=511 {547            let bucket = log_bucket_position(distance, 256, 512);548            assert!(bucket >= previous, "{bucket} < {previous}");549            assert_eq!(bucket, -log_bucket_position(-distance, 256, 512));550            previous = bucket;551        }552        assert_eq!(previous, 255);553    }554}