LINE1 10 20 30 40 50 60 70 80 90 100123456789use std::path::Path;1011use candle_core::{DType, Device, Result as CandleResult, Tensor};12use candle_nn::{LayerNorm, Linear, Module, VarBuilder};13use serde::Deserialize;141516#[derive(Debug, Clone, Deserialize)]17pub struct EncoderConfig {18 pub(crate) vocab_size: usize,19 pub(crate) hidden_size: usize,20 pub(crate) num_hidden_layers: usize,21 pub(crate) num_attention_heads: usize,22 pub(crate) intermediate_size: usize,23 pub(crate) hidden_act: String,24 pub(crate) layer_norm_eps: f64,25 pub(crate) max_position_embeddings: usize,26 pub(crate) relative_attention: bool,27 #[serde(default)]28 pub(crate) share_att_key: bool,29 #[serde(default = "default_buckets")]30 pub(crate) position_buckets: isize,31 #[serde(default = "default_relative_positions")]32 pub(crate) max_relative_positions: isize,33 #[serde(default)]34 pub(crate) pos_att_type: Vec<String>,35 #[serde(default)]36 pub(crate) norm_rel_ebd: Option<String>,37 #[serde(default = "default_true")]38 pub(crate) position_biased_input: bool,39 #[serde(default)]40 pub(crate) type_vocab_size: usize,41 #[serde(default)]42 pub(crate) embedding_size: Option<usize>,43}4445const fn default_buckets() -> isize {46 -147}4849const fn default_relative_positions() -> isize {50 -151}5253const fn default_true() -> bool {54 true55}5657impl EncoderConfig {58 59 fn check(&self) -> Result<(), String> {60 let unsupported = self.position_biased_input61 || self.type_vocab_size > 062 || self63 .embedding_size64 .is_some_and(|size| size != self.hidden_size)65 || !self.relative_attention66 || !self.share_att_key67 || self.position_buckets <= 068 || self.hidden_act != "gelu"69 || !self.pos_att_type.iter().any(|kind| kind == "c2p")70 || !self.pos_att_type.iter().any(|kind| kind == "p2c");71 if unsupported {72 return Err(73 "unsupported encoder configuration: this port implements DeBERTa-v3 \74 (relative attention with c2p and p2c, shared attention keys, gelu, no \75 position or token-type embeddings)"76 .to_owned(),77 );78 }79 if !self.hidden_size.is_multiple_of(self.num_attention_heads) {80 return Err("hidden_size is not a multiple of num_attention_heads".to_owned());81 }82 Ok(())83 }8485 fn max_relative_positions(&self) -> i64 {86 if self.max_relative_positions < 1 {87 i64::try_from(self.max_position_embeddings).unwrap_or(512)88 } else {89 i64::try_from(self.max_relative_positions).unwrap_or(512)90 }91 }92}9394struct LayerWeights {95 query: Linear,96 key: Linear,97 value: Linear,98 attention_output: Linear,99 attention_norm: LayerNorm,100 intermediate: Linear,101 output: Linear,102 output_norm: LayerNorm,103}104105/// Relative-position gather indices, shared by every layer of one sequence.106struct RelativePositions {107 content_to_position: Tensor,108 position_to_content: Tensor,109}110111/// `GLiNER`'s classification head: `Linear -> ReLU -> Linear`.112pub struct ClassifierHead {113 first: Linear,114 second: Linear,115}116117impl ClassifierHead {118 /// One logit per row of `embeddings`.119 ///120 /// # Errors121 /// Returns a candle error when the tensor shapes disagree.122 pub fn forward(&self, embeddings: &Tensor) -> CandleResult<Vec<f32>> {123 let hidden = self.first.forward(embeddings)?.relu()?;124 self.second125 .forward(&hidden)?126 .flatten_all()?127 .to_vec1::<f32>()128 }129130 /// A head of empty weights, for tests that never run it.131 #[cfg(test)]132 pub(crate) fn empty() -> Self {133 Self {134 first: Linear::new(135 Tensor::zeros(0, DType::F32, &Device::Cpu).expect("empty"),136 None,137 ),138 second: Linear::new(139 Tensor::zeros(0, DType::F32, &Device::Cpu).expect("empty"),140 None,141 ),142 }143 }144}145146/// The DeBERTa-v3 encoder.147pub struct Encoder {148 device: Device,149 config: EncoderConfig,150 heads: usize,151 head_size: usize,152 /// `sqrt(head_size * scale_factor)`, applied to the content and relative terms.153 #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]154 scale: f32,155 att_span: i64,156 word_embeddings: Tensor,157 embeddings_norm: LayerNorm,158 rel_embeddings: Tensor,159 rel_norm: Option<LayerNorm>,160 layers: Vec<LayerWeights>,161}162163/// Read `encoder_config/config.json`.164///165/// # Errors166/// Returns a message when the file is missing, unparsable, or describes an167/// encoder this port does not implement.168pub fn read_config(config_path: &Path) -> Result<EncoderConfig, String> {169 let raw = std::fs::read_to_string(config_path)170 .map_err(|error| format!("cannot read {}: {error}", config_path.display()))?;171 let config: EncoderConfig = serde_json::from_str(&raw)172 .map_err(|error| format!("cannot parse {}: {error}", config_path.display()))?;173 config.check()?;174 Ok(config)175}176177/// Memory-map the checkpoint and build the encoder plus the classification head.178///179/// # Errors180/// Returns a candle error when a weight is missing or has an unexpected shape.181pub fn load(182 config: &EncoderConfig,183 weights_path: &Path,184 device: &Device,185) -> CandleResult<(Encoder, ClassifierHead)> {186 let hidden = config.hidden_size;187 let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[weights_path], DType::F32, device) }?;188189 let word_embeddings = vb.get(190 (config.vocab_size, hidden),191 "encoder.embeddings.word_embeddings.weight",192 )?;193 let embeddings_norm = layer_norm(&vb, "encoder.embeddings.LayerNorm", hidden, config)?;194 let rel_embeddings = vb.get(195 (196 usize::try_from(config.position_buckets).unwrap_or(0) * 2,197 hidden,198 ),199 "encoder.encoder.rel_embeddings.weight",200 )?;201 let rel_norm = if config202 .norm_rel_ebd203 .as_deref()204 .is_some_and(|value| value.to_lowercase().contains("layer_norm"))205 {206 Some(layer_norm(207 &vb,208 "encoder.encoder.LayerNorm",209 hidden,210 config,211 )?)212 } else {213 None214 };215216 let mut layers = Vec::with_capacity(config.num_hidden_layers);217 for index in 0..config.num_hidden_layers {218 layers.push(LayerWeights::load(&vb, config, index)?);219 }220221 let head = ClassifierHead {222 first: linear(&vb, hidden * 2, hidden, "classifier.0")?,223 second: linear(&vb, 1, hidden * 2, "classifier.2")?,224 };225226 let heads = config.num_attention_heads;227 let head_size = hidden / heads;228 let mut scale_factor = 1.0;229 for kind in &config.pos_att_type {230 if kind == "c2p" || kind == "p2c" {231 scale_factor += 1.0;232 }233 }234235 let encoder = Encoder {236 device: device.clone(),237 config: config.clone(),238 heads,239 head_size,240 #[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]241 scale: (head_size as f32 * scale_factor as f32).sqrt(),242 att_span: config.position_buckets as i64,243 word_embeddings,244 embeddings_norm,245 rel_embeddings,246 rel_norm,247 layers,248 };249 Ok((encoder, head))250}251252impl LayerWeights {253 /// Read one `encoder.encoder.layer.{index}` block out of the checkpoint.254 fn load(vb: &VarBuilder, config: &EncoderConfig, index: usize) -> CandleResult<Self> {255 let hidden = config.hidden_size;256 let prefix = format!("encoder.encoder.layer.{index}");257 Ok(Self {258 query: linear(259 vb,260 hidden,261 hidden,262 &format!("{prefix}.attention.self.query_proj"),263 )?,264 key: linear(265 vb,266 hidden,267 hidden,268 &format!("{prefix}.attention.self.key_proj"),269 )?,270 value: linear(271 vb,272 hidden,273 hidden,274 &format!("{prefix}.attention.self.value_proj"),275 )?,276 attention_output: linear(277 vb,278 hidden,279 hidden,280 &format!("{prefix}.attention.output.dense"),281 )?,282 attention_norm: layer_norm(283 vb,284 &format!("{prefix}.attention.output.LayerNorm"),285 hidden,286 config,287 )?,288 intermediate: linear(289 vb,290 config.intermediate_size,291 hidden,292 &format!("{prefix}.intermediate.dense"),293 )?,294 output: linear(295 vb,296 hidden,297 config.intermediate_size,298 &format!("{prefix}.output.dense"),299 )?,300 output_norm: layer_norm(vb, &format!("{prefix}.output.LayerNorm"), hidden, config)?,301 })302 }303}304305/// Read an element-wise affine `LayerNorm` by its checkpoint path.306fn layer_norm(307 vb: &VarBuilder,308 path: &str,309 size: usize,310 config: &EncoderConfig,311) -> CandleResult<LayerNorm> {312 Ok(LayerNorm::new(313 vb.get(size, &format!("{path}.weight"))?,314 vb.get(size, &format!("{path}.bias"))?,315 config.layer_norm_eps,316 ))317}318319fn linear(vb: &VarBuilder, out: usize, in_features: usize, name: &str) -> CandleResult<Linear> {320 let weight = vb.get((out, in_features), &format!("{name}.weight"))?;321 let bias = vb.get(out, &format!("{name}.bias"))?;322 Ok(Linear::new(weight, Some(bias)))323}324325impl Encoder {326 /// Encode `input_ids` into per-token hidden states of shape `(len, hidden)`.327 ///328 /// # Errors329 /// Returns a candle error when the forward pass fails, for example on an330 /// out-of-range token id.331 pub fn forward(&self, input_ids: &[u32]) -> CandleResult<Tensor> {332 let ids = Tensor::new(input_ids.to_vec(), &self.device)?;333 let hidden = self334 .embeddings_norm335 .forward(&self.word_embeddings.embedding(&ids)?)?;336337 let rel = match &self.rel_norm {338 Some(norm) => norm.forward(&self.rel_embeddings)?,339 None => self.rel_embeddings.clone(),340 };341 let rel = rel.narrow(0, 0, span_rows(self.att_span))?;342 let positions = self.relative_positions(input_ids.len())?;343344 let mut state = hidden;345 for layer in &self.layers {346 state = self.layer_forward(layer, &state, &rel, &positions)?;347 }348 Ok(state)349 }350351 /// An encoder of empty weights, for tests that never run it.352 #[cfg(test)]353 pub(crate) fn empty() -> Self {354 let device = Device::Cpu;355 let config = EncoderConfig {356 vocab_size: 0,357 hidden_size: 0,358 num_hidden_layers: 0,359 num_attention_heads: 1,360 intermediate_size: 0,361 hidden_act: "gelu".to_owned(),362 layer_norm_eps: 1e-7,363 max_position_embeddings: 0,364 relative_attention: true,365 share_att_key: true,366 position_buckets: 256,367 max_relative_positions: -1,368 pos_att_type: ["c2p".to_owned(), "p2c".to_owned()].to_vec(),369 norm_rel_ebd: None,370 position_biased_input: false,371 type_vocab_size: 0,372 embedding_size: None,373 };374 let tensor = Tensor::zeros((0, 0), DType::F32, &device).expect("an empty tensor");375 Self {376 scale: 1.0,377 att_span: 0,378 config,379 heads: 0,380 head_size: 0,381 word_embeddings: tensor.clone(),382 embeddings_norm: LayerNorm::new(383 Tensor::zeros(0, DType::F32, &device).expect("an empty vector"),384 tensor.clone(),385 1e-7,386 ),387 rel_embeddings: tensor,388 rel_norm: None,389 layers: Vec::new(),390 device,391 }392 }393394 fn layer_forward(395 &self,396 layer: &LayerWeights,397 hidden: &Tensor,398 rel: &Tensor,399 positions: &RelativePositions,400 ) -> CandleResult<Tensor> {401 let query = self.split_heads(&layer.query.forward(hidden)?)?;402 let key = self.split_heads(&layer.key.forward(hidden)?)?;403 let value = self.split_heads(&layer.value.forward(hidden)?)?;404405 let scale = Tensor::new(self.scale, &self.device)?;406 let mut scores = query.matmul(&key.transpose(1, 2)?)?.broadcast_div(&scale)?;407408 if self.config.pos_att_type.iter().any(|kind| kind == "c2p") {409 let pos_key = self.split_heads(&layer.key.forward(rel)?)?;410 let bias = query411 .matmul(&pos_key.transpose(1, 2)?)?412 .gather(&positions.content_to_position, 2)?;413 scores = (&scores + &bias.broadcast_div(&scale)?)?;414 }415 if self.config.pos_att_type.iter().any(|kind| kind == "p2c") {416 let pos_query = self.split_heads(&layer.query.forward(rel)?)?;417 let bias = key418 .matmul(&pos_query.transpose(1, 2)?)?419 .gather(&positions.position_to_content, 2)?;420 let bias = bias.transpose(1, 2)?;421 scores = (&scores + &bias.broadcast_div(&scale)?)?;422 }423424 let context = candle_nn::ops::softmax(&scores, 2)?.matmul(&value)?;425 let context = self.merge_heads(&context)?;426427 let attention = layer.attention_output.forward(&context)?;428 let attention = layer.attention_norm.forward(&(&attention + hidden)?)?;429 let intermediate = layer.intermediate.forward(&attention)?.gelu_erf()?;430 let output = layer.output.forward(&intermediate)?;431 layer.output_norm.forward(&(&output + &attention)?)432 }433434 /// `(len, hidden)` -> `(heads, len, head_size)`.435 fn split_heads(&self, projected: &Tensor) -> CandleResult<Tensor> {436 let len = projected.dim(0)?;437 projected438 .reshape((len, self.heads, self.head_size))?439 .permute((1, 0, 2))?440 .contiguous()441 }442443 /// `(heads, len, head_size)` -> `(len, hidden)`.444 fn merge_heads(&self, context: &Tensor) -> CandleResult<Tensor> {445 let len = context.dim(1)?;446 context447 .permute((1, 0, 2))?448 .contiguous()?449 .reshape((len, self.config.hidden_size))450 }451452 /// `build_relative_position` plus the c2p/p2c gathers, for one sequence.453 #[allow(clippy::cast_possible_wrap)]454 fn relative_positions(&self, len: usize) -> CandleResult<RelativePositions> {455 let span = self.att_span;456 let buckets = self.config.position_buckets as i64;457 let max_relative = self.config.max_relative_positions();458 #[allow(clippy::cast_possible_wrap)]459 let len_i64 = len as i64;460 let total = self.heads * len * len;461462 let mut content_to_position = Vec::with_capacity(total);463 let mut position_to_content = Vec::with_capacity(total);464 for _ in 0..self.heads {465 for query in 0..len_i64 {466 for key in 0..len_i64 {467 let bucket = log_bucket_position(query - key, buckets, max_relative);468 content_to_position.push(clamp_index(bucket + span, span));469 position_to_content.push(clamp_index(-bucket + span, span));470 }471 }472 }473474 let shape = (self.heads, len, len);475 Ok(RelativePositions {476 content_to_position: Tensor::new(content_to_position, &self.device)?.reshape(shape)?,477 position_to_content: Tensor::new(position_to_content, &self.device)?.reshape(shape)?,478 })479 }480}481482/// The relative embedding slice is `2 * position_buckets` rows.483fn span_rows(att_span: i64) -> usize {484 usize::try_from(att_span * 2).unwrap_or(0)485}486487fn clamp_index(value: i64, span: i64) -> u32 {488 u32::try_from(value.clamp(0, 2 * span - 1)).unwrap_or(0)489}490491/// `make_log_bucket_position` from `modeling_deberta_v2.py`.492///493/// The arithmetic runs in `f32` on purpose: `PyTorch` evaluates the logarithm on494/// `float32` tensors there, and the ceiling makes small differences visible.495#[allow(496 clippy::cast_precision_loss,497 clippy::cast_possible_truncation,498 clippy::cast_sign_loss499)]500fn log_bucket_position(relative: i64, bucket_size: i64, max_position: i64) -> i64 {501 if bucket_size <= 0 || max_position <= 0 {502 return relative;503 }504 let mid = bucket_size / 2;505 let sign = relative.signum();506 let absolute = if relative < mid && relative > -mid {507 mid - 1508 } else {509 relative.abs()510 };511 let ratio = (absolute as f32 / mid as f32).ln() / ((max_position - 1) as f32 / mid as f32).ln();512 let logged = (ratio * (mid - 1) as f32).ceil() as i64 + mid;513 if absolute <= mid {514 relative515 } else {516 logged * sign517 }518}519520#[cfg(test)]521mod tests {522 use super::log_bucket_position;523524 #[test]525 fn near_positions_keep_their_distance() {526 for distance in -127..=127 {527 assert_eq!(log_bucket_position(distance, 256, 512), distance);528 }529 }530531 #[test]532 fn far_positions_share_buckets_per_side() {533 // Buckets for 256 positions, capped at 512: the values below are the ones534 // `make_log_bucket_position` produces for a 512 maximum.535 assert_eq!(log_bucket_position(128, 256, 512), 128);536 assert_eq!(log_bucket_position(-128, 256, 512), -128);537 assert_eq!(log_bucket_position(150, 256, 512), 143);538 assert_eq!(log_bucket_position(200, 256, 512), 169);539 assert_eq!(log_bucket_position(255, 256, 512), 192);540 assert_eq!(log_bucket_position(300, 256, 512), 207);541 }542543 #[test]544 fn buckets_are_monotonic_and_symmetric() {545 let mut previous = 0;546 for distance in 129..=511 {547 let bucket = log_bucket_position(distance, 256, 512);548 assert!(bucket >= previous, "{bucket} < {previous}");549 assert_eq!(bucket, -log_bucket_position(-distance, 256, 512));550 previous = bucket;551 }552 assert_eq!(previous, 255);553 }554}