mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 23:45:50 +00:00
116 lines
3.6 KiB
Rust
116 lines
3.6 KiB
Rust
//! Trainable LoRA linear layer.
|
|
//!
|
|
//! Base weights are frozen plain [`Tensor`]s (no autograd tracking); the LoRA
|
|
//! `A` / `B` factors are tracked tensors sourced from a [`hanzo_nn::VarMap`], so
|
|
//! only they receive gradients. Shapes match the engine's inference-side merge
|
|
//! (`hanzo_quant::lora::merge_lora_weights`): `A` is `(rank, in)`, `B` is `(out, rank)`,
|
|
//! and the delta is `scale * (x·Aᵀ)·Bᵀ` with `scale = alpha / rank`.
|
|
|
|
use hanzo_ml::{Module, Result, Tensor};
|
|
use hanzo_nn::Linear;
|
|
|
|
/// The trainable low-rank factors attached to a projection.
|
|
#[derive(Clone)]
|
|
pub struct LoraDelta {
|
|
/// Fully-qualified base name of the adapted module, e.g.
|
|
/// `model.layers.0.self_attn.q_proj`. Used to name the saved adapter tensors.
|
|
pub name: String,
|
|
/// `A`: `(rank, in_dim)`, tracked.
|
|
pub a: Tensor,
|
|
/// `B`: `(out_dim, rank)`, tracked.
|
|
pub b: Tensor,
|
|
pub scale: f64,
|
|
}
|
|
|
|
impl LoraDelta {
|
|
fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
|
// Reuse Linear's batched matmul (handles 2D/3D/4D inputs) for both factors.
|
|
let down = Linear::new(self.a.clone(), None).forward(x)?; // (.., rank)
|
|
let up = Linear::new(self.b.clone(), None).forward(&down)?; // (.., out)
|
|
up * self.scale
|
|
}
|
|
}
|
|
|
|
/// A linear projection with a frozen base weight and an optional trainable LoRA delta.
|
|
#[derive(Clone)]
|
|
pub struct LoraLinear {
|
|
base: Linear,
|
|
delta: Option<LoraDelta>,
|
|
}
|
|
|
|
impl LoraLinear {
|
|
pub fn frozen(base: Linear) -> Self {
|
|
Self { base, delta: None }
|
|
}
|
|
|
|
pub fn with_delta(base: Linear, delta: LoraDelta) -> Self {
|
|
Self {
|
|
base,
|
|
delta: Some(delta),
|
|
}
|
|
}
|
|
|
|
/// The adapter factors, if this layer is trainable — used when saving.
|
|
pub fn delta(&self) -> Option<&LoraDelta> {
|
|
self.delta.as_ref()
|
|
}
|
|
}
|
|
|
|
impl Module for LoraLinear {
|
|
fn forward(&self, x: &Tensor) -> Result<Tensor> {
|
|
let out = self.base.forward(x)?;
|
|
match &self.delta {
|
|
None => Ok(out),
|
|
Some(delta) => out + delta.forward(x)?,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// LoRA factor shapes for a projection of `in_dim -> out_dim` at the given rank.
|
|
/// Returns `(A_shape, B_shape)` = `((rank, in_dim), (out_dim, rank))`.
|
|
pub fn lora_shapes(rank: usize, in_dim: usize, out_dim: usize) -> ((usize, usize), (usize, usize)) {
|
|
((rank, in_dim), (out_dim, rank))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn shapes_match_engine_merge_convention() {
|
|
// in=8, out=4, rank=2 -> A=(2,8), B=(4,2); B·A = (4,8) = weight delta.
|
|
let (a, b) = lora_shapes(2, 8, 4);
|
|
assert_eq!(a, (2, 8));
|
|
assert_eq!(b, (4, 2));
|
|
// B(out,rank) · A(rank,in) is conformable and yields (out,in).
|
|
assert_eq!(b.1, a.0);
|
|
assert_eq!((b.0, a.1), (4, 8));
|
|
}
|
|
|
|
#[test]
|
|
fn zero_b_gives_zero_delta() {
|
|
use hanzo_ml::{Device, Var};
|
|
let dev = Device::Cpu;
|
|
// A random, B = 0 => initial delta must be exactly zero (base preserved).
|
|
let a = Var::randn(0f32, 1f32, (2, 8), &dev)
|
|
.unwrap()
|
|
.as_tensor()
|
|
.clone();
|
|
let b = Var::zeros((4, 2), hanzo_ml::DType::F32, &dev)
|
|
.unwrap()
|
|
.as_tensor()
|
|
.clone();
|
|
let delta = LoraDelta {
|
|
name: "t".into(),
|
|
a,
|
|
b,
|
|
scale: 4.0,
|
|
};
|
|
let x = Tensor::randn(0f32, 1f32, (3, 8), &dev).unwrap();
|
|
let out = delta.forward(&x).unwrap();
|
|
let hi = out.max_all().unwrap().to_scalar::<f32>().unwrap();
|
|
let lo = out.min_all().unwrap().to_scalar::<f32>().unwrap();
|
|
assert_eq!((hi, lo), (0.0, 0.0));
|
|
}
|
|
}
|