mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
1101 lines
35 KiB
Rust
1101 lines
35 KiB
Rust
// Numeric gate for the UNIFIED ROCm quant decode core (qmatvec_core<WTYPE> in quant.hip). ONE
|
||
// quant-agnostic warp-per-row core + one per-type decode_block covers the whole 1-bit -> 8-bit
|
||
// zoo. This test proves a REPRESENTATIVE SPREAD bit-exact against the CPU oracle:
|
||
// - Q8_0 : symmetric 8-bit (the proven decode path, re-validated through the core)
|
||
// - Q4_0 : symmetric nibble (-8)
|
||
// - Q4_K : ASYMMETRIC super-block (exercises the 2nd accumulation shape: d·sc·q − dmin·m)
|
||
// - Q6_K : symmetric K-quant (6-bit, signed sub-scales)
|
||
// - IQ4_XS : symmetric codebook (exercises grid-LUT decode via KVALUES_IQ4NL)
|
||
// - TQ2_0 : symmetric ternary 2-bit (the sub-4-bit / 1-2-bit end)
|
||
//
|
||
// For EACH type the reference is the REAL CPU `to_float` (the `quant_format!` / GgmlType decoder),
|
||
// reinterpreting the same raw GGML bytes the GPU reads -- so the GPU decode is checked against the
|
||
// exact oracle, not a re-derivation. The kernel and reference both accumulate in f32; `nbad` counts
|
||
// outputs beyond a tiny magnitude-scaled tolerance and the gate requires nbad == 0 for every type.
|
||
// Lives in hanzo-cli so it links the HIP runtime via hanzo-cli's build.rs.
|
||
#![cfg(feature = "rocm")]
|
||
|
||
use half::{bf16, f16};
|
||
use hanzo_ml::backend::{BackendDevice, BackendStorage};
|
||
use hanzo_ml::quantized::iq_quants::BlockTQ2_0;
|
||
use hanzo_ml::quantized::k_quants::{
|
||
BlockIQ4xs, BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ4_1, BlockQ5K, BlockQ5_0, BlockQ5_1,
|
||
BlockQ6K, BlockQ8_0, BlockQ8_1,
|
||
};
|
||
use hanzo_ml::quantized::GgmlType;
|
||
use hanzo_ml::{RocmDevice, RocmQuantType, RocmStorage};
|
||
|
||
// Deterministic activation value in roughly [-3, 3] (same generator as the Q4_K / Q8_0 gates).
|
||
fn val(i: usize) -> f32 {
|
||
let x = ((i.wrapping_mul(2654435761) >> 8) & 0xffff) as f32 / 65535.0;
|
||
(x - 0.5) * 6.0
|
||
}
|
||
|
||
// Deterministic byte stream for building GGML blocks; varied so all nibbles/bits/scale lanes and
|
||
// the full quant range are exercised across rows.
|
||
fn byte(seed: usize) -> u8 {
|
||
(seed.wrapping_mul(2654435761).wrapping_add(seed >> 7) & 0xff) as u8
|
||
}
|
||
|
||
fn to_f32(s: &RocmStorage) -> Vec<f32> {
|
||
match s.to_cpu_storage().expect("to cpu") {
|
||
hanzo_ml::CpuStorage::F16(v) => v.iter().map(|x| x.to_f32()).collect(),
|
||
hanzo_ml::CpuStorage::BF16(v) => v.iter().map(|x| x.to_f32()).collect(),
|
||
hanzo_ml::CpuStorage::F32(v) => v,
|
||
other => panic!("unexpected dtype {:?}", other.dtype()),
|
||
}
|
||
}
|
||
|
||
// Reinterpret raw GGML bytes as whole `T` blocks with correct alignment (fs/byte buffers are only
|
||
// 1-byte aligned; several blocks have u16/f16 fields needing >=2). Copies bytes, does not change them.
|
||
fn as_blocks<T: Clone>(bytes: &[u8]) -> Vec<T> {
|
||
let sz = std::mem::size_of::<T>();
|
||
assert_eq!(
|
||
bytes.len() % sz,
|
||
0,
|
||
"byte length {} not a multiple of {}",
|
||
bytes.len(),
|
||
sz
|
||
);
|
||
let n = bytes.len() / sz;
|
||
let mut v: Vec<T> = Vec::with_capacity(n);
|
||
unsafe {
|
||
std::ptr::copy_nonoverlapping(bytes.as_ptr(), v.as_mut_ptr() as *mut u8, n * sz);
|
||
v.set_len(n);
|
||
}
|
||
v
|
||
}
|
||
|
||
// Build a valid GGML weight tensor's raw bytes for `n` rows x `k` cols of type `T` (block size
|
||
// `blk`, byte size `tsz`). Most bytes are pseudo-random, EXCEPT the leading per-block f16 scale
|
||
// `d` (and Q4_K's dmin), which we force to small exactly-f16-representable magnitudes so the
|
||
// reference and kernel f16 scales agree to the last bit and the outputs stay O(1..10s). The scale
|
||
// position differs per type (TQ2_0 trails its d), so each builder sets it explicitly.
|
||
fn build_bytes<F: Fn(&mut [u8], usize, usize)>(
|
||
n: usize,
|
||
k: usize,
|
||
blk: usize,
|
||
tsz: usize,
|
||
fill: F,
|
||
) -> Vec<u8> {
|
||
assert_eq!(k % blk, 0);
|
||
let nblk = k / blk;
|
||
let mut out = vec![0u8; n * nblk * tsz];
|
||
for r in 0..n {
|
||
for b in 0..nblk {
|
||
let base = (r * nblk + b) * tsz;
|
||
let block = &mut out[base..base + tsz];
|
||
// pseudo-random body
|
||
for (i, by) in block.iter_mut().enumerate() {
|
||
*by = byte(r * 7919 + b * 131 + i * 17 + 3);
|
||
}
|
||
fill(block, r, b);
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
// Put a small exact-f16 scale at byte `off` in the block. Returns nothing; mutates block.
|
||
fn put_d(block: &mut [u8], off: usize, r: usize, b: usize, lo: f32, step: f32, m: usize) {
|
||
let d = lo + step * (((r + b) % m) as f32);
|
||
let bytes = f16::from_f32(d).to_le_bytes();
|
||
block[off] = bytes[0];
|
||
block[off + 1] = bytes[1];
|
||
}
|
||
|
||
// q8_1 activation reconstruction mirroring the GPU `quantize_q8_1` (quant.hip) BIT-FOR-BIT: per
|
||
// 32-elem block, scale = f16(absmax/127), int8 q = round-half-away(x*127/absmax) clamped to [-127,127],
|
||
// recon = f32(scale)*q. The dp4a decode dots the EXACT-dequantized weight against THIS reconstructed
|
||
// activation, so it is the dp4a path's exact reference (the q8_1-vs-f16 gap is the activation
|
||
// quantization, orthogonal to the weight decode the scalar path proves bit-exact against to_float).
|
||
fn q8_1_recon(x: &[f32]) -> Vec<f32> {
|
||
let mut out = vec![0f32; x.len()];
|
||
for (xc, oc) in x.chunks(32).zip(out.chunks_mut(32)) {
|
||
let absmax = xc.iter().fold(0f32, |m, &v| m.max(v.abs()));
|
||
let inv = if absmax > 0.0 { 127.0 / absmax } else { 0.0 };
|
||
let d8 = f16::from_f32(absmax / 127.0).to_f32();
|
||
for (xv, ov) in xc.iter().zip(oc.iter_mut()) {
|
||
*ov = d8 * (xv * inv).round().clamp(-127.0, 127.0);
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
// Exact CPU reference: dequantize the SAME bytes via the real GgmlType::to_float, then dot row `nn`
|
||
// against the dtype-rounded activation `x`. Accumulation order: per block, sequential within block.
|
||
fn reference_row<T: GgmlType>(
|
||
wq: &[u8],
|
||
nn: usize,
|
||
nblk: usize,
|
||
blk: usize,
|
||
tsz: usize,
|
||
x: &[f32],
|
||
) -> f32 {
|
||
let mut acc = 0f32;
|
||
let mut deq = vec![0f32; blk];
|
||
for b in 0..nblk {
|
||
let base = (nn * nblk + b) * tsz;
|
||
let blocks = as_blocks::<T>(&wq[base..base + tsz]);
|
||
T::to_float(&blocks, &mut deq);
|
||
let xb = &x[b * blk..b * blk + blk];
|
||
for j in 0..blk {
|
||
acc += deq[j] * xb[j];
|
||
}
|
||
}
|
||
acc
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn check<T: GgmlType, F: Fn(&mut [u8], usize, usize)>(
|
||
dev: &RocmDevice,
|
||
log: &mut String,
|
||
name: &str,
|
||
qt: RocmQuantType,
|
||
n: usize,
|
||
k: usize,
|
||
blk: usize,
|
||
tsz: usize,
|
||
fill: F,
|
||
) {
|
||
assert_eq!(k % blk, 0, "{name}: k must be a multiple of {blk}");
|
||
let nblk = k / blk;
|
||
|
||
let xf: Vec<f32> = (0..k).map(val).collect();
|
||
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
|
||
let xb: Vec<bf16> = xf.iter().map(|&v| bf16::from_f32(v)).collect();
|
||
let xf_h: Vec<f32> = xh.iter().map(|v| v.to_f32()).collect();
|
||
let xf_b: Vec<f32> = xb.iter().map(|v| v.to_f32()).collect();
|
||
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
|
||
let xst_b = dev.storage_from_slice(&xb).expect("upload x bf16");
|
||
|
||
let wq_bytes = build_bytes(n, k, blk, tsz, fill);
|
||
assert_eq!(wq_bytes.len(), n * nblk * tsz);
|
||
let wst = dev.storage_from_slice(&wq_bytes).expect("upload w");
|
||
|
||
let y_h = dev
|
||
.matvec_quant(qt, &wst, &xst_h, n, k)
|
||
.expect("matvec_quant f16");
|
||
let y_b = dev
|
||
.matvec_quant(qt, &wst, &xst_b, n, k)
|
||
.expect("matvec_quant bf16");
|
||
assert_eq!(
|
||
y_b.dtype(),
|
||
hanzo_ml::DType::BF16,
|
||
"{name}: bf16 matvec keeps bf16"
|
||
);
|
||
assert_eq!(
|
||
y_h.dtype(),
|
||
hanzo_ml::DType::F16,
|
||
"{name}: f16 matvec keeps f16"
|
||
);
|
||
let yh = to_f32(&y_h);
|
||
let yb = to_f32(&y_b);
|
||
|
||
// dp4a-path types dot the weight against the q8_1-reconstructed activation; the scalar path uses
|
||
// the exact f16/bf16 activation. Match the reference to whichever the GPU actually ran so nbad=0
|
||
// means bit-exact (not "within a q8_1 fudge"). dp4a_active() reflects set_force_scalar_matvec.
|
||
let dp4a = qt.dp4a_active();
|
||
let ax_h = if dp4a {
|
||
q8_1_recon(&xf_h)
|
||
} else {
|
||
xf_h.clone()
|
||
};
|
||
let ax_b = if dp4a {
|
||
q8_1_recon(&xf_b)
|
||
} else {
|
||
xf_b.clone()
|
||
};
|
||
let mut ref_h = vec![0f32; n];
|
||
let mut ref_b = vec![0f32; n];
|
||
for nn in 0..n {
|
||
ref_h[nn] = reference_row::<T>(&wq_bytes, nn, nblk, blk, tsz, &ax_h);
|
||
ref_b[nn] = reference_row::<T>(&wq_bytes, nn, nblk, blk, tsz, &ax_b);
|
||
}
|
||
|
||
let scale = ref_b
|
||
.iter()
|
||
.chain(ref_h.iter())
|
||
.fold(0f32, |m, &v| m.max(v.abs()))
|
||
.max(1.0);
|
||
let tol = 0.01 * scale; // f16: 1% of magnitude, far below any layout/packing/scale bug, above f32 reorder.
|
||
let tol_b = 0.02 * scale; // bf16 has ~3 fewer mantissa bits than f16; 2% gate covers its worst-case rounding at large k.
|
||
|
||
let mut max_err_h = 0f32;
|
||
let mut max_err_b = 0f32;
|
||
let mut nbad = 0usize;
|
||
for i in 0..n {
|
||
let eh = (yh[i] - ref_h[i]).abs();
|
||
let eb = (yb[i] - ref_b[i]).abs();
|
||
max_err_h = max_err_h.max(eh);
|
||
max_err_b = max_err_b.max(eb);
|
||
if eh > tol || eb > tol_b {
|
||
nbad += 1;
|
||
}
|
||
}
|
||
log.push_str(&format!(
|
||
"{name:8} n={n} k={k} nbad={nbad}/{n} max_err_f16={max_err_h:.5} max_err_bf16={max_err_b:.5} (tol {tol:.5}) scale={scale:.3}\n"
|
||
));
|
||
assert!(
|
||
nbad == 0,
|
||
"{name} unified-core mismatch: n={n} k={k} nbad={nbad} max_err_f16={max_err_h} max_err_bf16={max_err_b} tol={tol}"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn qmatvec_unified_numeric() {
|
||
let mut log = String::new();
|
||
let dev = RocmDevice::new(0).expect("rocm device");
|
||
|
||
// Shapes: single block per row, production decode (4096), wide-k FFN, partial last warp-row (17),
|
||
// and a vocab-sized output. Each type runs the full set so partial tails + warp tiling are covered.
|
||
let shapes: &[(usize, usize)] = &[
|
||
(64, 256),
|
||
(4096, 4096),
|
||
(1024, 3072),
|
||
(17, 4096),
|
||
(4096, 256),
|
||
];
|
||
|
||
// Q8_0: 34 B, 32 elems. d (f16) at byte 0.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ8_0, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q8_0",
|
||
RocmQuantType::Q8_0,
|
||
n,
|
||
k,
|
||
32,
|
||
34,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
}
|
||
// Q4_0: 18 B, 32 elems. d (f16) at byte 0.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ4_0, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_0",
|
||
RocmQuantType::Q4_0,
|
||
n,
|
||
k,
|
||
32,
|
||
18,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
}
|
||
// Q4_K: 144 B, 256 elems. d (f16) at 0, dmin (f16) at 2. ASYMMETRIC.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ4K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_K",
|
||
RocmQuantType::Q4K,
|
||
n,
|
||
k,
|
||
256,
|
||
144,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
}
|
||
// Q6_K: 210 B, 256 elems. d (f16) at 208. scales[16] are signed i8 (random bytes OK). Q6_K and
|
||
// IQ4_XS both carry ±127-magnitude quants (6-bit signed scale / 8-bit codebook), so with random
|
||
// scale bytes a few of 4096 random rows would sum past the f16 output range (65504) and store
|
||
// inf -- not a decode bug (the math is bit-identical to the oracle), just an output-range
|
||
// artifact. A small `d` (~0.008, still exact-f16) keeps worst-case rows well inside f16, exactly
|
||
// as the proven Q4_K gate uses small exact scales; the full quant/scale RANGE is still exercised.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ6K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q6_K",
|
||
RocmQuantType::Q6K,
|
||
n,
|
||
k,
|
||
256,
|
||
210,
|
||
|blk, r, b| {
|
||
put_d(blk, 208, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
}
|
||
// IQ4_XS: 136 B, 256 elems. d (f16) at 0; scales_h (u16) at 2; scales_l[4] at 4; qs[128] at 8.
|
||
for &(n, k) in shapes {
|
||
check::<BlockIQ4xs, _>(
|
||
&dev,
|
||
&mut log,
|
||
"IQ4_XS",
|
||
RocmQuantType::IQ4_XS,
|
||
n,
|
||
k,
|
||
256,
|
||
136,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
}
|
||
// TQ2_0: 66 B, 256 elems. qs[64] at 0; d (f16) at 64.
|
||
for &(n, k) in shapes {
|
||
check::<BlockTQ2_0, _>(
|
||
&dev,
|
||
&mut log,
|
||
"TQ2_0",
|
||
RocmQuantType::TQ2_0,
|
||
n,
|
||
k,
|
||
256,
|
||
66,
|
||
|blk, r, b| {
|
||
put_d(blk, 64, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
}
|
||
// Q2_K: 84 B, 256 elems. scales[16] at 0, qs[64] at 16, d (f16) at 80, dmin (f16) at 82.
|
||
// ASYMMETRIC (4-bit scale + 4-bit min per 16-elem sub-block; q in 0..3). Random scale bytes
|
||
// exercise the full 4-bit scale/min range; small exact-f16 d/dmin keep outputs in f16 range.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ2K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q2_K",
|
||
RocmQuantType::Q2K,
|
||
n,
|
||
k,
|
||
256,
|
||
84,
|
||
|blk, r, b| {
|
||
put_d(blk, 80, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 82, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
}
|
||
// Q3_K: 110 B, 256 elems. hmask[32] at 0, qs[64] at 32, scales[12] at 96, d (f16) at 108.
|
||
// SYMMETRIC 3-bit (signed 6-bit per-16 scale, q in -4..3). Random hmask/qs/scale bytes exercise
|
||
// the high-bit + signed-scale reconstruction; small exact-f16 d keeps outputs in f16 range.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ3K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q3_K",
|
||
RocmQuantType::Q3K,
|
||
n,
|
||
k,
|
||
256,
|
||
110,
|
||
|blk, r, b| {
|
||
put_d(blk, 108, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
}
|
||
// SCALAR-forced vs the EXACT oracle: the ground-truth bit-exact weight-decode gate for the low-bit
|
||
// k-quants. With the scalar core forced, dp4a_active() is false so `check` runs the scalar core AND
|
||
// references the exact (non-q8_1) activation -- proving the decode itself is exact.
|
||
hanzo_ml::set_force_scalar_matvec(true);
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ2K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q2_K-scal",
|
||
RocmQuantType::Q2K,
|
||
n,
|
||
k,
|
||
256,
|
||
84,
|
||
|blk, r, b| {
|
||
put_d(blk, 80, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 82, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
}
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ3K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q3_K-scal",
|
||
RocmQuantType::Q3K,
|
||
n,
|
||
k,
|
||
256,
|
||
110,
|
||
|blk, r, b| {
|
||
put_d(blk, 108, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
}
|
||
hanzo_ml::set_force_scalar_matvec(false);
|
||
// Q5_K: 176 B, 256 elems. d (f16) at 0, dmin (f16) at 2. ASYMMETRIC 5-bit (Q4_K + qh 5th bit).
|
||
// dp4a-capable, so check() exercises the dp4a path vs the to_float oracle (scalar gated in the A/B
|
||
// below). Small exact-f16 d/dmin keep the 5-bit*6-bit-scale worst case inside f16 range.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ5K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_K",
|
||
RocmQuantType::Q5K,
|
||
n,
|
||
k,
|
||
256,
|
||
176,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
|
||
put_d(blk, 2, r, b, 0.00390625, 0.001953125, 4);
|
||
},
|
||
);
|
||
}
|
||
// Q4_1: 20 B, 32 elems. d (f16) at 0, m (f16) at 2. ASYMMETRIC legacy (val = nib*d + m).
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ4_1, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_1",
|
||
RocmQuantType::Q4_1,
|
||
n,
|
||
k,
|
||
32,
|
||
20,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
}
|
||
// Q5_0: 22 B, 32 elems. d (f16) at 0; qh[4] at 2; qs[16] at 6. symmetric 5-bit (val = (q5-16)*d).
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ5_0, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_0",
|
||
RocmQuantType::Q5_0,
|
||
n,
|
||
k,
|
||
32,
|
||
22,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
}
|
||
// Q5_1: 24 B, 32 elems. d (f16) at 0; m (f16) at 2; qh[4] at 4; qs[16] at 8. ASYMMETRIC 5-bit.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ5_1, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_1",
|
||
RocmQuantType::Q5_1,
|
||
n,
|
||
k,
|
||
32,
|
||
24,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
}
|
||
// Q8_1: 36 B, 32 elems. d (f16) at 0; s (f16, sum, unused for dequant) at 2; qs[32] at 4. SYM 8-bit.
|
||
for &(n, k) in shapes {
|
||
check::<BlockQ8_1, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q8_1",
|
||
RocmQuantType::Q8_1,
|
||
n,
|
||
k,
|
||
32,
|
||
36,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
}
|
||
|
||
eprintln!("{log}");
|
||
}
|
||
|
||
// MoE gate: the resident expert bank [E,n,k] dispatched per routed slot through the SAME unified
|
||
// core (moe_matvec_quant). For each slot s with expert eid=ids[s], output row s must equal
|
||
// W_eid[n,k]·x[s] computed via the real CPU to_float. Proves MoE rides the one core for a wired
|
||
// quant (Q4_K here -- the 30B-A3B production type), nbad=0, no per-quant MoE code.
|
||
fn moe_check<T: GgmlType, F: Fn(&mut [u8], usize, usize)>(
|
||
dev: &RocmDevice,
|
||
log: &mut String,
|
||
name: &str,
|
||
qt: RocmQuantType,
|
||
e_cnt: usize,
|
||
nrows: usize,
|
||
n: usize,
|
||
k: usize,
|
||
blk: usize,
|
||
tsz: usize,
|
||
fill: F,
|
||
) {
|
||
let nblk = k / blk;
|
||
let expert_bytes = n * nblk * tsz;
|
||
|
||
// Bank: E experts, each [n,k] = n*nblk blocks. Reuse build_bytes per expert with a per-expert
|
||
// seed offset so experts differ (different decoded weights).
|
||
let mut bank: Vec<u8> = Vec::with_capacity(e_cnt * expert_bytes);
|
||
for e in 0..e_cnt {
|
||
let eb = build_bytes(n, k, blk, tsz, |b, r, bb| fill(b, r * (e + 1) + e, bb));
|
||
assert_eq!(eb.len(), expert_bytes);
|
||
bank.extend_from_slice(&eb);
|
||
}
|
||
let wbank = dev.storage_from_slice(&bank).expect("upload bank");
|
||
|
||
// Router ids: each slot picks an expert (round-robin + a twist so repeats + spread both occur).
|
||
// Uploaded to the device: the batched MoE kernels index experts ON-DEVICE (no host ids loop).
|
||
let ids: Vec<u32> = (0..nrows).map(|s| ((s * 3 + 1) % e_cnt) as u32).collect();
|
||
let ids_dev = dev.storage_from_slice(&ids).expect("upload ids");
|
||
|
||
// Activation matrix [nrows, k], f16 and bf16.
|
||
let xf: Vec<f32> = (0..nrows * k).map(val).collect();
|
||
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
|
||
let xbf: Vec<bf16> = xf.iter().map(|&v| bf16::from_f32(v)).collect();
|
||
let xf_h: Vec<f32> = xh.iter().map(|v| v.to_f32()).collect();
|
||
let xf_b: Vec<f32> = xbf.iter().map(|v| v.to_f32()).collect();
|
||
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
|
||
let xst_b = dev.storage_from_slice(&xbf).expect("upload x bf16");
|
||
|
||
let y_h = dev
|
||
.moe_matvec_quant(qt, &wbank, &xst_h, &ids_dev, nrows, n, k)
|
||
.expect("moe f16");
|
||
let y_b = dev
|
||
.moe_matvec_quant(qt, &wbank, &xst_b, &ids_dev, nrows, n, k)
|
||
.expect("moe bf16");
|
||
let yh = to_f32(&y_h);
|
||
let yb = to_f32(&y_b);
|
||
|
||
// moe_matvec_pair (the gate+up shared-quantize fusion) must be BYTE-IDENTICAL to two independent
|
||
// moe_matvec_quant calls: with gate==up==bank, pair(bank,bank) == (y, y). Proves quantizing the
|
||
// shared routed activation once and matvec-ing both banks reproduces the unfused result exactly.
|
||
let (yp_gh, yp_uh) = dev
|
||
.moe_matvec_pair(qt, &wbank, &wbank, &xst_h, &ids_dev, nrows, n, k)
|
||
.expect("moe pair f16");
|
||
let (yp_gb, yp_ub) = dev
|
||
.moe_matvec_pair(qt, &wbank, &wbank, &xst_b, &ids_dev, nrows, n, k)
|
||
.expect("moe pair bf16");
|
||
assert!(
|
||
to_f32(&yp_gh) == yh && to_f32(&yp_uh) == yh,
|
||
"{name} moe_matvec_pair f16 != moe_matvec_quant"
|
||
);
|
||
assert!(
|
||
to_f32(&yp_gb) == yb && to_f32(&yp_ub) == yb,
|
||
"{name} moe_matvec_pair bf16 != moe_matvec_quant"
|
||
);
|
||
|
||
// Reference: per slot, dequantize expert eid and dot with that slot's activation row.
|
||
let mut nbad = 0usize;
|
||
let mut max_err_h = 0f32;
|
||
let mut max_err_b = 0f32;
|
||
let mut scale = 1f32;
|
||
let dp4a = qt.dp4a_active();
|
||
for (s, &eid) in ids.iter().enumerate() {
|
||
let eb = &bank[eid as usize * expert_bytes..(eid as usize + 1) * expert_bytes];
|
||
let raw_h = &xf_h[s * k..s * k + k];
|
||
let raw_b = &xf_b[s * k..s * k + k];
|
||
let xrow_h = if dp4a {
|
||
q8_1_recon(raw_h)
|
||
} else {
|
||
raw_h.to_vec()
|
||
};
|
||
let xrow_b = if dp4a {
|
||
q8_1_recon(raw_b)
|
||
} else {
|
||
raw_b.to_vec()
|
||
};
|
||
for r in 0..n {
|
||
let rh = reference_row::<T>(eb, r, nblk, blk, tsz, &xrow_h);
|
||
let rb = reference_row::<T>(eb, r, nblk, blk, tsz, &xrow_b);
|
||
scale = scale.max(rh.abs()).max(rb.abs());
|
||
let eh = (yh[s * n + r] - rh).abs();
|
||
let ebb = (yb[s * n + r] - rb).abs();
|
||
max_err_h = max_err_h.max(eh);
|
||
max_err_b = max_err_b.max(ebb);
|
||
let tol = 0.01 * scale.max(1.0);
|
||
if eh > tol || ebb > tol {
|
||
nbad += 1;
|
||
}
|
||
}
|
||
}
|
||
log.push_str(&format!(
|
||
"{name:8} MoE E={e_cnt} nrows={nrows} n={n} k={k} nbad={nbad}/{} max_err_f16={max_err_h:.5} max_err_bf16={max_err_b:.5} scale={scale:.3}\n",
|
||
nrows * n
|
||
));
|
||
assert!(
|
||
nbad == 0,
|
||
"{name} MoE mismatch: nbad={nbad} max_err_f16={max_err_h} max_err_bf16={max_err_b}"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn moe_matvec_unified_numeric() {
|
||
let mut log = String::new();
|
||
let dev = RocmDevice::new(0).expect("rocm device");
|
||
|
||
// Q4_K (the 30B-A3B production MoE type): E experts, nrows routed slots (= t*topk), n out, k in.
|
||
// 8 experts, 16 slots (decode t=2,topk=8-ish), n=128 out, k=512 in (2 super-blocks).
|
||
moe_check::<BlockQ4K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_K",
|
||
RocmQuantType::Q4K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
144,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
// Q8_0 MoE (the other proven type) to show MoE is per-type generic via the same core.
|
||
moe_check::<BlockQ8_0, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q8_0",
|
||
RocmQuantType::Q8_0,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
32,
|
||
34,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
// Q6_K MoE: the mixed-precision down-proj type in Qwen3-30B-A3B-Q4_K_M. Exercises the batched
|
||
// on-device-ids `moe_qmatvecu_q6k_*` kernel (experts on grid.y) against the CPU oracle. Small d
|
||
// keeps worst-case rows inside f16 range, same as the single-expert Q6_K gate above.
|
||
moe_check::<BlockQ6K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q6_K",
|
||
RocmQuantType::Q6K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
210,
|
||
|blk, r, b| {
|
||
put_d(blk, 208, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
// Q2_K MoE (ASYMMETRIC) + Q3_K MoE (symmetric 3-bit): the decode-only k-quants run their experts
|
||
// through the SAME batched on-device-ids moe_qmatvec core (dp4a here) against the CPU oracle.
|
||
moe_check::<BlockQ2K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q2_K",
|
||
RocmQuantType::Q2K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
84,
|
||
|blk, r, b| {
|
||
put_d(blk, 80, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 82, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
moe_check::<BlockQ3K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q3_K",
|
||
RocmQuantType::Q3K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
110,
|
||
|blk, r, b| {
|
||
put_d(blk, 108, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
// IQ4_XS MoE: the dominant attention/expert type in dynamic mixed-type GGUFs. Its 256-elem
|
||
// super-block rides the SAME batched on-device-ids moe_qmatvec dp4a core (codebook nibble ->
|
||
// signed int8) vs oracle.
|
||
moe_check::<BlockIQ4xs, _>(
|
||
&dev,
|
||
&mut log,
|
||
"IQ4_XS",
|
||
RocmQuantType::IQ4_XS,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
136,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
// SCALAR-forced MoE vs EXACT oracle: exercises the moe_qmatvecu_q2k/q3k scalar kernels (the dp4a
|
||
// runs above used the q8_1 reference). Bit-exact weight decode through the batched expert path.
|
||
hanzo_ml::set_force_scalar_matvec(true);
|
||
moe_check::<BlockQ2K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q2_K-scal",
|
||
RocmQuantType::Q2K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
84,
|
||
|blk, r, b| {
|
||
put_d(blk, 80, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 82, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
moe_check::<BlockQ3K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q3_K-scal",
|
||
RocmQuantType::Q3K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
110,
|
||
|blk, r, b| {
|
||
put_d(blk, 108, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
hanzo_ml::set_force_scalar_matvec(false);
|
||
// Q5_K MoE (ASYMMETRIC 5-bit) -- dp4a expert-bank path (moe_qmatvec_dp4a_q5k) vs the CPU oracle.
|
||
moe_check::<BlockQ5K, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_K",
|
||
RocmQuantType::Q5K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
176,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
|
||
put_d(blk, 2, r, b, 0.00390625, 0.001953125, 4);
|
||
},
|
||
);
|
||
// Legacy 32-elem MoE types via the scalar moe_qmatvecu_* core (experts on grid.y).
|
||
moe_check::<BlockQ4_1, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_1",
|
||
RocmQuantType::Q4_1,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
32,
|
||
20,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
moe_check::<BlockQ5_0, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_0",
|
||
RocmQuantType::Q5_0,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
32,
|
||
22,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
moe_check::<BlockQ5_1, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_1",
|
||
RocmQuantType::Q5_1,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
32,
|
||
24,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
moe_check::<BlockQ8_1, _>(
|
||
&dev,
|
||
&mut log,
|
||
"Q8_1",
|
||
RocmQuantType::Q8_1,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
32,
|
||
36,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
},
|
||
);
|
||
|
||
eprintln!("{log}");
|
||
}
|
||
|
||
// dp4a-vs-scalar A/B gate: the int8-dp4a decode core (`qdp4a<WTYPE>`) must equal the scalar
|
||
// `qmatvec_core<WTYPE>` (the byte-faithful CPU-oracle reference) for every dp4a-capable type. Sets
|
||
// set_force_scalar_matvec(true) to force the scalar core, false to take dp4a, and asserts the
|
||
// two agree to a tight f16/bf16-reorder tolerance (both accumulate in f32; only the summation ORDER
|
||
// differs). Covers both the single-expert matvec and the indexed-MoE path. This proves the speedup
|
||
// changed nothing numeric -- the whole point of the bit-faithful dp4a port.
|
||
fn ab_matvec<F: Fn(&mut [u8], usize, usize)>(
|
||
dev: &RocmDevice,
|
||
log: &mut String,
|
||
name: &str,
|
||
qt: RocmQuantType,
|
||
n: usize,
|
||
k: usize,
|
||
blk: usize,
|
||
tsz: usize,
|
||
fill: F,
|
||
) {
|
||
let xf: Vec<f32> = (0..k).map(val).collect();
|
||
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
|
||
let xbf: Vec<bf16> = xf.iter().map(|&v| bf16::from_f32(v)).collect();
|
||
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
|
||
let xst_b = dev.storage_from_slice(&xbf).expect("upload x bf16");
|
||
let wq_bytes = build_bytes(n, k, blk, tsz, fill);
|
||
let wst = dev.storage_from_slice(&wq_bytes).expect("upload w");
|
||
|
||
// dp4a path (fallback unset).
|
||
hanzo_ml::set_force_scalar_matvec(false);
|
||
let dp4a_h = to_f32(&dev.matvec_quant(qt, &wst, &xst_h, n, k).expect("dp4a f16"));
|
||
let dp4a_b = to_f32(&dev.matvec_quant(qt, &wst, &xst_b, n, k).expect("dp4a bf16"));
|
||
// scalar path (fallback set) -- the byte-faithful reference core.
|
||
hanzo_ml::set_force_scalar_matvec(true);
|
||
let scal_h = to_f32(
|
||
&dev.matvec_quant(qt, &wst, &xst_h, n, k)
|
||
.expect("scalar f16"),
|
||
);
|
||
let scal_b = to_f32(
|
||
&dev.matvec_quant(qt, &wst, &xst_b, n, k)
|
||
.expect("scalar bf16"),
|
||
);
|
||
hanzo_ml::set_force_scalar_matvec(false);
|
||
|
||
let scale = scal_h
|
||
.iter()
|
||
.chain(scal_b.iter())
|
||
.fold(0f32, |m, &v| m.max(v.abs()))
|
||
.max(1.0);
|
||
let tol = 0.01 * scale;
|
||
let mut nbad = 0usize;
|
||
let mut max_err = 0f32;
|
||
for i in 0..n {
|
||
let eh = (dp4a_h[i] - scal_h[i]).abs();
|
||
let eb = (dp4a_b[i] - scal_b[i]).abs();
|
||
max_err = max_err.max(eh).max(eb);
|
||
if eh > tol || eb > tol {
|
||
nbad += 1;
|
||
}
|
||
}
|
||
log.push_str(&format!(
|
||
"{name:8} dp4a-vs-scalar n={n} k={k} nbad={nbad}/{n} max_err={max_err:.5} (tol {tol:.5})\n"
|
||
));
|
||
assert!(
|
||
nbad == 0,
|
||
"{name} dp4a != scalar core: nbad={nbad} max_err={max_err} tol={tol}"
|
||
);
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn ab_moe<F: Fn(&mut [u8], usize, usize)>(
|
||
dev: &RocmDevice,
|
||
log: &mut String,
|
||
name: &str,
|
||
qt: RocmQuantType,
|
||
e_cnt: usize,
|
||
nrows: usize,
|
||
n: usize,
|
||
k: usize,
|
||
blk: usize,
|
||
tsz: usize,
|
||
fill: F,
|
||
) {
|
||
let nblk = k / blk;
|
||
let expert_bytes = n * nblk * tsz;
|
||
let mut bank: Vec<u8> = Vec::with_capacity(e_cnt * expert_bytes);
|
||
for e in 0..e_cnt {
|
||
bank.extend_from_slice(&build_bytes(n, k, blk, tsz, |b, r, bb| {
|
||
fill(b, r * (e + 1) + e, bb)
|
||
}));
|
||
}
|
||
let wbank = dev.storage_from_slice(&bank).expect("upload bank");
|
||
let ids: Vec<u32> = (0..nrows).map(|s| ((s * 3 + 1) % e_cnt) as u32).collect();
|
||
let ids_dev = dev.storage_from_slice(&ids).expect("upload ids");
|
||
let xf: Vec<f32> = (0..nrows * k).map(val).collect();
|
||
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
|
||
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
|
||
|
||
hanzo_ml::set_force_scalar_matvec(false);
|
||
let dp4a = to_f32(
|
||
&dev.moe_matvec_quant(qt, &wbank, &xst_h, &ids_dev, nrows, n, k)
|
||
.expect("moe dp4a"),
|
||
);
|
||
hanzo_ml::set_force_scalar_matvec(true);
|
||
let scal = to_f32(
|
||
&dev.moe_matvec_quant(qt, &wbank, &xst_h, &ids_dev, nrows, n, k)
|
||
.expect("moe scalar"),
|
||
);
|
||
hanzo_ml::set_force_scalar_matvec(false);
|
||
|
||
let scale = scal.iter().fold(0f32, |m, &v| m.max(v.abs())).max(1.0);
|
||
let tol = 0.01 * scale;
|
||
let mut nbad = 0usize;
|
||
let mut max_err = 0f32;
|
||
for i in 0..nrows * n {
|
||
let e = (dp4a[i] - scal[i]).abs();
|
||
max_err = max_err.max(e);
|
||
if e > tol {
|
||
nbad += 1;
|
||
}
|
||
}
|
||
log.push_str(&format!(
|
||
"{name:8} MoE dp4a-vs-scalar nbad={nbad}/{} max_err={max_err:.5} (tol {tol:.5})\n",
|
||
nrows * n
|
||
));
|
||
assert!(
|
||
nbad == 0,
|
||
"{name} MoE dp4a != scalar core: nbad={nbad} max_err={max_err} tol={tol}"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn qmatvec_dp4a_vs_scalar() {
|
||
let mut log = String::new();
|
||
let dev = RocmDevice::new(0).expect("rocm device");
|
||
let shapes: &[(usize, usize)] = &[
|
||
(64, 256),
|
||
(4096, 4096),
|
||
(1024, 3072),
|
||
(17, 4096),
|
||
(4096, 256),
|
||
];
|
||
|
||
// Q4_K (ASYMMETRIC) and Q6_K (SYMMETRIC 6-bit) are the dp4a-capable types; each must match its
|
||
// scalar core bit-faithfully. Same small exact-f16 scales the oracle gate uses.
|
||
for &(n, k) in shapes {
|
||
ab_matvec(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_K",
|
||
RocmQuantType::Q4K,
|
||
n,
|
||
k,
|
||
256,
|
||
144,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
ab_matvec(
|
||
&dev,
|
||
&mut log,
|
||
"Q6_K",
|
||
RocmQuantType::Q6K,
|
||
n,
|
||
k,
|
||
256,
|
||
210,
|
||
|blk, r, b| {
|
||
put_d(blk, 208, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
ab_matvec(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_K",
|
||
RocmQuantType::Q5K,
|
||
n,
|
||
k,
|
||
256,
|
||
176,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
|
||
put_d(blk, 2, r, b, 0.00390625, 0.001953125, 4);
|
||
},
|
||
);
|
||
}
|
||
// MoE A/B for both (8 experts, 16 slots, n=128, k=512).
|
||
ab_moe(
|
||
&dev,
|
||
&mut log,
|
||
"Q4_K",
|
||
RocmQuantType::Q4K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
144,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0625, 0.03125, 5);
|
||
put_d(blk, 2, r, b, 0.03125, 0.015625, 4);
|
||
},
|
||
);
|
||
ab_moe(
|
||
&dev,
|
||
&mut log,
|
||
"Q6_K",
|
||
RocmQuantType::Q6K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
210,
|
||
|blk, r, b| {
|
||
put_d(blk, 208, r, b, 0.0078125, 0.00390625, 5);
|
||
},
|
||
);
|
||
ab_moe(
|
||
&dev,
|
||
&mut log,
|
||
"Q5_K",
|
||
RocmQuantType::Q5K,
|
||
8,
|
||
16,
|
||
128,
|
||
512,
|
||
256,
|
||
176,
|
||
|blk, r, b| {
|
||
put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
|
||
put_d(blk, 2, r, b, 0.00390625, 0.001953125, 4);
|
||
},
|
||
);
|
||
|
||
eprintln!("{log}");
|
||
}
|