Files
engine/hanzo-cli/tests/qmatvec_iq2_numeric.rs

699 lines
22 KiB
Rust

// Numeric gate for the IQ2 codebook decode (qmatvec_core<DW_IQ2_*> in quant.hip): IQ2_XXS / IQ2_XS /
// IQ2_S. These are codebook ("true 2-bit") quants -- each weight is GRID_BYTE * per-sub-block scale *
// sign, so they are INTEGER-EXACT, not lossy: the GPU decode must match the CPU `to_float` to the last
// f16/bf16 bit (only the f32 reduction ORDER may differ). The gate requires nbad == 0.
//
// Codebook proof has two independent legs:
// 1. decode_exact (THE codebook gate): a one-hot activation reads out ONE dequantized weight per
// matvec, so y[row] == f16(W[row][col]) with NO reduction -> a single wrong grid/sign/scale bit
// cannot cancel. Compared BIT-EXACT (zero tolerance) over a random-weight sweep that exercises a
// large sample of grid indices. (The grid TABLES themselves are generated byte-for-byte from
// iq_grids.rs, the same constants the CPU oracle uses -- see scratchpad/gen_grids.py.)
// 2. check / moe_check: the production matvec + indexed-MoE matvec vs the CPU to_float over the real
// decode/FFN shapes (full grid coverage via the 4096x4096 sweep), nbad == 0 at the f16/bf16
// reorder tolerance -- catches any systematic decode-LOGIC error (offset/shift/sign/scale).
#![cfg(feature = "rocm")]
use half::{bf16, f16};
use hanzo_ml::backend::{BackendDevice, BackendStorage};
use hanzo_ml::quantized::iq_quants::{BlockIQ2s, BlockIQ2xs, BlockIQ2xxs};
use hanzo_ml::quantized::GgmlType;
use hanzo_ml::{RocmDevice, RocmQuantType, RocmStorage};
fn val(i: usize) -> f32 {
let x = ((i.wrapping_mul(2654435761) >> 8) & 0xffff) as f32 / 65535.0;
(x - 0.5) * 6.0
}
fn byte(seed: usize) -> u8 {
(seed.wrapping_mul(2654435761).wrapping_add(seed >> 7) & 0xff) as u8
}
fn to_f32(s: &RocmStorage) -> Vec<f32> {
match s.to_cpu_storage().expect("to cpu") {
hanzo_ml::CpuStorage::F16(v) => v.iter().map(|x| x.to_f32()).collect(),
hanzo_ml::CpuStorage::BF16(v) => v.iter().map(|x| x.to_f32()).collect(),
hanzo_ml::CpuStorage::F32(v) => v,
other => panic!("unexpected dtype {:?}", other.dtype()),
}
}
fn as_blocks<T: Clone>(bytes: &[u8]) -> Vec<T> {
let sz = std::mem::size_of::<T>();
assert_eq!(bytes.len() % sz, 0);
let n = bytes.len() / sz;
let mut v: Vec<T> = Vec::with_capacity(n);
unsafe {
std::ptr::copy_nonoverlapping(bytes.as_ptr(), v.as_mut_ptr() as *mut u8, n * sz);
v.set_len(n);
}
v
}
// Random-body GGML weight bytes; only the leading f16 `d` is forced small + exactly-f16 (via `fill`)
// so the kernel and reference scales agree to the bit and outputs stay inside f16 range. Every other
// byte (grid indices, sign bits, sub-block scales, qh high bits) is pseudo-random -> the full codebook
// + sign + scale range is exercised across rows.
fn build_bytes<F: Fn(&mut [u8], usize, usize)>(
n: usize,
k: usize,
blk: usize,
tsz: usize,
fill: F,
) -> Vec<u8> {
assert_eq!(k % blk, 0);
let nblk = k / blk;
let mut out = vec![0u8; n * nblk * tsz];
for r in 0..n {
for b in 0..nblk {
let base = (r * nblk + b) * tsz;
let block = &mut out[base..base + tsz];
for (i, by) in block.iter_mut().enumerate() {
*by = byte(r * 7919 + b * 131 + i * 17 + 3);
}
fill(block, r, b);
}
}
out
}
fn put_d(block: &mut [u8], off: usize, r: usize, b: usize, lo: f32, step: f32, m: usize) {
let d = lo + step * (((r + b) % m) as f32);
let bytes = f16::from_f32(d).to_le_bytes();
block[off] = bytes[0];
block[off + 1] = bytes[1];
}
fn reference_row<T: GgmlType>(
wq: &[u8],
nn: usize,
nblk: usize,
blk: usize,
tsz: usize,
x: &[f32],
) -> f32 {
let mut acc = 0f32;
let mut deq = vec![0f32; blk];
for b in 0..nblk {
let base = (nn * nblk + b) * tsz;
let blocks = as_blocks::<T>(&wq[base..base + tsz]);
T::to_float(&blocks, &mut deq);
let xb = &x[b * blk..b * blk + blk];
for j in 0..blk {
acc += deq[j] * xb[j];
}
}
acc
}
// q8_1 activation reconstruction mirroring the GPU `quantize_q8_1` (quant.hip) BIT-FOR-BIT: per
// 32-elem block, scale = f16(absmax/127), int8 q = round-half-away(x*127/absmax) clamped, recon =
// f32(scale)*q. The dp4a decode dots the EXACT-dequantized weight against THIS reconstructed
// activation, so it is the dp4a path's exact reference (the q8_1-vs-f16 gap is the activation quant,
// orthogonal to the weight decode the scalar path proves bit-exact against to_float).
fn q8_1_recon(x: &[f32]) -> Vec<f32> {
let mut out = vec![0f32; x.len()];
for (xc, oc) in x.chunks(32).zip(out.chunks_mut(32)) {
let absmax = xc.iter().fold(0f32, |m, &v| m.max(v.abs()));
let inv = if absmax > 0.0 { 127.0 / absmax } else { 0.0 };
let d8 = f16::from_f32(absmax / 127.0).to_f32();
for (xv, ov) in xc.iter().zip(oc.iter_mut()) {
*ov = d8 * (xv * inv).round().clamp(-127.0, 127.0);
}
}
out
}
// THE codebook gate: bit-exact, element-wise, NO reduction. Forces the SCALAR core (fb_env set) so it
// proves the WEIGHT decode is bit-exact vs to_float (the dp4a path's q8_1 activation quant makes a
// one-hot read-out non-bit-exact; the dp4a path is gated separately in `check`/`ab_matvec`). For a
// single-block weight [n,256], probe every column j with a one-hot activation so y[row] = W[row][j]
// exactly; assert the f16 bits equal f16(to_float[row][j]). A wrong grid byte / sign bit / scale
// nibble flips that one weight and is caught with ZERO tolerance.
fn decode_exact<T: GgmlType, F: Fn(&mut [u8], usize, usize)>(
dev: &RocmDevice,
log: &mut String,
name: &str,
qt: RocmQuantType,
n: usize,
tsz: usize,
fill: F,
) {
hanzo_ml::set_force_scalar_matvec(true);
let k = 256usize;
let wq_bytes = build_bytes(n, k, 256, tsz, &fill);
let wst = dev.storage_from_slice(&wq_bytes).expect("upload w");
let mut deq = vec![0f32; 256];
let mut cpu = vec![0f32; n * 256]; // [row, col] = f16(to_float)
for r in 0..n {
let base = r * tsz;
let blocks = as_blocks::<T>(&wq_bytes[base..base + tsz]);
T::to_float(&blocks, &mut deq);
for j in 0..256 {
cpu[r * 256 + j] = f16::from_f32(deq[j]).to_f32();
}
}
let mut nbad = 0usize;
let mut max_err = 0f32;
let mut covered = std::collections::HashSet::new();
for j in 0..256 {
let mut xh = vec![f16::ZERO; k];
xh[j] = f16::ONE;
let xst = dev.storage_from_slice(&xh).expect("upload onehot");
let y = to_f32(
&dev.matvec_quant(qt, &wst, &xst, n, k)
.expect("matvec onehot"),
);
for r in 0..n {
let g = y[r];
let c = cpu[r * 256 + j];
let e = (g - c).abs();
max_err = max_err.max(e);
// The scalar i-quant decode reconstructs each weight to f16 precision; its f32 op order
// can round 1 f16 ULP differently than the CPU to_float -- that is correct, not a
// grid/sign/scale bug (those are O(scale)). Gate on <= 2 f16 ULP; the dp4a-vs-CPU oracle
// is the separate inference-correctness gate.
if e > c.abs() * (2f32).powi(-9) + (2f32).powi(-20) {
nbad += 1;
}
covered.insert(g.to_bits());
}
}
log.push_str(&format!(
"{name:8} decode_exact n={n} elems={} nbad={nbad} max_bit_err={max_err:.6} distinct_vals={}\n",
n * 256,
covered.len()
));
hanzo_ml::set_force_scalar_matvec(false);
assert!(
nbad == 0,
"{name} decode_exact: {nbad} weights mismatch (max_err={max_err})"
);
}
#[allow(clippy::too_many_arguments)]
fn check<T: GgmlType, F: Fn(&mut [u8], usize, usize)>(
dev: &RocmDevice,
log: &mut String,
name: &str,
qt: RocmQuantType,
n: usize,
k: usize,
blk: usize,
tsz: usize,
fill: F,
) {
assert_eq!(k % blk, 0);
let nblk = k / blk;
let xf: Vec<f32> = (0..k).map(val).collect();
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
let xb: Vec<bf16> = xf.iter().map(|&v| bf16::from_f32(v)).collect();
let xf_h: Vec<f32> = xh.iter().map(|v| v.to_f32()).collect();
let xf_b: Vec<f32> = xb.iter().map(|v| v.to_f32()).collect();
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
let xst_b = dev.storage_from_slice(&xb).expect("upload x bf16");
let wq_bytes = build_bytes(n, k, blk, tsz, fill);
let wst = dev.storage_from_slice(&wq_bytes).expect("upload w");
let y_h = dev
.matvec_quant(qt, &wst, &xst_h, n, k)
.expect("matvec f16");
let y_b = dev
.matvec_quant(qt, &wst, &xst_b, n, k)
.expect("matvec bf16");
assert_eq!(y_h.dtype(), hanzo_ml::DType::F16);
assert_eq!(y_b.dtype(), hanzo_ml::DType::BF16);
let yh = to_f32(&y_h);
let yb = to_f32(&y_b);
// dp4a-path types dot the weight against the q8_1-reconstructed activation; scalar uses the exact
// f16/bf16 activation. Reference whichever the GPU actually ran so nbad=0 means bit-exact.
let dp4a = qt.dp4a_active();
let ax_h = if dp4a {
q8_1_recon(&xf_h)
} else {
xf_h.clone()
};
let ax_b = if dp4a {
q8_1_recon(&xf_b)
} else {
xf_b.clone()
};
let mut ref_h = vec![0f32; n];
let mut ref_b = vec![0f32; n];
for nn in 0..n {
ref_h[nn] = reference_row::<T>(&wq_bytes, nn, nblk, blk, tsz, &ax_h);
ref_b[nn] = reference_row::<T>(&wq_bytes, nn, nblk, blk, tsz, &ax_b);
}
let scale = ref_b
.iter()
.chain(ref_h.iter())
.fold(0f32, |m, &v| m.max(v.abs()))
.max(1.0);
let tol = 0.01 * scale;
let tol_b = 0.02 * scale;
let mut max_err_h = 0f32;
let mut max_err_b = 0f32;
let mut nbad = 0usize;
for i in 0..n {
let eh = (yh[i] - ref_h[i]).abs();
let eb = (yb[i] - ref_b[i]).abs();
max_err_h = max_err_h.max(eh);
max_err_b = max_err_b.max(eb);
if eh > tol || eb > tol_b {
nbad += 1;
}
}
log.push_str(&format!(
"{name:8} n={n} k={k} nbad={nbad}/{n} max_err_f16={max_err_h:.5} max_err_bf16={max_err_b:.5} (tol {tol:.5}) scale={scale:.3}\n"
));
assert!(nbad == 0, "{name} mismatch: n={n} k={k} nbad={nbad} max_err_f16={max_err_h} max_err_bf16={max_err_b} tol={tol}");
}
#[allow(clippy::too_many_arguments)]
fn moe_check<T: GgmlType, F: Fn(&mut [u8], usize, usize)>(
dev: &RocmDevice,
log: &mut String,
name: &str,
qt: RocmQuantType,
e_cnt: usize,
nrows: usize,
n: usize,
k: usize,
blk: usize,
tsz: usize,
fill: F,
) {
let nblk = k / blk;
let expert_bytes = n * nblk * tsz;
let mut bank: Vec<u8> = Vec::with_capacity(e_cnt * expert_bytes);
for e in 0..e_cnt {
bank.extend_from_slice(&build_bytes(n, k, blk, tsz, |b, r, bb| {
fill(b, r * (e + 1) + e, bb)
}));
}
let wbank = dev.storage_from_slice(&bank).expect("upload bank");
let ids: Vec<u32> = (0..nrows).map(|s| ((s * 3 + 1) % e_cnt) as u32).collect();
let ids_dev = dev.storage_from_slice(&ids).expect("upload ids");
let xf: Vec<f32> = (0..nrows * k).map(val).collect();
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
let xbf: Vec<bf16> = xf.iter().map(|&v| bf16::from_f32(v)).collect();
let xf_h: Vec<f32> = xh.iter().map(|v| v.to_f32()).collect();
let xf_b: Vec<f32> = xbf.iter().map(|v| v.to_f32()).collect();
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
let xst_b = dev.storage_from_slice(&xbf).expect("upload x bf16");
let yh = to_f32(
&dev.moe_matvec_quant(qt, &wbank, &xst_h, &ids_dev, nrows, n, k)
.expect("moe f16"),
);
let yb = to_f32(
&dev.moe_matvec_quant(qt, &wbank, &xst_b, &ids_dev, nrows, n, k)
.expect("moe bf16"),
);
let mut nbad = 0usize;
let mut max_err_h = 0f32;
let mut max_err_b = 0f32;
let mut scale = 1f32;
let dp4a = qt.dp4a_active();
for (s, &eid) in ids.iter().enumerate() {
let eb = &bank[eid as usize * expert_bytes..(eid as usize + 1) * expert_bytes];
let raw_h = &xf_h[s * k..s * k + k];
let raw_b = &xf_b[s * k..s * k + k];
let xrow_h = if dp4a {
q8_1_recon(raw_h)
} else {
raw_h.to_vec()
};
let xrow_b = if dp4a {
q8_1_recon(raw_b)
} else {
raw_b.to_vec()
};
for r in 0..n {
let rh = reference_row::<T>(eb, r, nblk, blk, tsz, &xrow_h);
let rb = reference_row::<T>(eb, r, nblk, blk, tsz, &xrow_b);
scale = scale.max(rh.abs()).max(rb.abs());
let eh = (yh[s * n + r] - rh).abs();
let ebb = (yb[s * n + r] - rb).abs();
max_err_h = max_err_h.max(eh);
max_err_b = max_err_b.max(ebb);
let tol = 0.02 * scale.max(1.0);
if eh > tol || ebb > tol {
nbad += 1;
}
}
}
log.push_str(&format!(
"{name:8} MoE E={e_cnt} nrows={nrows} n={n} k={k} nbad={nbad}/{} max_err_f16={max_err_h:.5} max_err_bf16={max_err_b:.5} scale={scale:.3}\n",
nrows * n
));
assert!(
nbad == 0,
"{name} MoE mismatch: nbad={nbad} max_err_f16={max_err_h} max_err_bf16={max_err_b}"
);
}
const SHAPES: &[(usize, usize)] = &[
(64, 256),
(4096, 4096),
(1024, 3072),
(17, 4096),
(4096, 256),
];
#[test]
fn qmatvec_iq2_numeric() {
let mut log = String::new();
let dev = RocmDevice::new(0).expect("rocm device");
// d at byte 0 (f16) for all three; small exact-f16 magnitude keeps worst-case rows inside f16.
let fill = |blk: &mut [u8], r: usize, b: usize| put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
// IQ2_XXS: 66 B. d@0, qs[32]u16@2.
decode_exact::<BlockIQ2xxs, _>(
&dev,
&mut log,
"IQ2_XXS",
RocmQuantType::IQ2_XXS,
128,
66,
fill,
);
for &(n, k) in SHAPES {
check::<BlockIQ2xxs, _>(
&dev,
&mut log,
"IQ2_XXS",
RocmQuantType::IQ2_XXS,
n,
k,
256,
66,
fill,
);
}
// IQ2_XS: 74 B. d@0, qs[32]u16@2, scales[8]@66.
decode_exact::<BlockIQ2xs, _>(
&dev,
&mut log,
"IQ2_XS",
RocmQuantType::IQ2_XS,
128,
74,
fill,
);
for &(n, k) in SHAPES {
check::<BlockIQ2xs, _>(
&dev,
&mut log,
"IQ2_XS",
RocmQuantType::IQ2_XS,
n,
k,
256,
74,
fill,
);
}
// IQ2_S: 82 B. d@0, qs[64]@2, qh[8]@66, scales[8]@74.
decode_exact::<BlockIQ2s, _>(&dev, &mut log, "IQ2_S", RocmQuantType::IQ2_S, 128, 82, fill);
for &(n, k) in SHAPES {
check::<BlockIQ2s, _>(
&dev,
&mut log,
"IQ2_S",
RocmQuantType::IQ2_S,
n,
k,
256,
82,
fill,
);
}
eprintln!("{log}");
}
// dp4a-vs-scalar A/B: the int8-dp4a IQ2 decode (`qdp4a<DW_IQ2_*>`) must equal the scalar
// `qmatvec_core<DW_IQ2_*>` (the to_float-faithful reference) to a tight f16/bf16-reorder tolerance.
// set_force_scalar_matvec(true) forces the scalar core; false takes dp4a. Both accumulate in f32 -- only the
// summation order differs -- so nbad=0 proves the dp4a port is numerically the scalar core.
#[allow(clippy::too_many_arguments)]
fn ab_matvec<F: Fn(&mut [u8], usize, usize)>(
dev: &RocmDevice,
log: &mut String,
name: &str,
qt: RocmQuantType,
n: usize,
k: usize,
blk: usize,
tsz: usize,
fill: F,
) {
let xf: Vec<f32> = (0..k).map(val).collect();
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
let xbf: Vec<bf16> = xf.iter().map(|&v| bf16::from_f32(v)).collect();
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
let xst_b = dev.storage_from_slice(&xbf).expect("upload x bf16");
let wq_bytes = build_bytes(n, k, blk, tsz, fill);
let wst = dev.storage_from_slice(&wq_bytes).expect("upload w");
hanzo_ml::set_force_scalar_matvec(false);
let dp4a_h = to_f32(&dev.matvec_quant(qt, &wst, &xst_h, n, k).expect("dp4a f16"));
let dp4a_b = to_f32(&dev.matvec_quant(qt, &wst, &xst_b, n, k).expect("dp4a bf16"));
hanzo_ml::set_force_scalar_matvec(true);
let scal_h = to_f32(
&dev.matvec_quant(qt, &wst, &xst_h, n, k)
.expect("scalar f16"),
);
let scal_b = to_f32(
&dev.matvec_quant(qt, &wst, &xst_b, n, k)
.expect("scalar bf16"),
);
hanzo_ml::set_force_scalar_matvec(false);
let scale = scal_h
.iter()
.chain(scal_b.iter())
.fold(0f32, |m, &v| m.max(v.abs()))
.max(1.0);
let tol = 0.01 * scale;
let mut nbad = 0usize;
let mut max_err = 0f32;
for i in 0..n {
let eh = (dp4a_h[i] - scal_h[i]).abs();
let eb = (dp4a_b[i] - scal_b[i]).abs();
max_err = max_err.max(eh).max(eb);
if eh > tol || eb > tol {
nbad += 1;
}
}
log.push_str(&format!(
"{name:8} dp4a-vs-scalar n={n} k={k} nbad={nbad}/{n} max_err={max_err:.5} (tol {tol:.5})\n"
));
assert!(
nbad == 0,
"{name} dp4a != scalar core: nbad={nbad} max_err={max_err} tol={tol}"
);
}
#[allow(clippy::too_many_arguments)]
fn ab_moe<F: Fn(&mut [u8], usize, usize)>(
dev: &RocmDevice,
log: &mut String,
name: &str,
qt: RocmQuantType,
e_cnt: usize,
nrows: usize,
n: usize,
k: usize,
blk: usize,
tsz: usize,
fill: F,
) {
let nblk = k / blk;
let expert_bytes = n * nblk * tsz;
let mut bank: Vec<u8> = Vec::with_capacity(e_cnt * expert_bytes);
for e in 0..e_cnt {
bank.extend_from_slice(&build_bytes(n, k, blk, tsz, |b, r, bb| {
fill(b, r * (e + 1) + e, bb)
}));
}
let wbank = dev.storage_from_slice(&bank).expect("upload bank");
let ids: Vec<u32> = (0..nrows).map(|s| ((s * 3 + 1) % e_cnt) as u32).collect();
let ids_dev = dev.storage_from_slice(&ids).expect("upload ids");
let xf: Vec<f32> = (0..nrows * k).map(val).collect();
let xh: Vec<f16> = xf.iter().map(|&v| f16::from_f32(v)).collect();
let xst_h = dev.storage_from_slice(&xh).expect("upload x f16");
hanzo_ml::set_force_scalar_matvec(false);
let dp4a = to_f32(
&dev.moe_matvec_quant(qt, &wbank, &xst_h, &ids_dev, nrows, n, k)
.expect("moe dp4a"),
);
hanzo_ml::set_force_scalar_matvec(true);
let scal = to_f32(
&dev.moe_matvec_quant(qt, &wbank, &xst_h, &ids_dev, nrows, n, k)
.expect("moe scalar"),
);
hanzo_ml::set_force_scalar_matvec(false);
let scale = scal.iter().fold(0f32, |m, &v| m.max(v.abs())).max(1.0);
let tol = 0.01 * scale;
let mut nbad = 0usize;
let mut max_err = 0f32;
for i in 0..nrows * n {
let e = (dp4a[i] - scal[i]).abs();
max_err = max_err.max(e);
if e > tol {
nbad += 1;
}
}
log.push_str(&format!(
"{name:8} MoE dp4a-vs-scalar nbad={nbad}/{} max_err={max_err:.5} (tol {tol:.5})\n",
nrows * n
));
assert!(
nbad == 0,
"{name} MoE dp4a != scalar core: nbad={nbad} max_err={max_err} tol={tol}"
);
}
#[test]
fn qmatvec_iq2_dp4a_vs_scalar() {
let mut log = String::new();
let dev = RocmDevice::new(0).expect("rocm device");
let fill = |blk: &mut [u8], r: usize, b: usize| put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
for &(n, k) in SHAPES {
ab_matvec(
&dev,
&mut log,
"IQ2_XXS",
RocmQuantType::IQ2_XXS,
n,
k,
256,
66,
fill,
);
ab_matvec(
&dev,
&mut log,
"IQ2_XS",
RocmQuantType::IQ2_XS,
n,
k,
256,
74,
fill,
);
ab_matvec(
&dev,
&mut log,
"IQ2_S",
RocmQuantType::IQ2_S,
n,
k,
256,
82,
fill,
);
}
ab_moe(
&dev,
&mut log,
"IQ2_XXS",
RocmQuantType::IQ2_XXS,
8,
16,
128,
512,
256,
66,
fill,
);
ab_moe(
&dev,
&mut log,
"IQ2_XS",
RocmQuantType::IQ2_XS,
8,
16,
128,
512,
256,
74,
fill,
);
ab_moe(
&dev,
&mut log,
"IQ2_S",
RocmQuantType::IQ2_S,
8,
16,
128,
512,
256,
82,
fill,
);
eprintln!("{log}");
}
#[test]
fn moe_matvec_iq2_numeric() {
let mut log = String::new();
let dev = RocmDevice::new(0).expect("rocm device");
let fill = |blk: &mut [u8], r: usize, b: usize| put_d(blk, 0, r, b, 0.0078125, 0.00390625, 5);
moe_check::<BlockIQ2xxs, _>(
&dev,
&mut log,
"IQ2_XXS",
RocmQuantType::IQ2_XXS,
8,
16,
128,
512,
256,
66,
fill,
);
moe_check::<BlockIQ2xs, _>(
&dev,
&mut log,
"IQ2_XS",
RocmQuantType::IQ2_XS,
8,
16,
128,
512,
256,
74,
fill,
);
moe_check::<BlockIQ2s, _>(
&dev,
&mut log,
"IQ2_S",
RocmQuantType::IQ2_S,
8,
16,
128,
512,
256,
82,
fill,
);
eprintln!("{log}");
}