mirror of
https://github.com/hanzoai/ANE.git
synced 2026-10-30 18:53:51 +00:00
maderix
3f73962dc7
Implement Grouped-Query Attention (16q/8kv heads, head_dim=128) for Qwen3-0.6B (28 layers, 596M params). Model configs moved to models/*.h headers selected at build time via make MODEL=xxx. Key changes: - GQA-aware MIL kernels: sdpaFwd split from woFwd (Q_DIM!=DIM), qBwd/kvBwd split from qkvBwd (different IC dimensions) - K/V tile (KV_HEADS→HEADS) before SDPA backward, reduce after - 10 kernels total, all model-agnostic via compile-time defines - Makefile: make MODEL=qwen3_06b (default) or MODEL=stories110m - Both models verified: Stories110M ~115ms/step, Qwen3 ~412ms/step
18 lines
535 B
Makefile
18 lines
535 B
Makefile
CC = xcrun clang
|
|
CFLAGS = -O2 -DACCELERATE_NEW_LAPACK -framework Foundation -framework IOSurface -framework Accelerate \
|
|
-isysroot $(shell xcrun --show-sdk-path) -fobjc-arc
|
|
|
|
# Model selection: make MODEL=qwen3_06b (default)
|
|
# Available: stories110m, qwen3_06b
|
|
MODEL ?= qwen3_06b
|
|
MODEL_HDR = models/$(MODEL).h
|
|
|
|
train: train.m config.h io.h cpu_ops.h mil_dynamic.h $(MODEL_HDR)
|
|
@echo "Building for model: $(MODEL)"
|
|
$(CC) $(CFLAGS) -include $(MODEL_HDR) -o train train.m
|
|
|
|
clean:
|
|
rm -f train
|
|
|
|
.PHONY: clean
|