mirror of
https://github.com/zenlm/zen-router.git
synced 2026-10-30 14:57:44 +00:00
zeekay
f28120f1e0
Declarative suite registry (suites.yaml, 12 verified HF dataset ids), build_profiles.py to convert per-example/per-model records into router rows, replicate.py for the cost-quality frontier + AIQ metric with individual/random/ oracle baselines, prices.yaml fallback, and bench-data/bench-replicate Make targets. Validated on a 200-sample RouterBench slice; row-conversion + metric tests in tests/test_benchmarks.py.
39 lines
1.1 KiB
Makefile
39 lines
1.1 KiB
Makefile
.PHONY: data train grpo eval quantize test upload bench-data bench-replicate
|
|
|
|
PYTHON ?= uv run python
|
|
|
|
# --- benchmarks: replicate published LLM-routing research (see benchmarks/) ---
|
|
BENCH_SUITE ?= routerbench
|
|
BENCH_SAMPLE ?= 200
|
|
BENCH_DATA ?= data/profiles-$(BENCH_SUITE).jsonl
|
|
BENCH_POLICY ?= oracle
|
|
|
|
data:
|
|
$(PYTHON) training/build_dataset.py --evals data/evals.jsonl --ledger data/usage-ledger.jsonl --out data/routing-sft.jsonl
|
|
|
|
train:
|
|
$(PYTHON) training/sft.py --config training/config.yaml
|
|
|
|
grpo:
|
|
$(PYTHON) training/grpo.py --config training/config.yaml
|
|
|
|
eval:
|
|
$(PYTHON) eval/eval_routing.py --model out/zen-router --data data/routing-eval.jsonl
|
|
|
|
quantize:
|
|
scripts/quantize.sh out/zen-router
|
|
|
|
test:
|
|
uv run pytest -q
|
|
|
|
upload:
|
|
$(PYTHON) upload_to_huggingface.py --repo zenlm/zen-router --path out/zen-router
|
|
|
|
bench-data:
|
|
uv run --with datasets,huggingface_hub,pandas python benchmarks/build_profiles.py \
|
|
--suite $(BENCH_SUITE) --sample $(BENCH_SAMPLE) --out $(BENCH_DATA)
|
|
|
|
bench-replicate:
|
|
$(PYTHON) benchmarks/replicate.py --data $(BENCH_DATA) --policy $(BENCH_POLICY) \
|
|
--out-prefix out/bench/$(BENCH_SUITE)
|