mirror of
https://github.com/hanzoai/papers.git
synced 2026-10-22 17:20:17 +00:00
zeekay
f041a6c1ca
Systems paper + numbers-as-data results. Honest board (Qwen3-1.7B-Q4_K_M, N=7, 95% CI): prefill at parity (CUDA WIN 1.09x at pp4096); decode behind on all four backends (CUDA 0.84, Metal 0.72, ROCm 0.72, Vulkan 0.52), decomposed into APU bandwidth-utilization (measured 213 GB/s peak; llama at 93%, us at 48-72%) and discrete fixed-overhead that a 4B control amortizes. Root-causes the Vulkan pp4096 cliff (not VK_WORK_CAP -- refuted). Overturns several internal claims against a properly-built baseline. latexmk-clean.
92 lines
3.6 KiB
BibTeX
92 lines
3.6 KiB
BibTeX
@inproceedings{paged2023,
|
|
title = {Efficient Memory Management for Large Language Model Serving with {PagedAttention}},
|
|
author = {Kwon, Woosuk and Li, Zhuohan and Zhuang, Siyuan and Sheng, Ying and Zheng, Lianmin and Yu, Cody Hao and Gonzalez, Joseph E. and Zhang, Hao and Stoica, Ion},
|
|
booktitle = {Proceedings of the 29th Symposium on Operating Systems Principles (SOSP)},
|
|
year = {2023},
|
|
note = {arXiv:2309.06180}
|
|
}
|
|
|
|
@article{sglang2024,
|
|
title = {{SGLang}: Efficient Execution of Structured Language Model Programs},
|
|
author = {Zheng, Lianmin and Yin, Liangsheng and Xie, Zhiqiang and Sun, Chuyue and Huang, Jeff and Yu, Cody Hao and Cao, Shiyi and Kozyrakis, Christos and Stoica, Ion and Gonzalez, Joseph E. and Barrett, Clark and Sheng, Ying},
|
|
journal = {Advances in Neural Information Processing Systems (NeurIPS)},
|
|
year = {2024},
|
|
note = {arXiv:2312.07104}
|
|
}
|
|
|
|
@inproceedings{orca2022,
|
|
title = {{Orca}: A Distributed Serving System for Transformer-Based Generative Models},
|
|
author = {Yu, Gyeong-In and Jeong, Joo Seong and Kim, Geon-Woo and Kim, Soojeong and Chun, Byung-Gon},
|
|
booktitle = {16th USENIX Symposium on Operating Systems Design and Implementation (OSDI)},
|
|
year = {2022}
|
|
}
|
|
|
|
@article{flashattention2022,
|
|
title = {{FlashAttention}: Fast and Memory-Efficient Exact Attention with {IO}-Awareness},
|
|
author = {Dao, Tri and Fu, Daniel Y. and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
|
|
journal = {Advances in Neural Information Processing Systems (NeurIPS)},
|
|
year = {2022},
|
|
note = {arXiv:2205.14135}
|
|
}
|
|
|
|
@article{flashattention2_2023,
|
|
title = {{FlashAttention-2}: Faster Attention with Better Parallelism and Work Partitioning},
|
|
author = {Dao, Tri},
|
|
journal = {arXiv preprint arXiv:2307.08691},
|
|
year = {2023}
|
|
}
|
|
|
|
@article{kivi2024,
|
|
title = {{KIVI}: A Tuning-Free Asymmetric 2bit Quantization for {KV} Cache},
|
|
author = {Liu, Zirui and Yuan, Jiayi and Jin, Hongye and Zhong, Shaochen and Xu, Zhaozhuo and Braverman, Vladimir and Chen, Beidi and Hu, Xia},
|
|
journal = {International Conference on Machine Learning (ICML)},
|
|
year = {2024},
|
|
note = {arXiv:2402.02750}
|
|
}
|
|
|
|
@article{leviathan2023,
|
|
title = {Fast Inference from Transformers via Speculative Decoding},
|
|
author = {Leviathan, Yaniv and Kalman, Matan and Matias, Yossi},
|
|
journal = {International Conference on Machine Learning (ICML)},
|
|
year = {2023},
|
|
note = {arXiv:2211.17192}
|
|
}
|
|
|
|
@misc{promptlookup2023,
|
|
title = {Prompt Lookup Decoding},
|
|
author = {Saxena, Apoorv},
|
|
year = {2023},
|
|
howpublished = {\url{https://github.com/apoorvumang/prompt-lookup-decoding}}
|
|
}
|
|
|
|
@article{roofline2009,
|
|
title = {Roofline: An Insightful Visual Performance Model for Multicore Architectures},
|
|
author = {Williams, Samuel and Waterman, Andrew and Patterson, David},
|
|
journal = {Communications of the ACM},
|
|
volume = {52},
|
|
number = {4},
|
|
pages = {65--76},
|
|
year = {2009}
|
|
}
|
|
|
|
@misc{llamacpp,
|
|
title = {llama.cpp: {LLM} inference in {C/C++}},
|
|
author = {Gerganov, Georgi and {llama.cpp contributors}},
|
|
howpublished = {\url{https://github.com/ggml-org/llama.cpp}},
|
|
note = {Accessed 2026}
|
|
}
|
|
|
|
@misc{tensorrtllm,
|
|
title = {{TensorRT-LLM}},
|
|
author = {{NVIDIA}},
|
|
howpublished = {\url{https://github.com/NVIDIA/TensorRT-LLM}},
|
|
note = {Accessed 2026}
|
|
}
|
|
|
|
@misc{mistralrs,
|
|
title = {mistral.rs: blazingly fast {LLM} inference},
|
|
author = {Buehler, Eric L. and {contributors}},
|
|
howpublished = {\url{https://github.com/EricLBuehler/mistral.rs}},
|
|
note = {Upstream lineage of hanzo-engine}
|
|
}
|