mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
26 lines
768 B
TOML
26 lines
768 B
TOML
[package]
|
|
name = "hanzo-tokenizer"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
authors.workspace = true
|
|
license.workspace = true
|
|
rust-version.workspace = true
|
|
homepage.workspace = true
|
|
repository.workspace = true
|
|
description = "Clean-room high-throughput BPE tokenizer for Hanzo inference."
|
|
|
|
[dependencies]
|
|
aho-corasick = { workspace = true }
|
|
parking_lot = { workspace = true }
|
|
rayon = { workspace = true }
|
|
regex = { workspace = true }
|
|
rustc-hash = { workspace = true }
|
|
serde = { workspace = true, features = ["derive"] }
|
|
serde_json = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
unicode_categories = { workspace = true }
|
|
unicode-normalization-alignments = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
tokenizers = { workspace = true, features = ["onig"] }
|