Files

26 lines
768 B
TOML

[package]
name = "hanzo-tokenizer"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
rust-version.workspace = true
homepage.workspace = true
repository.workspace = true
description = "Clean-room high-throughput BPE tokenizer for Hanzo inference."
[dependencies]
aho-corasick = { workspace = true }
parking_lot = { workspace = true }
rayon = { workspace = true }
regex = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
thiserror = { workspace = true }
unicode_categories = { workspace = true }
unicode-normalization-alignments = { workspace = true }
[dev-dependencies]
tokenizers = { workspace = true, features = ["onig"] }