mirror of
https://github.com/hanzoai/ml.git
synced 2026-11-20 23:08:29 +00:00
# Conflicts: # candle-core/benches/benchmarks/broadcast.rs # candle-examples/examples/quantized-lfm2/main.rs # candle-examples/examples/smollm3/main.rs # candle-flash-attn-v3/Cargo.toml # candle-flash-attn-v3/build.rs # candle-flash-attn-v3/hkernel/flash_api.cu # candle-flash-attn-v3/hkernel/flash_fwd_launch_template.h # candle-flash-attn-v3/hkernel/static_switch.h # candle-flash-attn-v3/src/ffi.rs # candle-flash-attn-v3/src/lib.rs # candle-flash-attn-v3/tests/flash_attn_tests.rs # candle-kernels/Cargo.toml # candle-kernels/build.rs # candle-metal-kernels/src/kernels/mlx_gemm.rs # candle-nn/tests/cpu_flash_attn.rs # candle-transformers/src/models/quantized_lfm2.rs # hanzo-flash-attn/kernels/flash_fwd_hdim512_bf16_causal_sm80.cu # hanzo-flash-attn/kernels/flash_fwd_hdim512_bf16_sm80.cu # hanzo-flash-attn/kernels/flash_fwd_hdim512_fp16_causal_sm80.cu # hanzo-flash-attn/kernels/flash_fwd_hdim512_fp16_sm80.cu # hanzo-kernels/src/mmq_gguf/mmq_common.cuh # hanzo-kernels/src/mmq_gguf/mmq_gguf.cuh # hanzo-kernels/src/mmq_gguf/mmq_instance_q2_k.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q3_k.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q4_0.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q4_1.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q4_k.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q5_0.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q5_1.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q5_k.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q6_k.cu # hanzo-kernels/src/mmq_gguf/mmq_instance_q8_0.cu # hanzo-kernels/src/mmq_gguf/mmq_mma.cuh # hanzo-kernels/src/mmq_gguf/mmq_quantize.cu # hanzo-kernels/src/mmq_gguf/mmq_vecdotq.cuh # hanzo-kernels/src/mmvq_gguf.cu # hanzo-metal-kernels/src/metal/fence.rs # hanzo-metal-kernels/src/metal/residency_set.rs # hanzo-metal-kernels/src/metal_src/gemv.metal # hanzo-ml/benches/benchmarks/cat.rs # hanzo-ml/benches/benchmarks/contiguous.rs # hanzo-ml/benches/benchmarks/vec_dot.rs # hanzo-ml/src/quantized/fast_mmq.rs # hanzo-ml/src/quantized/fast_mmvq.rs # hanzo-nn/tests/varlen_flash_attention.rs
199 lines
5.1 KiB
JavaScript
199 lines
5.1 KiB
JavaScript
export async function extractEmbeddings(
|
|
worker,
|
|
weightsURL,
|
|
tokenizerURL,
|
|
configURL,
|
|
modelID,
|
|
sentences,
|
|
updateStatus,
|
|
normalize_embeddings = true
|
|
) {
|
|
return new Promise((resolve, reject) => {
|
|
worker.postMessage({
|
|
weightsURL,
|
|
tokenizerURL,
|
|
configURL,
|
|
modelID,
|
|
sentences,
|
|
normalize_embeddings,
|
|
});
|
|
function messageHandler(event) {
|
|
if ("error" in event.data) {
|
|
worker.removeEventListener("message", messageHandler);
|
|
reject(new Error(event.data.error));
|
|
}
|
|
if (event.data.status === "complete") {
|
|
worker.removeEventListener("message", messageHandler);
|
|
resolve(event.data);
|
|
}
|
|
if (updateStatus) updateStatus(event.data);
|
|
}
|
|
worker.addEventListener("message", messageHandler);
|
|
});
|
|
}
|
|
|
|
export async function generateText(
|
|
worker,
|
|
weightsURL,
|
|
tokenizerURL,
|
|
configURL,
|
|
modelID,
|
|
prompt,
|
|
params,
|
|
updateStatus
|
|
) {
|
|
return new Promise((resolve, reject) => {
|
|
worker.postMessage({
|
|
weightsURL,
|
|
tokenizerURL,
|
|
configURL,
|
|
modelID,
|
|
prompt,
|
|
params,
|
|
});
|
|
function messageHandler(event) {
|
|
if ("error" in event.data) {
|
|
worker.removeEventListener("message", messageHandler);
|
|
reject(new Error(event.data.error));
|
|
}
|
|
if (event.data.status === "complete") {
|
|
worker.removeEventListener("message", messageHandler);
|
|
resolve(event.data);
|
|
}
|
|
if (updateStatus) updateStatus(event.data);
|
|
}
|
|
worker.addEventListener("message", messageHandler);
|
|
});
|
|
}
|
|
|
|
export const MODELS = {
|
|
t5_small_quantized: {
|
|
size: "64.4 MB",
|
|
base_url: "https://huggingface.co/lmz/hanzo-ml-quantized-t5/resolve/main/",
|
|
model: "model.gguf",
|
|
tokenizer: "tokenizer.json",
|
|
config: "config.json",
|
|
tasks: {
|
|
translation_en_to_de: {
|
|
prefix: "translate English to German: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_fr: {
|
|
prefix: "translate English to French: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_ro: {
|
|
prefix: "translate English to Romanian: ",
|
|
max_length: 300,
|
|
},
|
|
summarization: { prefix: "summarize: ", max_length: 200 },
|
|
},
|
|
},
|
|
t5_small: {
|
|
size: "242 MB",
|
|
base_url: "https://huggingface.co/t5-small/resolve/main/",
|
|
model: "model.safetensors",
|
|
tokenizer: "tokenizer.json",
|
|
config: "config.json",
|
|
tasks: {
|
|
translation_en_to_de: {
|
|
prefix: "translate English to German: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_fr: {
|
|
prefix: "translate English to French: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_ro: {
|
|
prefix: "translate English to Romanian: ",
|
|
max_length: 300,
|
|
},
|
|
summarization: { prefix: "summarize: ", max_length: 200 },
|
|
},
|
|
},
|
|
flan_t5_small: {
|
|
size: "308 MB",
|
|
base_url:
|
|
"https://huggingface.co/google/flan-t5-small/resolve/refs%2Fpr%2F14/",
|
|
model: "model.safetensors",
|
|
tokenizer: "tokenizer.json",
|
|
config: "config.json",
|
|
tasks: {
|
|
translation_en_to_de: {
|
|
prefix: "translate English to German: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_fr: {
|
|
prefix: "translate English to French: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_ro: {
|
|
prefix: "translate English to Romanian: ",
|
|
max_length: 300,
|
|
},
|
|
summarization: { prefix: "summarize: ", max_length: 200 },
|
|
},
|
|
},
|
|
flan_t5_base_quantized: {
|
|
size: "263 MB",
|
|
base_url: "https://huggingface.co/lmz/hanzo-ml-quantized-t5/resolve/main/",
|
|
model: "model-flan-t5-base.gguf",
|
|
tokenizer: "tokenizer.json",
|
|
config: "config-flan-t5-base.json",
|
|
tasks: {
|
|
translation_en_to_de: {
|
|
prefix: "translate English to German: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_fr: {
|
|
prefix: "translate English to French: ",
|
|
max_length: 300,
|
|
},
|
|
translation_en_to_ro: {
|
|
prefix: "translate English to Romanian: ",
|
|
max_length: 300,
|
|
},
|
|
summarization: { prefix: "summarize: ", max_length: 200 },
|
|
},
|
|
},
|
|
coedit_large_quantized: {
|
|
size: "643 MB",
|
|
base_url: "https://huggingface.co/jbochi/hanzo-ml-coedit-quantized/resolve/main/",
|
|
model: "model.gguf",
|
|
tokenizer: "tokenizer.json",
|
|
config: "config.json",
|
|
tasks: {
|
|
fluency: {
|
|
prefix: "Fix the grammar: ",
|
|
max_length: 300,
|
|
},
|
|
coherence: {
|
|
prefix: "Rewrite to make this easier to understand: ",
|
|
max_length: 300,
|
|
},
|
|
simplification: {
|
|
prefix: "Paraphrase this: ",
|
|
max_length: 300,
|
|
},
|
|
formalization: {
|
|
prefix: "Write this more formally: ",
|
|
max_length: 300,
|
|
},
|
|
neutralize: {
|
|
prefix: "Write in a more neutral way: ",
|
|
max_length: 300,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
export function getModelInfo(id, taskID) {
|
|
const model = MODELS[id];
|
|
return {
|
|
modelURL: model.base_url + model.model,
|
|
configURL: model.base_url + model.config,
|
|
tokenizerURL: model.base_url + model.tokenizer,
|
|
maxLength: model.tasks[taskID].max_length,
|
|
};
|
|
}
|