Files
zeekay 92e5c2412d merge: upstream/main (-X ours, brand preserved)
# Conflicts:
#	candle-core/benches/benchmarks/broadcast.rs
#	candle-examples/examples/quantized-lfm2/main.rs
#	candle-examples/examples/smollm3/main.rs
#	candle-flash-attn-v3/Cargo.toml
#	candle-flash-attn-v3/build.rs
#	candle-flash-attn-v3/hkernel/flash_api.cu
#	candle-flash-attn-v3/hkernel/flash_fwd_launch_template.h
#	candle-flash-attn-v3/hkernel/static_switch.h
#	candle-flash-attn-v3/src/ffi.rs
#	candle-flash-attn-v3/src/lib.rs
#	candle-flash-attn-v3/tests/flash_attn_tests.rs
#	candle-kernels/Cargo.toml
#	candle-kernels/build.rs
#	candle-metal-kernels/src/kernels/mlx_gemm.rs
#	candle-nn/tests/cpu_flash_attn.rs
#	candle-transformers/src/models/quantized_lfm2.rs
#	hanzo-flash-attn/kernels/flash_fwd_hdim512_bf16_causal_sm80.cu
#	hanzo-flash-attn/kernels/flash_fwd_hdim512_bf16_sm80.cu
#	hanzo-flash-attn/kernels/flash_fwd_hdim512_fp16_causal_sm80.cu
#	hanzo-flash-attn/kernels/flash_fwd_hdim512_fp16_sm80.cu
#	hanzo-kernels/src/mmq_gguf/mmq_common.cuh
#	hanzo-kernels/src/mmq_gguf/mmq_gguf.cuh
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q2_k.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q3_k.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q4_0.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q4_1.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q4_k.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q5_0.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q5_1.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q5_k.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q6_k.cu
#	hanzo-kernels/src/mmq_gguf/mmq_instance_q8_0.cu
#	hanzo-kernels/src/mmq_gguf/mmq_mma.cuh
#	hanzo-kernels/src/mmq_gguf/mmq_quantize.cu
#	hanzo-kernels/src/mmq_gguf/mmq_vecdotq.cuh
#	hanzo-kernels/src/mmvq_gguf.cu
#	hanzo-metal-kernels/src/metal/fence.rs
#	hanzo-metal-kernels/src/metal/residency_set.rs
#	hanzo-metal-kernels/src/metal_src/gemv.metal
#	hanzo-ml/benches/benchmarks/cat.rs
#	hanzo-ml/benches/benchmarks/contiguous.rs
#	hanzo-ml/benches/benchmarks/vec_dot.rs
#	hanzo-ml/src/quantized/fast_mmq.rs
#	hanzo-ml/src/quantized/fast_mmvq.rs
#	hanzo-nn/tests/varlen_flash_attention.rs
2026-06-02 10:20:51 -07:00

199 lines
5.1 KiB
JavaScript

export async function extractEmbeddings(
worker,
weightsURL,
tokenizerURL,
configURL,
modelID,
sentences,
updateStatus,
normalize_embeddings = true
) {
return new Promise((resolve, reject) => {
worker.postMessage({
weightsURL,
tokenizerURL,
configURL,
modelID,
sentences,
normalize_embeddings,
});
function messageHandler(event) {
if ("error" in event.data) {
worker.removeEventListener("message", messageHandler);
reject(new Error(event.data.error));
}
if (event.data.status === "complete") {
worker.removeEventListener("message", messageHandler);
resolve(event.data);
}
if (updateStatus) updateStatus(event.data);
}
worker.addEventListener("message", messageHandler);
});
}
export async function generateText(
worker,
weightsURL,
tokenizerURL,
configURL,
modelID,
prompt,
params,
updateStatus
) {
return new Promise((resolve, reject) => {
worker.postMessage({
weightsURL,
tokenizerURL,
configURL,
modelID,
prompt,
params,
});
function messageHandler(event) {
if ("error" in event.data) {
worker.removeEventListener("message", messageHandler);
reject(new Error(event.data.error));
}
if (event.data.status === "complete") {
worker.removeEventListener("message", messageHandler);
resolve(event.data);
}
if (updateStatus) updateStatus(event.data);
}
worker.addEventListener("message", messageHandler);
});
}
export const MODELS = {
t5_small_quantized: {
size: "64.4 MB",
base_url: "https://huggingface.co/lmz/hanzo-ml-quantized-t5/resolve/main/",
model: "model.gguf",
tokenizer: "tokenizer.json",
config: "config.json",
tasks: {
translation_en_to_de: {
prefix: "translate English to German: ",
max_length: 300,
},
translation_en_to_fr: {
prefix: "translate English to French: ",
max_length: 300,
},
translation_en_to_ro: {
prefix: "translate English to Romanian: ",
max_length: 300,
},
summarization: { prefix: "summarize: ", max_length: 200 },
},
},
t5_small: {
size: "242 MB",
base_url: "https://huggingface.co/t5-small/resolve/main/",
model: "model.safetensors",
tokenizer: "tokenizer.json",
config: "config.json",
tasks: {
translation_en_to_de: {
prefix: "translate English to German: ",
max_length: 300,
},
translation_en_to_fr: {
prefix: "translate English to French: ",
max_length: 300,
},
translation_en_to_ro: {
prefix: "translate English to Romanian: ",
max_length: 300,
},
summarization: { prefix: "summarize: ", max_length: 200 },
},
},
flan_t5_small: {
size: "308 MB",
base_url:
"https://huggingface.co/google/flan-t5-small/resolve/refs%2Fpr%2F14/",
model: "model.safetensors",
tokenizer: "tokenizer.json",
config: "config.json",
tasks: {
translation_en_to_de: {
prefix: "translate English to German: ",
max_length: 300,
},
translation_en_to_fr: {
prefix: "translate English to French: ",
max_length: 300,
},
translation_en_to_ro: {
prefix: "translate English to Romanian: ",
max_length: 300,
},
summarization: { prefix: "summarize: ", max_length: 200 },
},
},
flan_t5_base_quantized: {
size: "263 MB",
base_url: "https://huggingface.co/lmz/hanzo-ml-quantized-t5/resolve/main/",
model: "model-flan-t5-base.gguf",
tokenizer: "tokenizer.json",
config: "config-flan-t5-base.json",
tasks: {
translation_en_to_de: {
prefix: "translate English to German: ",
max_length: 300,
},
translation_en_to_fr: {
prefix: "translate English to French: ",
max_length: 300,
},
translation_en_to_ro: {
prefix: "translate English to Romanian: ",
max_length: 300,
},
summarization: { prefix: "summarize: ", max_length: 200 },
},
},
coedit_large_quantized: {
size: "643 MB",
base_url: "https://huggingface.co/jbochi/hanzo-ml-coedit-quantized/resolve/main/",
model: "model.gguf",
tokenizer: "tokenizer.json",
config: "config.json",
tasks: {
fluency: {
prefix: "Fix the grammar: ",
max_length: 300,
},
coherence: {
prefix: "Rewrite to make this easier to understand: ",
max_length: 300,
},
simplification: {
prefix: "Paraphrase this: ",
max_length: 300,
},
formalization: {
prefix: "Write this more formally: ",
max_length: 300,
},
neutralize: {
prefix: "Write in a more neutral way: ",
max_length: 300,
},
},
},
};
export function getModelInfo(id, taskID) {
const model = MODELS[id];
return {
modelURL: model.base_url + model.model,
configURL: model.base_url + model.config,
tokenizerURL: model.base_url + model.tokenizer,
maxLength: model.tasks[taskID].max_length,
};
}