mirror of
https://github.com/zooai/brain.git
synced 2026-10-30 14:50:48 +00:00
Closed every remaining π‘ / π΅ row in PARITY.md. Each algorithm FortΓ©mi ships now has a real implementation in @hanzo/bot-memory v2.0.0 with matching ports in hanzo-memory (Python), bot-go (Go), and hanzoai/mcp (Rust). Cross-runtime tests: 121 TS + 53 Python + 58 Go + 38 Rust = 270 passing. Landed under packages/memory/ Retrieval & ranking fusion RRF (k=20 BEIR) + RSF (Weaviate v1.24) + adaptive k + adaptive FTS/semantic weights rerank MMR (Carbonell & Goldstein 1998), cosine dedup chunk-aware deduplication script Unicode script detection (Latin/CJK/emoji/Cyrillic/ Arabic/Hebrew/Greek/Devanagari) fts feature-flag matrix, CJK bigrams, emoji trigrams, websearch_to_tsquery parser, SQLite FTS5 MATCH renderer embed embedding registry, asymmetric E5 prefixes, Matryoshka truncation, coarse-dim picker two-stage coarse β fine retrieval via MRL federated cross-archive search with RRF fusion multi-memory X-Hanzo-Memory routing (X-Fortemi-Memory alias) filters composable tag/temporal/collection/scope predicates temporal UUIDv7 floor/ceiling + named ranges spatial Haversine + bbox + in-box (PostGIS fallback) Knowledge & graph graph normalize β SNN β PFNET-β β Louvain maintenance pipeline skos W3C SKOS concept tree (broader/narrower closures, Turtle export) fair Dublin Core XML + JSON-LD doc-types document type registry + auto-detect (filename/MIME/ content sniff) captions WebVTT / SRT / RTTM renderers sprite thumbnail sprite-sheet WebVTT cue builder code-ast language-aware syntactic chunker email RFC 2822 / MIME parser (multipart, qp, base64, HTML strip) exif EXIF parser (camera / time / GPS) Operational events EventBus + SSE renderer + webhook HMAC signer tokenizer BPE-style estimator + truncate-to-budget eval MRR / recall@k / precision@k / NDCG@k / benchmark hardware tier autodetect + per-tier model recommender circuit-breaker closed / open / half-open retry exponential backoff with full jitter range HTTP Range parser + Content-Range formatter tus TUS v1.0.0 (Create / Patch / metadata / checksum) Crypto address wallet-style content-addressable ids (hanzo: / mm:) mmpke01 multi-recipient envelope parser/builder + magic-byte sniff Inference (packages/memory/inference/) index InferenceBackend trait + registry slug provider-qualified routing (provider:model:tag) capabilities capability flags + task-based selector runtime-config hot-swap config (db_override β env β default) mock deterministic backend (for tests) few-shot ICL prompt builder (3-5 examples capped) refinement Self-Refine / ReAct / Reflexion patterns link-types rule + LLM link classifier (11 canonical types) vision VisionBackend trait + Ollama default transcription Whisper-compatible HTTP client diarization pyannote-compatible HTTP client + alignSpeakers / relabel gliner GLiNER zero-shot NER HTTP client Extraction (packages/memory/extract/) index registry + dispatcher text native UTF-8 passthrough email RFC 2822 adapter spreadsheet CSV/TSV pure-JS; xlsx via registerXlsxDecoder archive ZIP central-directory pure-JS reader exif-adapter EXIF for images code tree-sitter-style syntactic chunker pdf PdfSidecar HTTP adapter factory image VisionBackend-backed adapter factory audio TranscriptionBackend-backed adapter factory video keyframe + scene + transcript adapter factory 3d multi-view 3D model adapter factory media-optimize ffmpeg variants (faststart / web-compat / 720p) adapter factory structured JSON-schema structured-extract adapter factory summarizer fast/standard cascade adapter factory sprite-adapter thumbnail sprite-sheet adapter factory Cross-runtime ports hanzo-memory (Python) 53 tests, ports of fusion / rerank / dedup / script / fts / embed / temporal / captions / tokenizer / eval / spatial / range / address / graph / doc-types / circuit / retry / slug / runtime-config / link-types bot-go (Go) 58 tests, same algorithm set ported with standard-library-only primitives hanzoai/mcp::brain (Rust) 38 tests, same algorithm set; sha2 added as dependency (blake3 stand-in) PARITY.md is now 100% green (β in-monorepo + π’ sibling-repo). No π΅ placeholders remain.
120 lines
3.7 KiB
TypeScript
120 lines
3.7 KiB
TypeScript
/**
|
|
* TUS resumable-upload protocol v1.0.0 β pure parser/builder.
|
|
*
|
|
* - Creation extension: client posts POST with `Upload-Length` and
|
|
* optional `Upload-Metadata`, server replies 201 with `Location`.
|
|
* - Patch: client sends `PATCH` with `Upload-Offset`, server stores
|
|
* and bumps offset.
|
|
* - Termination: `DELETE` removes the partial upload.
|
|
* - Checksum extension: `Upload-Checksum: <algo> <base64>`.
|
|
*
|
|
* This module returns the protocol decisions; storage is provided by the
|
|
* caller (so any backend works).
|
|
*/
|
|
|
|
export interface TusCreateOpts {
|
|
uploadLength: number;
|
|
/** Decoded metadata key-value pairs from `Upload-Metadata`. */
|
|
metadata?: Record<string, string>;
|
|
/** Caller-allocated upload id (URL component). */
|
|
uploadId: string;
|
|
}
|
|
|
|
export interface TusCreatedResponse {
|
|
status: 201;
|
|
headers: Record<string, string>;
|
|
}
|
|
|
|
export function tusCreate(opts: TusCreateOpts): TusCreatedResponse {
|
|
return {
|
|
status: 201,
|
|
headers: {
|
|
"Tus-Resumable": "1.0.0",
|
|
"Location": `/uploads/${opts.uploadId}`,
|
|
"Upload-Offset": "0",
|
|
},
|
|
};
|
|
}
|
|
|
|
export interface TusPatchInput {
|
|
/** Current persisted offset. */
|
|
currentOffset: number;
|
|
/** `Upload-Offset` from the request. */
|
|
uploadOffset: number;
|
|
/** Chunk size from `Content-Length`. */
|
|
chunkSize: number;
|
|
/** Total declared length. */
|
|
uploadLength: number;
|
|
/** Optional checksum header value, e.g. "sha256 <b64>". */
|
|
checksum?: string;
|
|
}
|
|
|
|
export interface TusPatchResponse {
|
|
status: 204 | 409 | 460;
|
|
headers: Record<string, string>;
|
|
/** True when the upload has reached uploadLength. */
|
|
complete: boolean;
|
|
/** New persisted offset. */
|
|
newOffset: number;
|
|
}
|
|
|
|
export function tusPatch(input: TusPatchInput): TusPatchResponse {
|
|
if (input.uploadOffset !== input.currentOffset) {
|
|
return {
|
|
status: 409,
|
|
headers: { "Tus-Resumable": "1.0.0", "Upload-Offset": String(input.currentOffset) },
|
|
complete: false,
|
|
newOffset: input.currentOffset,
|
|
};
|
|
}
|
|
const next = input.currentOffset + input.chunkSize;
|
|
if (next > input.uploadLength) {
|
|
return {
|
|
status: 409,
|
|
headers: { "Tus-Resumable": "1.0.0", "Upload-Offset": String(input.currentOffset) },
|
|
complete: false,
|
|
newOffset: input.currentOffset,
|
|
};
|
|
}
|
|
return {
|
|
status: 204,
|
|
headers: { "Tus-Resumable": "1.0.0", "Upload-Offset": String(next) },
|
|
complete: next === input.uploadLength,
|
|
newOffset: next,
|
|
};
|
|
}
|
|
|
|
/** Decode the `Upload-Metadata` header (RFC: comma-separated `key base64value`). */
|
|
export function parseUploadMetadata(header: string | undefined): Record<string, string> {
|
|
if (!header) return {};
|
|
const out: Record<string, string> = {};
|
|
for (const pair of header.split(",")) {
|
|
const trimmed = pair.trim();
|
|
if (!trimmed) continue;
|
|
const sp = trimmed.indexOf(" ");
|
|
if (sp === -1) { out[trimmed] = ""; continue; }
|
|
const key = trimmed.slice(0, sp);
|
|
const b64 = trimmed.slice(sp + 1);
|
|
try {
|
|
out[key] = Buffer.from(b64, "base64").toString("utf8");
|
|
} catch {
|
|
out[key] = "";
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Verify an `Upload-Checksum` header against a chunk. */
|
|
export async function verifyChecksum(checksum: string, chunk: Uint8Array): Promise<boolean> {
|
|
const sp = checksum.indexOf(" ");
|
|
if (sp === -1) return false;
|
|
const algo = checksum.slice(0, sp).toLowerCase();
|
|
const expected = checksum.slice(sp + 1);
|
|
if (algo !== "sha256" && algo !== "sha1" && algo !== "md5") return false;
|
|
const algoName = { sha256: "SHA-256", sha1: "SHA-1", md5: "MD5" }[algo] ?? "SHA-256";
|
|
if (algo === "md5") return false; // not supported by SubtleCrypto
|
|
const buf = await crypto.subtle.digest(algoName, chunk);
|
|
const got = Buffer.from(buf).toString("base64");
|
|
return got === expected;
|
|
}
|