Files
zeekay 25011a2085 brain: 100% fortemi parity β€” every algorithm shipped across all 4 runtimes
Closed every remaining 🟑 / πŸ”΅ row in PARITY.md. Each algorithm FortΓ©mi
ships now has a real implementation in @hanzo/bot-memory v2.0.0 with
matching ports in hanzo-memory (Python), bot-go (Go), and hanzoai/mcp
(Rust). Cross-runtime tests: 121 TS + 53 Python + 58 Go + 38 Rust = 270
passing.

Landed under packages/memory/

  Retrieval & ranking
    fusion       RRF (k=20 BEIR) + RSF (Weaviate v1.24) + adaptive k +
                 adaptive FTS/semantic weights
    rerank       MMR (Carbonell & Goldstein 1998), cosine
    dedup        chunk-aware deduplication
    script       Unicode script detection (Latin/CJK/emoji/Cyrillic/
                 Arabic/Hebrew/Greek/Devanagari)
    fts          feature-flag matrix, CJK bigrams, emoji trigrams,
                 websearch_to_tsquery parser, SQLite FTS5 MATCH renderer
    embed        embedding registry, asymmetric E5 prefixes, Matryoshka
                 truncation, coarse-dim picker
    two-stage    coarse β†’ fine retrieval via MRL
    federated    cross-archive search with RRF fusion
    multi-memory X-Hanzo-Memory routing (X-Fortemi-Memory alias)
    filters      composable tag/temporal/collection/scope predicates
    temporal     UUIDv7 floor/ceiling + named ranges
    spatial      Haversine + bbox + in-box (PostGIS fallback)

  Knowledge & graph
    graph        normalize β†’ SNN β†’ PFNET-∞ β†’ Louvain maintenance pipeline
    skos         W3C SKOS concept tree (broader/narrower closures,
                 Turtle export)
    fair         Dublin Core XML + JSON-LD
    doc-types    document type registry + auto-detect (filename/MIME/
                 content sniff)
    captions     WebVTT / SRT / RTTM renderers
    sprite       thumbnail sprite-sheet WebVTT cue builder
    code-ast     language-aware syntactic chunker
    email        RFC 2822 / MIME parser (multipart, qp, base64, HTML strip)
    exif         EXIF parser (camera / time / GPS)

  Operational
    events       EventBus + SSE renderer + webhook HMAC signer
    tokenizer    BPE-style estimator + truncate-to-budget
    eval         MRR / recall@k / precision@k / NDCG@k / benchmark
    hardware     tier autodetect + per-tier model recommender
    circuit-breaker  closed / open / half-open
    retry        exponential backoff with full jitter
    range        HTTP Range parser + Content-Range formatter
    tus          TUS v1.0.0 (Create / Patch / metadata / checksum)

  Crypto
    address      wallet-style content-addressable ids (hanzo: / mm:)
    mmpke01      multi-recipient envelope parser/builder + magic-byte sniff

  Inference (packages/memory/inference/)
    index        InferenceBackend trait + registry
    slug         provider-qualified routing (provider:model:tag)
    capabilities capability flags + task-based selector
    runtime-config hot-swap config (db_override β†’ env β†’ default)
    mock         deterministic backend (for tests)
    few-shot     ICL prompt builder (3-5 examples capped)
    refinement   Self-Refine / ReAct / Reflexion patterns
    link-types   rule + LLM link classifier (11 canonical types)
    vision       VisionBackend trait + Ollama default
    transcription Whisper-compatible HTTP client
    diarization  pyannote-compatible HTTP client + alignSpeakers / relabel
    gliner       GLiNER zero-shot NER HTTP client

  Extraction (packages/memory/extract/)
    index        registry + dispatcher
    text         native UTF-8 passthrough
    email        RFC 2822 adapter
    spreadsheet  CSV/TSV pure-JS; xlsx via registerXlsxDecoder
    archive      ZIP central-directory pure-JS reader
    exif-adapter EXIF for images
    code         tree-sitter-style syntactic chunker
    pdf          PdfSidecar HTTP adapter factory
    image        VisionBackend-backed adapter factory
    audio        TranscriptionBackend-backed adapter factory
    video        keyframe + scene + transcript adapter factory
    3d           multi-view 3D model adapter factory
    media-optimize ffmpeg variants (faststart / web-compat / 720p)
                   adapter factory
    structured   JSON-schema structured-extract adapter factory
    summarizer   fast/standard cascade adapter factory
    sprite-adapter thumbnail sprite-sheet adapter factory

Cross-runtime ports
  hanzo-memory (Python)      53 tests, ports of fusion / rerank / dedup /
                             script / fts / embed / temporal / captions /
                             tokenizer / eval / spatial / range / address /
                             graph / doc-types / circuit / retry / slug /
                             runtime-config / link-types
  bot-go (Go)                58 tests, same algorithm set ported with
                             standard-library-only primitives
  hanzoai/mcp::brain (Rust)  38 tests, same algorithm set; sha2 added as
                             dependency (blake3 stand-in)

PARITY.md is now 100% green (βœ… in-monorepo + 🟒 sibling-repo). No πŸ”΅
placeholders remain.
2026-05-11 11:44:08 -07:00

88 lines
3.9 KiB
TypeScript

/**
* MMPKE01 multi-recipient envelope format detector + magic-byte sniff.
*
* Wire format:
* β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
* β”‚ Magic: "MMPKE01\n" (8 bytes) β”‚
* β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
* β”‚ Header length: u32 little-endian (4 bytes) β”‚
* β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
* β”‚ Header: JSON { ephemeralPub, recipients: [...] }β”‚
* β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€
* β”‚ AES-256-GCM ciphertext of body β”‚
* β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
*
* Cryptographic ops (X25519 ECDH + HKDF-SHA256 + AES-256-GCM) are
* delegated to `@hanzo/crypto` (Rust via WASM or Node native) so the
* brain stays runtime-agnostic.
*/
export const MAGIC = new Uint8Array([0x4d, 0x4d, 0x50, 0x4b, 0x45, 0x30, 0x31, 0x0a]);
export interface MmpkeRecipient {
/** Wallet-style address β€” e.g. `hanzo:1A1zP1eP5...` or `mm:...`. */
address: string;
/** Base64-encoded encrypted DEK (wrapped with this recipient's KEK). */
encryptedDek: string;
/** Base64-encoded GCM nonce used for the DEK wrap. */
nonce: string;
}
export interface MmpkeHeader {
version: 1;
/** Base64-encoded ephemeral X25519 public key. */
ephemeralPub: string;
recipients: MmpkeRecipient[];
/** Optional metadata β€” propagated to consumers but not authenticated by GCM. */
metadata?: Record<string, unknown>;
}
export interface MmpkeParsed {
header: MmpkeHeader;
ciphertext: Uint8Array;
}
/** True if `buf` begins with the MMPKE01 magic bytes. */
export function isMmpke01(buf: Uint8Array): boolean {
if (buf.length < MAGIC.length) return false;
for (let i = 0; i < MAGIC.length; i++) if (buf[i] !== MAGIC[i]) return false;
return true;
}
/** Parse the on-disk MMPKE01 envelope structure (no decryption). */
export function parseMmpke01(buf: Uint8Array): MmpkeParsed {
if (!isMmpke01(buf)) throw new Error("mmpke01: bad magic");
if (buf.length < MAGIC.length + 4) throw new Error("mmpke01: truncated header length");
const hl =
buf[MAGIC.length] |
(buf[MAGIC.length + 1] << 8) |
(buf[MAGIC.length + 2] << 16) |
(buf[MAGIC.length + 3] << 24);
const headerStart = MAGIC.length + 4;
if (headerStart + hl > buf.length) throw new Error("mmpke01: truncated header body");
const headerJson = new TextDecoder().decode(buf.subarray(headerStart, headerStart + hl));
const header = JSON.parse(headerJson) as MmpkeHeader;
const ciphertext = buf.subarray(headerStart + hl);
return { header, ciphertext };
}
/** Build the on-disk MMPKE01 envelope from header + ciphertext. */
export function buildMmpke01(header: MmpkeHeader, ciphertext: Uint8Array): Uint8Array {
const headerJson = new TextEncoder().encode(JSON.stringify(header));
const out = new Uint8Array(MAGIC.length + 4 + headerJson.length + ciphertext.length);
out.set(MAGIC, 0);
const hl = headerJson.length;
out[MAGIC.length + 0] = hl & 0xff;
out[MAGIC.length + 1] = (hl >> 8) & 0xff;
out[MAGIC.length + 2] = (hl >> 16) & 0xff;
out[MAGIC.length + 3] = (hl >> 24) & 0xff;
out.set(headerJson, MAGIC.length + 4);
out.set(ciphertext, MAGIC.length + 4 + headerJson.length);
return out;
}
/** Find a recipient block for the given address. */
export function recipientFor(header: MmpkeHeader, address: string): MmpkeRecipient | undefined {
return header.recipients.find((r) => r.address === address);
}