mirror of
https://github.com/hanzoai/vfs.git
synced 2026-10-30 18:18:21 +00:00
zeekay
ffa5a24c01
Phase 0.1.0 — production-shaped scaffold for the S3-backed virtual block filesystem. Stateful services (IAM, BD, ATS, TA, KMS, onyxd, lqd snapshots) get unlimited disk via S3 with PQ-encrypted blocks; the hot tier is a configurable LRU cache. Companion to hanzo/replicate (WAL streaming) — `replicate` is file-level granularity, `vfs` is block-level (4 KiB pages, content-addressable via blake3-256). Included -------- - pkg/backend: pluggable Backend interface + Open(URL) dispatch. - pkg/backend/file: file:// backend (atomic put via temp+rename, path traversal blocked, recursive list with prefix filter). - pkg/backend/s3: AWS SDK v2 S3 backend with `?endpoint=` override for S3-compat (R2, MinIO, Hanzo Storage). NotFound → ErrNotFound. - pkg/vfs/block: blake3-256 hashing → BlockID, 2-hex shard prefix for the backend key, BlockSize=4096, zero-pad to BlockSize before encryption (block boundaries don't leak file lengths). - pkg/vfs/crypto: luxfi/age v1.5.0 wrapper. Recipients (write-side) + Identities (read-side); hybrid X25519 + ML-KEM-768 supported via the same recipient list. - pkg/vfs/cache: in-memory LRU bound by total bytes; touch-promotes on read; eviction on Put when over cap. NVMe write-back tier lands in 0.2.0. - pkg/vfs: top-level handle. PutBlock/GetBlock with cache hit-path + content-hash integrity check on read. Stats() surfaces cache + backend metadata. - pkg/mount: fuse_stub.go (default build) + fuse_unix.go (-tags fuse). Real FUSE wiring lands in 0.2.0; current stub returns a build-tag error pointing at `make build-fuse`. - cmd/vfs: cobra CLI (put/get/stats/mount). Backend registered via blank-import; --age-recipient repeatable; --age-key reads identity PEM from file or VFS_AGE_KEY env. Tests ----- - block: hash determinism, path shape, verify match/mismatch, pad. - cache: LRU eviction, touch-promotion, basic ops. - file backend: put/get/delete/stat/list, path-traversal block. - vfs roundtrip: encrypt → backend → decrypt → byte-equal plaintext, zero-pad correctness; multiple writes of identical plaintext both round-trip; stats reflect insertions. CI / build ---------- - .github/workflows/build.yml: vet + race-tests on Go 1.24, amd64 binary upload, GHCR image build + push on main / v* tags. - Dockerfile: distroless static, CGO_ENABLED=0, ARG VERSION ldflag. - Makefile: build / build-fuse / test / fmt / vet / check / image / install. VERSION-pinned. Roadmap ------- - 0.2.0: bazil.org/fuse mount, NVMe disk cache layer - 0.3.0: gcs + azureblob backends - 0.4.0: K8s sidecar mode + Helm chart - 0.5.0: WASI guest mode - 1.0.0: chunked uploads, multipart resume, GC for orphan blocks, metrics + SLO targets
97 lines
2.2 KiB
Go
97 lines
2.2 KiB
Go
package file
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/hanzoai/vfs/pkg/backend"
|
|
)
|
|
|
|
func newFileBE(t *testing.T) (backend.Backend, string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
be, err := backend.Open(context.Background(), "file://"+filepath.ToSlash(dir))
|
|
if err != nil {
|
|
t.Fatalf("open: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = be.Close() })
|
|
return be, dir
|
|
}
|
|
|
|
func TestFilePutGetDelete(t *testing.T) {
|
|
be, _ := newFileBE(t)
|
|
ctx := context.Background()
|
|
|
|
if err := be.Put(ctx, "blocks/aa/abc.zap.age", []byte("hello")); err != nil {
|
|
t.Fatalf("put: %v", err)
|
|
}
|
|
got, err := be.Get(ctx, "blocks/aa/abc.zap.age")
|
|
if err != nil {
|
|
t.Fatalf("get: %v", err)
|
|
}
|
|
if string(got) != "hello" {
|
|
t.Fatalf("got %q want hello", got)
|
|
}
|
|
|
|
sz, err := be.Stat(ctx, "blocks/aa/abc.zap.age")
|
|
if err != nil {
|
|
t.Fatalf("stat: %v", err)
|
|
}
|
|
if sz != 5 {
|
|
t.Fatalf("stat size %d want 5", sz)
|
|
}
|
|
|
|
if err := be.Delete(ctx, "blocks/aa/abc.zap.age"); err != nil {
|
|
t.Fatalf("delete: %v", err)
|
|
}
|
|
if _, err := be.Get(ctx, "blocks/aa/abc.zap.age"); !errors.Is(err, backend.ErrNotFound) {
|
|
t.Fatalf("expected ErrNotFound, got %v", err)
|
|
}
|
|
|
|
// Delete-of-absent must not error
|
|
if err := be.Delete(ctx, "blocks/zz/missing.zap.age"); err != nil {
|
|
t.Fatalf("idempotent delete: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFileList(t *testing.T) {
|
|
be, _ := newFileBE(t)
|
|
ctx := context.Background()
|
|
|
|
keys := []string{
|
|
"blocks/aa/1.zap.age",
|
|
"blocks/aa/2.zap.age",
|
|
"blocks/bb/3.zap.age",
|
|
}
|
|
for _, k := range keys {
|
|
if err := be.Put(ctx, k, []byte("x")); err != nil {
|
|
t.Fatalf("put %s: %v", k, err)
|
|
}
|
|
}
|
|
|
|
ch, errCh := be.List(ctx, "blocks/aa")
|
|
got := map[string]bool{}
|
|
for k := range ch {
|
|
got[k] = true
|
|
}
|
|
if err := <-errCh; err != nil {
|
|
t.Fatalf("list err: %v", err)
|
|
}
|
|
if !got["blocks/aa/1.zap.age"] || !got["blocks/aa/2.zap.age"] {
|
|
t.Fatalf("missing aa keys: %v", got)
|
|
}
|
|
if got["blocks/bb/3.zap.age"] {
|
|
t.Fatalf("bb key leaked into aa prefix list: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestFileTraversalBlocked(t *testing.T) {
|
|
be, _ := newFileBE(t)
|
|
ctx := context.Background()
|
|
if err := be.Put(ctx, "../../../etc/evil", []byte("x")); err == nil {
|
|
t.Fatal("path traversal should be blocked")
|
|
}
|
|
}
|