mirror of
https://github.com/hanzoai/vfs.git
synced 2026-10-30 18:18:21 +00:00
Phase 0.1.0 — production-shaped scaffold for the S3-backed virtual block filesystem. Stateful services (IAM, BD, ATS, TA, KMS, onyxd, lqd snapshots) get unlimited disk via S3 with PQ-encrypted blocks; the hot tier is a configurable LRU cache. Companion to hanzo/replicate (WAL streaming) — `replicate` is file-level granularity, `vfs` is block-level (4 KiB pages, content-addressable via blake3-256). Included -------- - pkg/backend: pluggable Backend interface + Open(URL) dispatch. - pkg/backend/file: file:// backend (atomic put via temp+rename, path traversal blocked, recursive list with prefix filter). - pkg/backend/s3: AWS SDK v2 S3 backend with `?endpoint=` override for S3-compat (R2, MinIO, Hanzo Storage). NotFound → ErrNotFound. - pkg/vfs/block: blake3-256 hashing → BlockID, 2-hex shard prefix for the backend key, BlockSize=4096, zero-pad to BlockSize before encryption (block boundaries don't leak file lengths). - pkg/vfs/crypto: luxfi/age v1.5.0 wrapper. Recipients (write-side) + Identities (read-side); hybrid X25519 + ML-KEM-768 supported via the same recipient list. - pkg/vfs/cache: in-memory LRU bound by total bytes; touch-promotes on read; eviction on Put when over cap. NVMe write-back tier lands in 0.2.0. - pkg/vfs: top-level handle. PutBlock/GetBlock with cache hit-path + content-hash integrity check on read. Stats() surfaces cache + backend metadata. - pkg/mount: fuse_stub.go (default build) + fuse_unix.go (-tags fuse). Real FUSE wiring lands in 0.2.0; current stub returns a build-tag error pointing at `make build-fuse`. - cmd/vfs: cobra CLI (put/get/stats/mount). Backend registered via blank-import; --age-recipient repeatable; --age-key reads identity PEM from file or VFS_AGE_KEY env. Tests ----- - block: hash determinism, path shape, verify match/mismatch, pad. - cache: LRU eviction, touch-promotion, basic ops. - file backend: put/get/delete/stat/list, path-traversal block. - vfs roundtrip: encrypt → backend → decrypt → byte-equal plaintext, zero-pad correctness; multiple writes of identical plaintext both round-trip; stats reflect insertions. CI / build ---------- - .github/workflows/build.yml: vet + race-tests on Go 1.24, amd64 binary upload, GHCR image build + push on main / v* tags. - Dockerfile: distroless static, CGO_ENABLED=0, ARG VERSION ldflag. - Makefile: build / build-fuse / test / fmt / vet / check / image / install. VERSION-pinned. Roadmap ------- - 0.2.0: bazil.org/fuse mount, NVMe disk cache layer - 0.3.0: gcs + azureblob backends - 0.4.0: K8s sidecar mode + Helm chart - 0.5.0: WASI guest mode - 1.0.0: chunked uploads, multipart resume, GC for orphan blocks, metrics + SLO targets
178 lines
4.0 KiB
Go
178 lines
4.0 KiB
Go
// Package file is a filesystem-backed Backend for local dev. NOT
|
|
// suitable for production — no fsync barrier discipline, no checksum.
|
|
package file
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/hanzoai/vfs/pkg/backend"
|
|
)
|
|
|
|
func init() {
|
|
backend.Register("file", openFile)
|
|
}
|
|
|
|
type fileBackend struct {
|
|
root string
|
|
|
|
closeOnce sync.Once
|
|
closed bool
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func openFile(_ context.Context, u *url.URL) (backend.Backend, error) {
|
|
// file:///abs/path → u.Path = "/abs/path"
|
|
// file://./relpath → u.Host = ".", u.Path = "/relpath"
|
|
root := u.Path
|
|
if u.Host != "" && u.Host != "localhost" {
|
|
root = filepath.Join(u.Host, u.Path)
|
|
}
|
|
if root == "" {
|
|
return nil, fmt.Errorf("file backend: missing path in %q", u.String())
|
|
}
|
|
if err := os.MkdirAll(root, 0o755); err != nil {
|
|
return nil, fmt.Errorf("file backend: mkdir %q: %w", root, err)
|
|
}
|
|
return &fileBackend{root: root}, nil
|
|
}
|
|
|
|
func (f *fileBackend) keyPath(key string) string {
|
|
// Disallow "..". Backend keys are simple `/`-delimited names; never user paths.
|
|
if strings.Contains(key, "..") {
|
|
return ""
|
|
}
|
|
return filepath.Join(f.root, filepath.FromSlash(key))
|
|
}
|
|
|
|
func (f *fileBackend) Get(_ context.Context, key string) ([]byte, error) {
|
|
p := f.keyPath(key)
|
|
if p == "" {
|
|
return nil, fmt.Errorf("file backend: invalid key %q", key)
|
|
}
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil, fmt.Errorf("%w: %s", backend.ErrNotFound, key)
|
|
}
|
|
return nil, err
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func (f *fileBackend) Put(_ context.Context, key string, data []byte) error {
|
|
p := f.keyPath(key)
|
|
if p == "" {
|
|
return fmt.Errorf("file backend: invalid key %q", key)
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
return err
|
|
}
|
|
// Atomic write: temp file + rename.
|
|
tmp, err := os.CreateTemp(filepath.Dir(p), ".vfs-"+filepath.Base(p)+"-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() {
|
|
_ = os.Remove(tmp.Name())
|
|
}()
|
|
if _, err := tmp.Write(data); err != nil {
|
|
_ = tmp.Close()
|
|
return err
|
|
}
|
|
if err := tmp.Sync(); err != nil {
|
|
_ = tmp.Close()
|
|
return err
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp.Name(), p)
|
|
}
|
|
|
|
func (f *fileBackend) Delete(_ context.Context, key string) error {
|
|
p := f.keyPath(key)
|
|
if p == "" {
|
|
return fmt.Errorf("file backend: invalid key %q", key)
|
|
}
|
|
if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fileBackend) List(ctx context.Context, prefix string) (<-chan string, <-chan error) {
|
|
keys := make(chan string, 64)
|
|
errCh := make(chan error, 1)
|
|
go func() {
|
|
defer close(keys)
|
|
defer close(errCh)
|
|
base := f.root
|
|
walkRoot := base
|
|
if prefix != "" {
|
|
walkRoot = filepath.Join(base, filepath.FromSlash(prefix))
|
|
}
|
|
err := filepath.WalkDir(walkRoot, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil // empty prefix
|
|
}
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
return nil
|
|
}
|
|
rel, rerr := filepath.Rel(base, path)
|
|
if rerr != nil {
|
|
return rerr
|
|
}
|
|
rel = filepath.ToSlash(rel)
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case keys <- rel:
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil && !errors.Is(err, context.Canceled) {
|
|
errCh <- err
|
|
}
|
|
}()
|
|
return keys, errCh
|
|
}
|
|
|
|
func (f *fileBackend) Stat(_ context.Context, key string) (int64, error) {
|
|
p := f.keyPath(key)
|
|
if p == "" {
|
|
return 0, fmt.Errorf("file backend: invalid key %q", key)
|
|
}
|
|
st, err := os.Stat(p)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return 0, fmt.Errorf("%w: %s", backend.ErrNotFound, key)
|
|
}
|
|
return 0, err
|
|
}
|
|
return st.Size(), nil
|
|
}
|
|
|
|
func (f *fileBackend) Close() error {
|
|
f.closeOnce.Do(func() { f.closed = true })
|
|
return nil
|
|
}
|
|
|
|
func (f *fileBackend) String() string { return "file://" + f.root }
|
|
|
|
// Compile-time interface check.
|
|
var _ backend.Backend = (*fileBackend)(nil)
|
|
|
|
// Unused but reserved for streaming when we need it.
|
|
type _ = io.Reader
|