Files
zeekay ffa5a24c01 initial: vfs scaffold (block layer + file:// + s3 backends + age PQ crypto)
Phase 0.1.0 — production-shaped scaffold for the S3-backed virtual block
filesystem. Stateful services (IAM, BD, ATS, TA, KMS, onyxd, lqd
snapshots) get unlimited disk via S3 with PQ-encrypted blocks; the
hot tier is a configurable LRU cache. Companion to hanzo/replicate
(WAL streaming) — `replicate` is file-level granularity, `vfs` is
block-level (4 KiB pages, content-addressable via blake3-256).

Included
--------
- pkg/backend: pluggable Backend interface + Open(URL) dispatch.
- pkg/backend/file: file:// backend (atomic put via temp+rename, path
  traversal blocked, recursive list with prefix filter).
- pkg/backend/s3: AWS SDK v2 S3 backend with `?endpoint=` override
  for S3-compat (R2, MinIO, Hanzo Storage). NotFound → ErrNotFound.
- pkg/vfs/block: blake3-256 hashing → BlockID, 2-hex shard prefix
  for the backend key, BlockSize=4096, zero-pad to BlockSize before
  encryption (block boundaries don't leak file lengths).
- pkg/vfs/crypto: luxfi/age v1.5.0 wrapper. Recipients (write-side)
  + Identities (read-side); hybrid X25519 + ML-KEM-768 supported via
  the same recipient list.
- pkg/vfs/cache: in-memory LRU bound by total bytes; touch-promotes
  on read; eviction on Put when over cap. NVMe write-back tier lands
  in 0.2.0.
- pkg/vfs: top-level handle. PutBlock/GetBlock with cache hit-path
  + content-hash integrity check on read. Stats() surfaces cache
  + backend metadata.
- pkg/mount: fuse_stub.go (default build) + fuse_unix.go (-tags fuse).
  Real FUSE wiring lands in 0.2.0; current stub returns a build-tag
  error pointing at `make build-fuse`.
- cmd/vfs: cobra CLI (put/get/stats/mount). Backend registered via
  blank-import; --age-recipient repeatable; --age-key reads identity
  PEM from file or VFS_AGE_KEY env.

Tests
-----
- block: hash determinism, path shape, verify match/mismatch, pad.
- cache: LRU eviction, touch-promotion, basic ops.
- file backend: put/get/delete/stat/list, path-traversal block.
- vfs roundtrip: encrypt → backend → decrypt → byte-equal plaintext,
  zero-pad correctness; multiple writes of identical plaintext both
  round-trip; stats reflect insertions.

CI / build
----------
- .github/workflows/build.yml: vet + race-tests on Go 1.24, amd64
  binary upload, GHCR image build + push on main / v* tags.
- Dockerfile: distroless static, CGO_ENABLED=0, ARG VERSION ldflag.
- Makefile: build / build-fuse / test / fmt / vet / check / image /
  install. VERSION-pinned.

Roadmap
-------
- 0.2.0: bazil.org/fuse mount, NVMe disk cache layer
- 0.3.0: gcs + azureblob backends
- 0.4.0: K8s sidecar mode + Helm chart
- 0.5.0: WASI guest mode
- 1.0.0: chunked uploads, multipart resume, GC for orphan blocks,
  metrics + SLO targets
2026-05-08 10:59:05 -07:00

178 lines
4.0 KiB
Go

// Package file is a filesystem-backed Backend for local dev. NOT
// suitable for production — no fsync barrier discipline, no checksum.
package file
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"github.com/hanzoai/vfs/pkg/backend"
)
func init() {
backend.Register("file", openFile)
}
type fileBackend struct {
root string
closeOnce sync.Once
closed bool
mu sync.Mutex
}
func openFile(_ context.Context, u *url.URL) (backend.Backend, error) {
// file:///abs/path → u.Path = "/abs/path"
// file://./relpath → u.Host = ".", u.Path = "/relpath"
root := u.Path
if u.Host != "" && u.Host != "localhost" {
root = filepath.Join(u.Host, u.Path)
}
if root == "" {
return nil, fmt.Errorf("file backend: missing path in %q", u.String())
}
if err := os.MkdirAll(root, 0o755); err != nil {
return nil, fmt.Errorf("file backend: mkdir %q: %w", root, err)
}
return &fileBackend{root: root}, nil
}
func (f *fileBackend) keyPath(key string) string {
// Disallow "..". Backend keys are simple `/`-delimited names; never user paths.
if strings.Contains(key, "..") {
return ""
}
return filepath.Join(f.root, filepath.FromSlash(key))
}
func (f *fileBackend) Get(_ context.Context, key string) ([]byte, error) {
p := f.keyPath(key)
if p == "" {
return nil, fmt.Errorf("file backend: invalid key %q", key)
}
b, err := os.ReadFile(p)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("%w: %s", backend.ErrNotFound, key)
}
return nil, err
}
return b, nil
}
func (f *fileBackend) Put(_ context.Context, key string, data []byte) error {
p := f.keyPath(key)
if p == "" {
return fmt.Errorf("file backend: invalid key %q", key)
}
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
return err
}
// Atomic write: temp file + rename.
tmp, err := os.CreateTemp(filepath.Dir(p), ".vfs-"+filepath.Base(p)+"-*")
if err != nil {
return err
}
defer func() {
_ = os.Remove(tmp.Name())
}()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), p)
}
func (f *fileBackend) Delete(_ context.Context, key string) error {
p := f.keyPath(key)
if p == "" {
return fmt.Errorf("file backend: invalid key %q", key)
}
if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}
func (f *fileBackend) List(ctx context.Context, prefix string) (<-chan string, <-chan error) {
keys := make(chan string, 64)
errCh := make(chan error, 1)
go func() {
defer close(keys)
defer close(errCh)
base := f.root
walkRoot := base
if prefix != "" {
walkRoot = filepath.Join(base, filepath.FromSlash(prefix))
}
err := filepath.WalkDir(walkRoot, func(path string, d os.DirEntry, err error) error {
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil // empty prefix
}
return err
}
if d.IsDir() {
return nil
}
rel, rerr := filepath.Rel(base, path)
if rerr != nil {
return rerr
}
rel = filepath.ToSlash(rel)
select {
case <-ctx.Done():
return ctx.Err()
case keys <- rel:
}
return nil
})
if err != nil && !errors.Is(err, context.Canceled) {
errCh <- err
}
}()
return keys, errCh
}
func (f *fileBackend) Stat(_ context.Context, key string) (int64, error) {
p := f.keyPath(key)
if p == "" {
return 0, fmt.Errorf("file backend: invalid key %q", key)
}
st, err := os.Stat(p)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return 0, fmt.Errorf("%w: %s", backend.ErrNotFound, key)
}
return 0, err
}
return st.Size(), nil
}
func (f *fileBackend) Close() error {
f.closeOnce.Do(func() { f.closed = true })
return nil
}
func (f *fileBackend) String() string { return "file://" + f.root }
// Compile-time interface check.
var _ backend.Backend = (*fileBackend)(nil)
// Unused but reserved for streaming when we need it.
type _ = io.Reader