mirror of
https://github.com/zap-proto/zip.git
synced 2026-10-19 15:38:57 +00:00
fetch: key the cache on the digest alone
The cache path was Name+"-"+Sum while the comment said the name IS the digest. It was not, and the gap is expensive: one multi-call binary mounted under 108 names downloaded 195MB 108 times and kept 108 identical copies — ~21GB to negate the whole point of shipping one artifact. Keyed on Sum, the first plugin to start warms every other. Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
package zip
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// One artifact serving many plugins must be ONE download and one cached file:
|
||||
// a multi-call binary mounted under 108 names is 195MB, not 108 x 195MB.
|
||||
func TestFetch_SharedArtifactCachedOnceAcrossNames(t *testing.T) {
|
||||
bin := []byte("#!/bin/sh\nexit 0\n")
|
||||
sum := sha256.Sum256(bin)
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
|
||||
var hits int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
atomic.AddInt64(&hits, 1)
|
||||
w.Write(bin)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
dir := t.TempDir()
|
||||
var paths []string
|
||||
for _, name := range []string{"dns", "flags", "iam", "billing"} {
|
||||
p, err := fetch(Plugin{Name: name, URL: srv.URL, Sum: digest, Dir: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
paths = append(paths, p)
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Fatalf("downloaded %d times, want 1 — the digest is the cache key", hits)
|
||||
}
|
||||
for _, p := range paths[1:] {
|
||||
if p != paths[0] {
|
||||
t.Fatalf("same digest cached at two paths: %s vs %s", paths[0], p)
|
||||
}
|
||||
}
|
||||
if ents, _ := os.ReadDir(filepath.Join(dir, "zip-plugins")); len(ents) != 1 {
|
||||
t.Fatalf("%d cache entries for one artifact, want 1", len(ents))
|
||||
}
|
||||
}
|
||||
|
||||
// Different bytes must never collide, whatever they are called.
|
||||
func TestFetch_DifferentDigestsAreDistinct(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var paths []string
|
||||
for _, body := range []string{"#!/bin/sh\nexit 0\n", "#!/bin/sh\nexit 1\n"} {
|
||||
b := []byte(body)
|
||||
sum := sha256.Sum256(b)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Write(b)
|
||||
}))
|
||||
p, err := fetch(Plugin{Name: "same", URL: srv.URL, Sum: hex.EncodeToString(sum[:]), Dir: dir})
|
||||
srv.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths = append(paths, p)
|
||||
}
|
||||
if paths[0] == paths[1] {
|
||||
t.Fatal("two different binaries cached at one path")
|
||||
}
|
||||
}
|
||||
@@ -520,7 +520,11 @@ func fetch(spec Plugin) (string, error) {
|
||||
if err := os.MkdirAll(cache, 0o700); err != nil {
|
||||
return "", fmt.Errorf("plugin cache: %w", err)
|
||||
}
|
||||
final := filepath.Join(cache, spec.Name+"-"+spec.Sum)
|
||||
// Keyed on the digest ALONE, never on the name: one artifact commonly serves
|
||||
// many plugins — a multi-call binary mounted 108 times under 108 names is
|
||||
// one 195MB download, not 108 of the same bytes. The first plugin to start
|
||||
// warms every other.
|
||||
final := filepath.Join(cache, spec.Sum)
|
||||
if _, err := os.Stat(final); err == nil {
|
||||
return final, nil // already verified once; the name IS the digest
|
||||
}
|
||||
|
||||
@@ -719,3 +719,4 @@ func TestUnload_LazyStaysDown(t *testing.T) {
|
||||
t.Fatal("still marked disabled after a successful Reload")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user