mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zeekay
8f5d0b9333
Full internal rename — build-verified, every module green (root + s3-rdma-sidecar
+ telemetry/server + all test modules):
- Dirs: weed/→s3/ (main tree); cmd/weed-{db,sql}→cmd/s3-{db,sql};
seaweed-volume→s3-volume; seaweedfs-rdma-sidecar→s3-rdma-sidecar;
s3/mount/weedfs_*.go→s3fs_*.go (~113 path renames).
- Imports: github.com/hanzoai/s3/weed/ → /s3/ across 1700+ files.
- gRPC proto services Seaweed/SeaweedFiler/SeaweedMessaging/... → Hanzo*
(8 services) via full protoc regeneration (correct descriptors, no sed-corruption).
- Brand SeaweedFS/Seaweed/seaweed → Hanzo/hanzo; internal X-Seaweedfs-* headers
→ X-Hanzo-*; .seaweedfs config dir → .s3.
- CLI is `s3`: go install github.com/hanzoai/s3/s3@latest | make install | Docker.
install.sh rewritten as a Hanzo S3 installer.
- External forks github.com/seaweedfs/{raft,go-fuse,goexif,cockroachdb-parser}
kept verbatim — immutable third-party module paths; NOTICE retained.
Residual (cosmetic, non-build): kafka loadtest test-proto package com.seaweedfs.loadtest.
98 lines
2.5 KiB
Go
98 lines
2.5 KiB
Go
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/hanzoai/s3/s3/glog"
|
|
"github.com/hanzoai/s3/s3/storage"
|
|
"github.com/hanzoai/s3/s3/storage/backend"
|
|
"github.com/hanzoai/s3/s3/storage/needle"
|
|
"github.com/hanzoai/s3/s3/storage/super_block"
|
|
util_http "github.com/hanzoai/s3/s3/util/http"
|
|
)
|
|
|
|
var (
|
|
volumePath = flag.String("dir", "/tmp", "data directory to store files")
|
|
volumeCollection = flag.String("collection", "", "the volume collection name")
|
|
volumeId = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
|
|
)
|
|
|
|
func Checksum(n *needle.Needle) string {
|
|
return fmt.Sprintf("%s%x", n.Id, n.Cookie)
|
|
}
|
|
|
|
type VolumeFileScanner4SeeDat struct {
|
|
version needle.Version
|
|
block super_block.SuperBlock
|
|
|
|
dir string
|
|
hashes map[string]bool
|
|
dat *os.File
|
|
datBackend backend.BackendStorageFile
|
|
}
|
|
|
|
func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock super_block.SuperBlock) error {
|
|
scanner.version = superBlock.Version
|
|
scanner.block = superBlock
|
|
return nil
|
|
|
|
}
|
|
func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
|
|
return true
|
|
}
|
|
|
|
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
|
|
|
|
if scanner.datBackend == nil {
|
|
newFileName := filepath.Join(*volumePath, "dat_fixed")
|
|
newDatFile, err := os.Create(newFileName)
|
|
if err != nil {
|
|
glog.Fatalf("Write New Volume Data %v", err)
|
|
}
|
|
scanner.datBackend = backend.NewDiskFile(newDatFile)
|
|
scanner.datBackend.WriteAt(scanner.block.Bytes(), 0)
|
|
}
|
|
|
|
checksum := Checksum(n)
|
|
|
|
if scanner.hashes[checksum] {
|
|
glog.V(0).Infof("duplicate checksum:%s fid:%d,%s%x @ offset:%d", checksum, *volumeId, n.Id, n.Cookie, offset)
|
|
return nil
|
|
}
|
|
scanner.hashes[checksum] = true
|
|
|
|
_, s, _, e := n.Append(scanner.datBackend, scanner.version)
|
|
fmt.Printf("size %d error %v\n", s, e)
|
|
|
|
return nil
|
|
}
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
util_http.InitGlobalHttpClient()
|
|
|
|
vid := needle.VolumeId(*volumeId)
|
|
|
|
outpath, _ := filepath.Abs(filepath.Dir(os.Args[0]))
|
|
|
|
scanner := &VolumeFileScanner4SeeDat{
|
|
dir: filepath.Join(outpath, "out"),
|
|
hashes: map[string]bool{},
|
|
}
|
|
|
|
if _, err := os.Stat(scanner.dir); err != nil {
|
|
if err := os.MkdirAll(scanner.dir, os.ModePerm); err != nil {
|
|
glog.Fatalf("could not create output dir : %s", err)
|
|
}
|
|
}
|
|
|
|
err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
|
|
if err != nil {
|
|
glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
|
|
}
|
|
|
|
}
|