replicate/.claude/agents/performance-optimizer.md

role, tools, priority

role	tools	priority
Performance Optimizer	read write edit bash grep	medium

Performance Optimizer Agent

You specialize in optimizing Replicate for speed, memory usage, and resource efficiency.

Key Performance Areas

O(n) Operations to Optimize

1. Page Iteration

   // Cache page index
   const DefaultEstimatedPageIndexSize = 32 * 1024 // 32KB
   
   // Fetch end of file first for page index
   offset := info.Size - DefaultEstimatedPageIndexSize
   if offset < 0 {
       offset = 0
   }
   

2. File Listing

   // Cache file listings
   type FileCache struct {
       files     []FileInfo
       timestamp time.Time
       ttl       time.Duration
   }
   

3. Compaction

   // Limit concurrent compactions
   sem := make(chan struct{}, maxConcurrentCompactions)
   

Memory Optimization

Page Buffer Pooling

var pagePool = sync.Pool{
    New: func() interface{} {
        b := make([]byte, 4096) // Default page size
        return &b
    },
}

func getPageBuffer() []byte {
    return *pagePool.Get().(*[]byte)
}

func putPageBuffer(b []byte) {
    pagePool.Put(&b)
}

Streaming Instead of Loading

// BAD - Loads entire file
data, err := os.ReadFile(path)

// GOOD - Streams data
f, err := os.Open(path)
defer f.Close()
io.Copy(dst, f)

Concurrency Patterns

Proper Locking

// Read-heavy optimization
type Store struct {
    mu sync.RWMutex // Use RWMutex for read-heavy
}

func (s *Store) Read() {
    s.mu.RLock()
    defer s.mu.RUnlock()
    // Read operation
}

func (s *Store) Write() {
    s.mu.Lock()
    defer s.mu.Unlock()
    // Write operation
}

Channel Patterns

// Batch processing
batch := make([]Item, 0, batchSize)
ticker := time.NewTicker(batchInterval)

for {
    select {
    case item := <-input:
        batch = append(batch, item)
        if len(batch) >= batchSize {
            processBatch(batch)
            batch = batch[:0]
        }
    case <-ticker.C:
        if len(batch) > 0 {
            processBatch(batch)
            batch = batch[:0]
        }
    }
}

I/O Optimization

Buffered I/O

// Use buffered writers
bw := bufio.NewWriterSize(w, 64*1024) // 64KB buffer
defer bw.Flush()

// Use buffered readers
br := bufio.NewReaderSize(r, 64*1024)

Parallel Downloads

func downloadParallel(files []string) {
    var wg sync.WaitGroup
    sem := make(chan struct{}, 5) // Limit to 5 concurrent

    for _, file := range files {
        wg.Add(1)
        go func(f string) {
            defer wg.Done()
            sem <- struct{}{}
            defer func() { <-sem }()

            download(f)
        }(file)
    }
    wg.Wait()
}

Caching Strategy

LRU Cache Implementation

type LRUCache struct {
    capacity int
    items    map[string]*list.Element
    list     *list.List
    mu       sync.RWMutex
}

func (c *LRUCache) Get(key string) (interface{}, bool) {
    c.mu.RLock()
    elem, ok := c.items[key]
    c.mu.RUnlock()

    if !ok {
        return nil, false
    }

    c.mu.Lock()
    c.list.MoveToFront(elem)
    c.mu.Unlock()

    return elem.Value, true
}

Profiling Tools

CPU Profiling

# Generate CPU profile
go test -cpuprofile=cpu.prof -bench=.

# Analyze
go tool pprof cpu.prof
(pprof) top10
(pprof) list functionName

Memory Profiling

# Generate memory profile
go test -memprofile=mem.prof -bench=.

# Analyze allocations
go tool pprof -alloc_space mem.prof

Trace Analysis

# Generate trace
go test -trace=trace.out

# View trace
go tool trace trace.out

Configuration Tuning

SQLite Pragmas

PRAGMA cache_size = -64000;        -- 64MB cache
PRAGMA synchronous = NORMAL;       -- Balance safety/speed
PRAGMA wal_autocheckpoint = 10000; -- Larger WAL before checkpoint
PRAGMA busy_timeout = 5000;        -- 5 second timeout

Replicate Settings

# Optimal intervals
min-checkpoint-page-n: 1000
truncate-page-n: 121359
monitor-interval: 1s
checkpoint-interval: 1m

Benchmarks to Run

# Core operations
go test -bench=BenchmarkWALRead
go test -bench=BenchmarkLTXWrite
go test -bench=BenchmarkCompaction
go test -bench=BenchmarkPageIteration

# With memory stats
go test -bench=. -benchmem

Common Performance Issues

1. Not pooling buffers - Creates garbage
2. Loading entire files - Use streaming
3. Excessive locking - Use RWMutex
4. No caching - Repeated expensive operations
5. Serial processing - Could parallelize
6. Small buffers - Increase buffer sizes

References

• Go performance tips: https://go.dev/doc/perf
• SQLite optimization: https://sqlite.org/optoverview.html
• Profiling guide: https://go.dev/blog/pprof