A pool's volume and the protocol's come from one answer

Valuing the history fixed the protocol's total and left every pool reporting the
window a pass had just read, so a landing page claimed $206,718 in all-time
volume above a list of pools claiming $4,159. Two sums of the same trades, taken
in different places over different sets, and the reader has no way to tell which
is which.

They are one question sliced two ways. The store groups the swap table by pool
in a single scan — the pool column has been indexed since a pool's own trade
history needed it — and the pass folds that: each pool takes its row, each token
takes the rows of the pools holding it, and the protocol takes the sum. What the
pass valued this time around stays in the log, where it says what the pass did
rather than what the chain has done.

Co-authored-by: Hanzo Dev <[email protected]>

LLM.md

@@ -78,6 +78,11 @@ themselves; the ALTER failing is how a later start knows not to do it again.
 Arriving empty would have zeroed every trade older than the current window — the
 fix erasing the number it exists to correct.
 
+**A pool's volume and the protocol's are one question sliced two ways**, so
+`TradedByPool()` answers both and the total is the fold. Asked separately they
+drifted fifty times apart — the protocol summing every trade ever indexed while
+each pool summed the window a pass had just read.
+
 **Total volume is not the pass's total.** `maxValuedSwaps` bounds what one
 valuation pass values, because the swap table grows forever. Summing that window
 answers for the newest trades only. Summing the day series does not escape it

indexer/supply_test.go

@@ -115,30 +115,44 @@ func TestNativeSupplySurvivesAStaleCopy(t *testing.T) {
 //
 // A pass values a bounded slice of the swap table because that table grows
 // forever. The bound is right for the pass and wrong for the field the wire
-// calls total volume. Summing the day series does not escape it — those rows
-// are folded from the same window, so their sum is that window again. This
-// gives the store more trades than any window would carry and asks it what has
-// traded.
-func TestFactoryVolumeCoversEveryTrade(t *testing.T) {
+// calls total volume. Summing the day series does not escape it — those rows are
+// folded from the same window, so their sum is that window again.
+//
+// A pool's own figure and the protocol's total are the same question sliced two
+// ways, so one grouped answer serves both: asked separately they drifted fifty
+// times apart, a landing page claiming $206,718 over a list of pools claiming
+// $4,159.
+func TestTradedByPoolCoversEveryTrade(t *testing.T) {
 	s := newMemSQLiteStore(t)
 
-	for i, usd := range []string{"1000.00", "2000.00", "500.00"} {
+	for i, tr := range []struct {
+		pool string
+		usd  string
+	}{{"0xaaa", "1000.00"}, {"0xaaa", "2000.00"}, {"0xbbb", "500.00"}} {
 		s.SeedSwap(fmt.Sprintf("0xabc#%d", i), &storage.SeedSwapData{
 			Timestamp: int64(1767225600 + i*86400),
 			Block:     uint64(100 + i),
-			Pool:      "0xpool",
-			AmountUSD: usd,
+			Pool:      tr.pool,
+			AmountUSD: tr.usd,
 		})
 	}
 
-	traded, volume, err := s.Traded()
+	got, err := s.TradedByPool()
 	if err != nil {
-		t.Fatalf("Traded: %v", err)
+		t.Fatalf("TradedByPool: %v", err)
 	}
-	if traded != 3 {
-		t.Errorf("trades = %d, want 3", traded)
+	if got["0xaaa"].VolumeUSD != 3000 || got["0xaaa"].Trades != 2 {
+		t.Errorf("0xaaa = %+v, want 2 trades worth 3000", got["0xaaa"])
 	}
-	if volume != 3500 {
-		t.Errorf("volume = %v, want 3500 \u2014 a trade outside the window still traded", volume)
+	if got["0xbbb"].VolumeUSD != 500 || got["0xbbb"].Trades != 1 {
+		t.Errorf("0xbbb = %+v, want 1 trade worth 500", got["0xbbb"])
+	}
+
+	var total float64
+	for _, v := range got {
+		total += v.VolumeUSD
+	}
+	if total != 3500 {
+		t.Errorf("protocol total = %v, want 3500 — it is the fold of the pools, not a second sum", total)
 	}
 }

indexer/valuation.go

@@ -186,7 +186,7 @@ func (idx *Indexer) revalue(parent context.Context) {
 	poolTVL := map[string]float64{}
 	ratio := map[string]float64{}
 	swapUSD := map[string]string{}
-	poolVol := valueSwaps(trades, vps, tokenVol, swapUSD)
+	windowVol := valueSwaps(trades, vps, map[string]float64{}, swapUSD)
 	valuedRows, valuedErr := idx.store.ValueSwaps(swapUSD)
 	// How far the walk through history got, remembered only once the values are
 	// on disk. Moved before the write, a failed write would leave those trades
@@ -199,7 +199,29 @@ func (idx *Indexer) revalue(parent context.Context) {
 		idx.store.SetPricedThrough(reached)
 	}
 
-	var totalTVL, totalVol float64
+	// What every pool has traded, over the whole table, asked once.
+	//
+	// A pool's own figure and the protocol's total are one question sliced two
+	// ways, so they come from one answer. Asked separately they drifted fifty
+	// times apart: the protocol summed every trade ever indexed while each pool
+	// summed the window a pass had just read, and a landing page claimed $206,718
+	// above a list of pools claiming $4,159.
+	traded, tradedErr := idx.store.TradedByPool()
+	if tradedErr != nil {
+		idx.logf("[valuation] all-time volume unavailable, reporting this window: %v", tradedErr)
+		traded = map[string]storage.Traded{}
+		for id, v := range windowVol {
+			traded[id] = storage.Traded{VolumeUSD: v}
+		}
+	}
+	var allTimeVol float64
+	var allTimeTrades int64
+	for _, t := range traded {
+		allTimeVol += t.VolumeUSD
+		allTimeTrades += t.Trades
+	}
+
+	var totalTVL, windowTotal float64
 	for _, vp := range vps {
 		p := pools[vp.id]
 		if p == nil {
@@ -239,12 +261,17 @@ func (idx *Indexer) revalue(parent context.Context) {
 		p.TotalValueLockedUSD = fmtUSD(tvl)
 		p.Token0Price = fmtRatio(vp.bal0, vp.bal1)
 		p.Token1Price = fmtRatio(vp.bal1, vp.bal0)
-		if v, ok := poolVol[vp.id]; ok {
-			p.VolumeUSD = fmtUSD(v)
-			totalVol += v
+		if t, ok := traded[strings.ToLower(vp.id)]; ok {
+			p.VolumeUSD = fmtUSD(t.VolumeUSD)
+			// A token's volume is what the pools holding it have traded. Both
+			// sides of a trade count toward their own token, which is what the
+			// per-leg fold this replaced did and what the wire format means.
+			tokenVol[vp.t0] += t.VolumeUSD
+			tokenVol[vp.t1] += t.VolumeUSD
 		} else if p.VolumeUSD == "" {
 			p.VolumeUSD = fmtUSD(0)
 		}
+		windowTotal += windowVol[vp.id]
 		idx.store.SeedPool(vp.id, p)
 		totalTVL += tvl
 	}
@@ -296,16 +323,11 @@ func (idx *Indexer) revalue(parent context.Context) {
 	if m, ok := f.(map[string]interface{}); ok {
 		txCount = asInt64(m["txCount"])
 	}
-	traded, volume, err := idx.store.Traded()
-	if err != nil {
-		idx.logf("[valuation] all-time volume unavailable, reporting this window: %v", err)
-		traded, volume = int64(len(trades)), totalVol
-	}
 	idx.store.SeedFactory("1", &storage.SeedFactoryData{
 		PoolCount:           int64(len(pools)),
 		TxCount:             txCount,
 		TotalValueLockedUSD: fmtUSD(totalTVL),
-		TotalVolumeUSD:      fmtUSD(volume),
+		TotalVolumeUSD:      fmtUSD(allTimeVol),
 	})
 
 	// ── The day series: the same trades, grouped by the day they happened ──
@@ -332,8 +354,8 @@ func (idx *Indexer) revalue(parent context.Context) {
 	// sees one alone cannot tell which he has. The first is what this pass just
 	// valued; the second is what the chain has traded and what is served.
 	idx.logf("[valuation] %d/%d pools valued, %d tokens priced, %d swaps — TVL $%s, volume $%s of $%s over %d trades%s (%s)",
-		len(vps), len(pools), len(prices), len(trades), fmtUSD(totalTVL), fmtUSD(totalVol),
-		fmtUSD(volume), traded, priced, time.Since(started).Round(time.Millisecond))
+		len(vps), len(pools), len(prices), len(trades), fmtUSD(totalTVL), fmtUSD(windowTotal),
+		fmtUSD(allTimeVol), allTimeTrades, priced, time.Since(started).Round(time.Millisecond))
 }
 
 // readBalances asks each pool's two tokens what the pool holds. One eth_call

storage/store_memory.go

@@ -479,15 +479,25 @@ func (s *Store) SetPricedThrough(ts int64) {
 	s.pricedThrough = ts
 }
 
-// Traded reports every trade the store holds and what they were worth.
-func (s *Store) Traded() (int64, float64, error) {
+// Traded is a count of trades and what they were worth.
+type Traded struct {
+	Trades    int64
+	VolumeUSD float64
+}
+
+// TradedByPool reports what each pool has traded, over everything stored.
+func (s *Store) TradedByPool() (map[string]Traded, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
-	var volume float64
+	out := map[string]Traded{}
 	for _, sw := range s.swaps {
-		volume += dollars(sw.AmountUSD)
+		k := strings.ToLower(sw.Pool)
+		t := out[k]
+		t.Trades++
+		t.VolumeUSD += dollars(sw.AmountUSD)
+		out[k] = t
 	}
-	return int64(len(s.swaps)), volume, nil
+	return out, nil
 }
 
 // dollars reads a formatted dollar figure back as a number. Anything unreadable

storage/store_sqlite.go

@@ -212,22 +212,42 @@ func (s *Store) SetPricedThrough(ts int64) {
 		strconv.FormatInt(ts, 10))
 }
 
-// Traded reports every trade the store holds and what they were worth.
+// Traded is a count of trades and what they were worth.
+type Traded struct {
+	Trades    int64
+	VolumeUSD float64
+}
+
+// TradedByPool reports what each pool has traded, over the whole table.
 //
-// Over the whole table, not over a window: a valuation pass reads a bounded
-// slice of this table because the table grows forever, and that bound is right
-// for the pass and wrong for a figure the wire calls all-time. Summing a column
-// costs one scan and answers for every trade ever indexed.
-func (s *Store) Traded() (int64, float64, error) {
+// One question sliced one way, and the protocol's own total is the fold of it —
+// asked separately they drifted apart by fifty times: the protocol summed every
+// trade ever indexed while each pool summed the window one pass had just read,
+// so a landing page claimed $206,718 above a list of pools claiming $4,159.
+//
+// Over the whole table, not over a window: a pass values a bounded slice because
+// the table grows forever, and that bound is right for the pass and wrong for a
+// figure the wire calls all-time. Grouping costs one scan and there are as many
+// rows out as there are pools.
+func (s *Store) TradedByPool() (map[string]Traded, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
-	var count int64
-	var volume float64
-	err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(amountUSD), 0) FROM swaps`).Scan(&count, &volume)
+	rows, err := s.db.Query(
+		`SELECT pool, COUNT(*), COALESCE(SUM(amountUSD), 0) FROM swaps GROUP BY pool COLLATE NOCASE`)
 	if err != nil {
-		return 0, 0, fmt.Errorf("storage: total traded: %w", err)
+		return nil, fmt.Errorf("storage: traded by pool: %w", err)
 	}
-	return count, volume, nil
+	defer rows.Close()
+	out := map[string]Traded{}
+	for rows.Next() {
+		var pool string
+		var t Traded
+		if err := rows.Scan(&pool, &t.Trades, &t.VolumeUSD); err != nil {
+			continue
+		}
+		out[strings.ToLower(pool)] = t
+	}
+	return out, rows.Err()
 }
 
 // Close closes the database.

storage/traded_test.go

@@ -51,9 +51,9 @@ func TestTradedSurvivesTheColumnArriving(t *testing.T) {
 		t.Fatalf("init schema: %v", err)
 	}
 
-	traded, volume, err := s.Traded()
+	traded, volume, err := traded3(s)
 	if err != nil {
-		t.Fatalf("Traded: %v", err)
+		t.Fatalf("TradedByPool: %v", err)
 	}
 	if traded != 3 {
 		t.Errorf("trades = %d, want 3", traded)
@@ -71,7 +71,7 @@ func TestTradedSurvivesTheColumnArriving(t *testing.T) {
 	if err := s.Init(context.Background()); err != nil {
 		t.Fatalf("re-init: %v", err)
 	}
-	if _, volume, _ = s.Traded(); volume != 4000 {
+	if _, volume, _ = traded3(s); volume != 4000 {
 		t.Errorf("volume = %v, want 4000 after repricing s1", volume)
 	}
 }
@@ -91,13 +91,10 @@ func TestValueSwapsMovesTheColumn(t *testing.T) {
 	}
 	s.SeedSwap("s1", &SeedSwapData{Timestamp: 100, Pool: "0xpool", AmountUSD: "0"})
 
-	if _, _, err := traded(t, s); err != nil {
-		t.Fatal(err)
-	}
 	if _, err := s.ValueSwaps(map[string]string{"s1": "42.50"}); err != nil {
 		t.Fatalf("ValueSwaps: %v", err)
 	}
-	count, volume, err := traded(t, s)
+	count, volume, err := traded3(s)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -106,9 +103,20 @@ func TestValueSwapsMovesTheColumn(t *testing.T) {
 	}
 }
 
-func traded(t *testing.T, s *Store) (int64, float64, error) {
-	t.Helper()
-	return s.Traded()
+// The protocol's total is the fold of the per-pool answer, which is what the
+// valuation pass does with it.
+func traded3(s *Store) (int64, float64, error) {
+	byPool, err := s.TradedByPool()
+	if err != nil {
+		return 0, 0, err
+	}
+	var n int64
+	var usd float64
+	for _, v := range byPool {
+		n += v.Trades
+		usd += v.VolumeUSD
+	}
+	return n, usd, nil
 }
 
 // A history longer than any window has to be reachable a batch at a time.