lux/indexer
2
0
Fork 0
mirror of https://github.com/luxfi/indexer.git synced 2026-09-24 00:25:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: standalone indexerd, drop dead bundles

Browse Source 
luxfi/indexer is now a single-purpose repo: the standalone indexerd
chain-indexer daemon. The unified Go binary (indexer + graph + embedded
SPA) lives in luxfi/explorer; the standalone graph daemon lives in
luxfi/graph. One way to do everything.

Changes:
- cmd/indexerd/    new entrypoint (replaces broken cmd/indexer/)
- daemon/          public package: Config, RunIndexers, MountIndexerAPI,
                   MountHealth, ListenAndServe — importable by
                   luxfi/explorer for the unified binary
- Dockerfile       builds only /usr/local/bin/indexerd (port 8091)
- Makefile         single target

Removed:
- cmd/explorer/, cmd/explorerd/  — moved to luxfi/explorer
- cmd/indexer/                   — broken (imports nonexistent
                                    {a,b,k,p,q,t,x,z}chain packages)
- cmd/multichain/, multichain/   — outdated multi-chain manager
- cmd/evmchains/                 — outdated EVM chain registry tool
- cmd/graphd/                    — moved to luxfi/graph
- Dockerfile.{indexer,orig,simple} — pre-canonical leftovers

Indexer config (daemon.Config) is intentionally minimal: data_dir,
http_addr, chains[]. Brand, networks, graph, token, and subgraph
customization is the unified-explorer concern (per-chain in chains.yaml,
read by luxfi/explorer). Extra YAML fields are silently ignored here.
This commit is contained in:
zeekay
2026-05-05 20:42:41 -07:00
parent e98acb1e9d
commit 826c3d03da
30 changed files with 604 additions and 12748 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+10 -16
View File
@@ -1,4 +1,5 @@
# luxfi/indexer — Go backend only (API + block indexer)
# luxfi/indexer — standalone chain-indexer daemon.
# The unified explorer + graph live in luxfi/explorer and luxfi/graph.
FROM golang:1.26-alpine AS builder
RUN apk add --no-cache gcc musl-dev sqlite-dev
WORKDIR /src
@@ -6,23 +7,16 @@ COPY go.mod go.sum ./
RUN go mod download
COPY . .
ARG VERSION=dev
# Ensure embedded static/ is non-empty (real Vite build copied in via COPY above).
# If the build context did not include the SPA, fall back to a minimal placeholder
# so go:embed succeeds. Real builds ship the full SPA from cmd/explorer/static/.
RUN test -s cmd/explorer/static/index.html || ( \
mkdir -p cmd/explorer/static && \
echo '<html><body>Explorer placeholder — SPA not built. Run vite build before docker build.</body></html>' > cmd/explorer/static/index.html \
)
RUN CGO_ENABLED=1 CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux go build \
-ldflags="-s -w -X main.version=${VERSION}" -o /explorer ./cmd/explorer/
-ldflags="-s -w -X main.version=${VERSION}" -o /indexerd ./cmd/indexerd/
FROM alpine:3.21
RUN apk add --no-cache ca-certificates sqlite-libs
COPY --from=builder /explorer /usr/local/bin/explorer
RUN adduser -D -u 65532 explorer
USER explorer
COPY --from=builder /indexerd /usr/local/bin/indexerd
RUN adduser -D -u 65532 indexer
USER indexer
VOLUME /data
ENV DATA_DIR=/data HTTP_ADDR=:8090
EXPOSE 8090
HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://localhost:8090/health || exit 1
ENTRYPOINT ["explorer"]
ENV DATA_DIR=/data HTTP_ADDR=:8091
EXPOSE 8091
HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://localhost:8091/health || exit 1
ENTRYPOINT ["indexerd"]
Dockerfile.indexer
-41
View File
@@ -1,41 +0,0 @@
# Build stage - use BUILDPLATFORM to run Go natively on host (avoids QEMU segfaults)
FROM --platform=$BUILDPLATFORM golang:1.26-alpine AS builder
ARG TARGETOS TARGETARCH
RUN apk add --no-cache git ca-certificates
WORKDIR /app
# Copy go mod files first for caching
COPY go.mod go.sum ./
RUN go mod download
# Copy source
COPY . .
# Build with version info - cross-compile via GOOS/GOARCH
ARG VERSION=dev
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -tags postgres -ldflags "-X main.version=${VERSION}" -o /indexer ./cmd/indexer
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -tags postgres -ldflags "-X main.version=${VERSION}" -o /evmchains ./cmd/evmchains
# Runtime stage
FROM --platform=linux/amd64 alpine:3.19
RUN apk add --no-cache ca-certificates tzdata
WORKDIR /app
COPY --from=builder /indexer /usr/local/bin/indexer
COPY --from=builder /evmchains /usr/local/bin/evmchains
COPY configs/ /app/configs/
# Default environment
ENV RPC_ENDPOINT=""
ENV DATABASE_URL=""
ENV HTTP_PORT="4200"
ENV POLL_INTERVAL="30s"
EXPOSE 4000 4100 4200 4300 4400 4500 4600 4700 5000 5100 5200 5300 9000
ENTRYPOINT ["indexer"]
CMD ["--help"]
Dockerfile.orig
-22
View File
@@ -1,22 +0,0 @@
FROM golang:1.26-alpine AS builder
RUN apk add --no-cache gcc musl-dev sqlite-dev
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
ARG VERSION=dev
# -D_LARGEFILE64_SOURCE fixes pread64/pwrite64 on musl
RUN CGO_ENABLED=1 CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux go build \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /indexer ./cmd/indexer/
FROM alpine:3.21
RUN apk add --no-cache ca-certificates sqlite-libs
COPY --from=builder /indexer /usr/local/bin/indexer
RUN adduser -D -u 65532 explorer
USER explorer
VOLUME /data
ENV DATA_DIR=/data HTTP_ADDR=:8090
EXPOSE 8090
HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://localhost:8090/health || exit 1
ENTRYPOINT ["indexer"]
Dockerfile.simple
-22
View File
@@ -1,22 +0,0 @@
FROM golang:1.26-alpine AS builder
RUN apk add --no-cache gcc musl-dev sqlite-dev
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
ARG VERSION=dev
# -D_LARGEFILE64_SOURCE fixes pread64/pwrite64 on musl
RUN CGO_ENABLED=1 CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux go build \
-ldflags="-s -w -X main.version=${VERSION}" \
-o /explorer ./cmd/explorer/
FROM alpine:3.21
RUN apk add --no-cache ca-certificates sqlite-libs
COPY --from=builder /explorer /usr/local/bin/explorer
RUN adduser -D -u 65532 explorer
USER explorer
VOLUME /data
ENV DATA_DIR=/data HTTP_ADDR=:8090
EXPOSE 8090
HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://localhost:8090/health || exit 1
ENTRYPOINT ["explorer"]
Makefile
+6 -45
View File
@@ -1,33 +1,22 @@
.PHONY: build build-indexer test test-coverage clean docker lint
.PHONY: build test test-coverage clean docker lint fmt vet dev deps
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
LDFLAGS := -ldflags "-X main.version=$(VERSION)"
# Build targets
build:
go build $(LDFLAGS) -o bin/explorer ./cmd/explorer
go build $(LDFLAGS) -o bin/indexerd ./cmd/indexerd
build-indexer:
go build $(LDFLAGS) -o bin/indexer ./cmd/indexer
# Test targets
test:
go test -v ./...
test-coverage:
go test -coverprofile=coverage.out ./...
go tool cover -html=coverage.out -o coverage.html
@echo "Coverage report: coverage.html"
test-integration:
go test -v -tags=integration ./test/...
# Docker — single binary with embedded frontend
docker:
docker build -f Dockerfile.explorer -t ghcr.io/luxfi/explorer:$(VERSION) .
docker tag ghcr.io/luxfi/explorer:$(VERSION) ghcr.io/luxfi/explorer:latest
docker build -t ghcr.io/luxfi/indexer:$(VERSION) .
docker tag ghcr.io/luxfi/indexer:$(VERSION) ghcr.io/luxfi/indexer:latest
# Code quality
lint:
golangci-lint run ./...
@@ -37,43 +26,15 @@ fmt:
vet:
go vet ./...
# Development
dev:
go run ./cmd/explorer --rpc=http://localhost:9630/ext/bc/C/rpc --chain-name="C-Chain" --coin=LUX --chain-id=96369
go run ./cmd/indexerd --rpc=http://localhost:9630/ext/bc/C/rpc --chain-name="C-Chain" --coin=LUX --chain-id=96369
dev-config:
go run ./cmd/explorer --config=chains.example.yaml
go run ./cmd/indexerd --config=chains.example.yaml
# Clean
clean:
rm -rf bin/ coverage.out coverage.html
# Dependencies
deps:
go mod download
go mod tidy
# Database migrations (placeholder)
migrate-up:
@echo "Run indexer with --init to create schema"
# Help
help:
@echo "LUX Indexer - Go indexers for LUX Network native chains"
@echo ""
@echo "Build:"
@echo " make build Build indexer binary"
@echo " make docker Build Docker image"
@echo ""
@echo "Test:"
@echo " make test Run unit tests"
@echo " make test-coverage Run tests with coverage"
@echo ""
@echo "Development:"
@echo " make dev-xchain Run X-Chain indexer locally"
@echo " make dev-pchain Run P-Chain indexer locally"
@echo " make dev-zchain Run Z-Chain indexer locally"
@echo ""
@echo "Code quality:"
@echo " make lint Run linter"
@echo " make fmt Format code"
cmd/evmchains/main.go
-288
View File
@@ -1,288 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
// Package main runs multiple EVM chain indexers in a single process.
// Each chain gets its own explorer API server on a dedicated port,
// while all chains share one PostgreSQL database using table prefixes.
package main
import (
"context"
"database/sql"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
_ "github.com/lib/pq"
"gopkg.in/yaml.v3"
"github.com/luxfi/indexer/evm/api"
)
var version = "dev"
// ChainDef defines a single EVM chain to index.
type ChainDef struct {
Slug string `yaml:"slug"`
Name string `yaml:"name"`
ChainID int64 `yaml:"chain_id"`
Port int `yaml:"port"`
RPC string `yaml:"rpc"`
Enabled bool `yaml:"enabled"`
}
// Config is the top-level config for the multi-chain EVM indexer.
type Config struct {
DatabaseURL string `yaml:"database_url"`
Chains []ChainDef `yaml:"chains"`
}
func main() {
var (
configFile = flag.String("config", "", "Path to evmchains.yaml config file")
dbURL = flag.String("db", "", "PostgreSQL connection URL (overrides config)")
showVersion = flag.Bool("version", false, "Show version and exit")
listChains = flag.Bool("list", false, "List configured chains and exit")
)
flag.Parse()
if *showVersion {
fmt.Printf("lux-evmchains %s\n", version)
os.Exit(0)
}
cfg, err := loadConfig(*configFile)
if err != nil {
log.Fatalf("Failed to load config: %v", err)
}
if *listChains {
printChains(cfg)
os.Exit(0)
}
// CLI flag overrides config
if *dbURL != "" {
cfg.DatabaseURL = *dbURL
}
// Env var overrides config if not set by flag
if cfg.DatabaseURL == "" {
cfg.DatabaseURL = os.Getenv("DATABASE_URL")
}
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required (set via --db flag, config file, or env var)")
}
// Count enabled chains
var enabled int
for _, c := range cfg.Chains {
if c.Enabled {
enabled++
}
}
if enabled == 0 {
log.Fatal("No enabled chains in config")
}
// Connect to PostgreSQL (shared across all chains)
db, err := sql.Open("postgres", cfg.DatabaseURL)
if err != nil {
log.Fatalf("Failed to open database: %v", err)
}
defer db.Close()
db.SetMaxOpenConns(enabled * 5)
db.SetMaxIdleConns(enabled * 2)
db.SetConnMaxLifetime(5 * time.Minute)
if err := db.Ping(); err != nil {
log.Fatalf("Failed to connect to database: %v", err)
}
log.Printf("Connected to PostgreSQL (max_conns=%d)", enabled*5)
// Setup context with signal handling
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
sig := <-sigChan
log.Printf("Received signal %v, shutting down...", sig)
cancel()
}()
// Start each enabled chain's API server
var wg sync.WaitGroup
for _, chain := range cfg.Chains {
if !chain.Enabled {
log.Printf("[%s] Skipping (disabled)", chain.Slug)
continue
}
wg.Add(1)
go func(c ChainDef) {
defer wg.Done()
runChain(ctx, db, c)
}(chain)
}
// Start management HTTP server on port 9000
go startManagementServer(ctx, cfg)
log.Printf("Started %d EVM chain indexers", enabled)
// Wait for shutdown
<-ctx.Done()
log.Println("Waiting for all chain servers to stop...")
wg.Wait()
log.Println("All chain servers stopped")
}
// runChain starts an explorer API server for a single chain.
func runChain(ctx context.Context, db *sql.DB, chain ChainDef) {
log.Printf("[%s] Starting API server on port %d (chain_id=%d, rpc=%s)",
chain.Slug, chain.Port, chain.ChainID, chain.RPC)
// Ensure all required tables exist for this chain prefix
if err := api.EnsureTables(db, chain.Slug); err != nil {
log.Printf("[%s] Failed to ensure tables: %v", chain.Slug, err)
return
}
cfg := api.Config{
HTTPPort: chain.Port,
ChainID: chain.ChainID,
ChainName: chain.Name,
ChainSlug: chain.Slug,
RPCEndpoint: chain.RPC,
}
server := api.NewServer(cfg, db)
if err := server.Run(ctx); err != nil && ctx.Err() == nil {
log.Printf("[%s] API server error: %v", chain.Slug, err)
}
log.Printf("[%s] API server stopped", chain.Slug)
}
// loadConfig loads the YAML config, searching default paths if none given.
func loadConfig(path string) (*Config, error) {
if path == "" {
// Try default locations
defaults := []string{
"configs/evmchains.yaml",
"config/evmchains.yaml",
"/etc/lux/indexer/evmchains.yaml",
}
for _, p := range defaults {
if _, err := os.Stat(p); err == nil {
path = p
break
}
}
}
if path == "" {
return defaultConfig(), nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read config: %w", err)
}
// Expand environment variables
expanded := os.ExpandEnv(string(data))
var cfg Config
if err := yaml.Unmarshal([]byte(expanded), &cfg); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
log.Printf("Loaded config from %s (%d chains)", path, len(cfg.Chains))
return &cfg, nil
}
// defaultConfig returns the 5 Lux mainnet EVM chains.
func defaultConfig() *Config {
return &Config{
Chains: []ChainDef{
{Slug: "cchain", Name: "Lux C-Chain", ChainID: 96369, Port: 4000, RPC: "http://luxd-headless.lux-mainnet.svc:9630/ext/bc/C/rpc", Enabled: true},
{Slug: "zoo", Name: "Zoo", ChainID: 200200, Port: 5000, RPC: "http://luxd-headless.lux-mainnet.svc:9630/ext/bc/Zoo/rpc", Enabled: true},
{Slug: "hanzo", Name: "Hanzo", ChainID: 36963, Port: 5100, RPC: "http://luxd-headless.lux-mainnet.svc:9630/ext/bc/Hanzo/rpc", Enabled: true},
{Slug: "spc", Name: "SPC", ChainID: 36911, Port: 5200, RPC: "http://luxd-headless.lux-mainnet.svc:9630/ext/bc/SPC/rpc", Enabled: true},
{Slug: "pars", Name: "Pars", ChainID: 494949, Port: 5300, RPC: "http://luxd-headless.lux-mainnet.svc:9630/ext/bc/Pars/rpc", Enabled: true},
},
}
}
func printChains(cfg *Config) {
fmt.Printf("Configured EVM chains (%d total):\n\n", len(cfg.Chains))
fmt.Printf(" %-10s %-20s %-10s %-6s %s\n", "SLUG", "NAME", "CHAIN_ID", "PORT", "STATUS")
fmt.Printf(" %-10s %-20s %-10s %-6s %s\n", "----", "----", "--------", "----", "------")
for _, c := range cfg.Chains {
status := "enabled"
if !c.Enabled {
status = "disabled"
}
fmt.Printf(" %-10s %-20s %-10d %-6d %s\n", c.Slug, c.Name, c.ChainID, c.Port, status)
}
}
// startManagementServer runs a simple HTTP server for health/stats.
func startManagementServer(ctx context.Context, cfg *Config) {
mux := http.NewServeMux()
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"status": "ok",
"version": version,
"chains": len(cfg.Chains),
})
})
mux.HandleFunc("/chains", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
var chains []map[string]interface{}
for _, c := range cfg.Chains {
chains = append(chains, map[string]interface{}{
"slug": c.Slug,
"name": c.Name,
"chain_id": c.ChainID,
"port": c.Port,
"enabled": c.Enabled,
"rpc": c.RPC,
})
}
json.NewEncoder(w).Encode(chains)
})
server := &http.Server{
Addr: ":9000",
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
}
go func() {
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
server.Shutdown(shutdownCtx)
}()
log.Printf("Management API on :9000 (/health, /chains)")
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Printf("Management server error: %v", err)
}
}
cmd/explorer/frontend.go
-62
View File
@@ -1,62 +0,0 @@
package main
import (
"embed"
"io/fs"
"net/http"
"path/filepath"
"strings"
)
//go:embed static/*
var staticFS embed.FS
// frontendHandler returns an http.Handler that serves the embedded Vite SPA
// static export. Falls back to index.html for client-side routing.
func frontendHandler() http.Handler {
sub, err := fs.Sub(staticFS, "static")
if err != nil {
panic("embedded static/ not found: " + err.Error())
}
fileServer := http.FileServer(http.FS(sub))
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Security headers for all frontend responses.
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=()")
w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
w.Header().Set("Content-Security-Policy",
"default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self' wss: ws:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'")
// Try the exact file first.
// Use the cleaned path to prevent percent-encoded traversal (%2e%2e).
path := r.URL.Path
cleaned := filepath.Clean(path)
if len(path) > 1 {
// Sanitize: reject path traversal attempts (literal or encoded).
if strings.Contains(path, "..") || strings.Contains(cleaned, "..") || cleaned != path {
http.Error(w, "not found", http.StatusNotFound)
return
}
f, err := sub.Open(path[1:]) // strip leading /
if err == nil {
f.Close()
// Cache static assets (hashed filenames) aggressively.
if strings.Contains(path, "/assets/") {
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
}
fileServer.ServeHTTP(w, r)
return
}
}
// SPA fallback: serve index.html for unknown paths.
// No caching for the HTML shell.
w.Header().Set("Cache-Control", "no-cache")
r.URL.Path = "/"
fileServer.ServeHTTP(w, r)
})
}
cmd/explorer/graph_integration_test.go
-307
View File
@@ -1,307 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/luxfi/graph/engine"
graphindexer "github.com/luxfi/graph/indexer"
graphstorage "github.com/luxfi/graph/storage"
)
// TestGraphIntegration boots a graph instance with a fake RPC,
// submits a subgraph query, and asserts the GraphQL response shape.
func TestGraphIntegration(t *testing.T) {
// Fake EVM RPC that returns block 0 and empty logs.
fakeRPC := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
var req struct {
Method string `json:"method"`
}
json.Unmarshal(body, &req)
w.Header().Set("Content-Type", "application/json")
switch req.Method {
case "eth_blockNumber":
fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":"0x0"}`)
case "eth_getLogs":
fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":[]}`)
default:
fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":null}`)
}
}))
defer fakeRPC.Close()
// Temp data dir
dataDir := t.TempDir()
graphDir := filepath.Join(dataDir, "testchain", "graph")
if err := os.MkdirAll(graphDir, 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
// Create graph storage
store, err := graphstorage.New(graphDir)
if err != nil {
t.Fatalf("storage: %v", err)
}
defer store.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := store.Init(ctx); err != nil {
t.Fatalf("storage init: %v", err)
}
// Seed test data
store.SeedToken("0xabc", &graphstorage.SeedTokenData{
Symbol: "TEST", Name: "Test Token", Decimals: 18,
})
store.SeedFactory("uniswap", &graphstorage.SeedFactoryData{
PoolCount: 42, TxCount: 100,
TotalVolumeUSD: "1000000", TotalValueLockedUSD: "500000",
})
// Start indexer (will poll fake RPC)
idx := graphindexer.New(fakeRPC.URL, store)
go idx.Run(ctx)
// Create engine
eng := engine.New(store, &engine.Config{
MaxQueryDepth: 10,
MaxResultSize: 1 << 20,
QueryTimeoutMs: 5000,
})
eng.LoadBuiltin("amm")
// Mount handlers on a test mux
mux := http.NewServeMux()
gi := &graphInstance{
slug: "testchain",
store: store,
indexer: idx,
eng: eng,
cancel: cancel,
stopRep: func() {},
}
mountGraphHandlers(mux, "testchain", gi)
// Start test server
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
srv := &http.Server{Handler: mux}
go srv.Serve(ln)
defer srv.Close()
addr := fmt.Sprintf("http://%s", ln.Addr().String())
// Wait briefly for server readiness
time.Sleep(100 * time.Millisecond)
// --- Test 1: POST GraphQL query ---
t.Run("graphql_query", func(t *testing.T) {
query := `{"query":"{ tokens(first: 10) { id symbol name decimals } }"}`
resp, err := http.Post(addr+"/v1/graph/testchain/", "application/json", strings.NewReader(query))
if err != nil {
t.Fatalf("POST: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("status %d: %s", resp.StatusCode, body)
}
var gqlResp engine.Response
if err := json.NewDecoder(resp.Body).Decode(&gqlResp); err != nil {
t.Fatalf("decode: %v", err)
}
if len(gqlResp.Errors) > 0 {
t.Fatalf("graphql errors: %v", gqlResp.Errors)
}
if gqlResp.Data == nil {
t.Fatal("expected data, got nil")
}
data, ok := gqlResp.Data.(map[string]interface{})
if !ok {
t.Fatalf("data not a map: %T", gqlResp.Data)
}
tokens, ok := data["tokens"]
if !ok {
t.Fatal("missing 'tokens' in response data")
}
tokenList, ok := tokens.([]interface{})
if !ok {
t.Fatalf("tokens not a list: %T", tokens)
}
if len(tokenList) != 1 {
t.Fatalf("expected 1 token, got %d", len(tokenList))
}
tok, ok := tokenList[0].(map[string]interface{})
if !ok {
t.Fatal("token not a map")
}
if tok["symbol"] != "TEST" {
t.Errorf("expected symbol TEST, got %v", tok["symbol"])
}
})
// --- Test 2: GET /schema ---
t.Run("schema_endpoint", func(t *testing.T) {
resp, err := http.Get(addr + "/v1/graph/testchain/schema")
if err != nil {
t.Fatalf("GET schema: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("schema status %d: %s", resp.StatusCode, body)
}
body, _ := io.ReadAll(resp.Body)
if len(body) == 0 {
t.Fatal("schema response empty")
}
// Verify it's valid JSON
var v interface{}
if err := json.Unmarshal(body, &v); err != nil {
t.Fatalf("schema not valid JSON: %v", err)
}
})
// --- Test 3: GET /playground ---
t.Run("playground_endpoint", func(t *testing.T) {
resp, err := http.Get(addr + "/v1/graph/testchain/playground")
if err != nil {
t.Fatalf("GET playground: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatalf("playground status %d", resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
if !strings.Contains(string(body), "graphiql") {
t.Fatal("playground HTML missing graphiql reference")
}
})
// --- Test 4: GET /health ---
t.Run("health_endpoint", func(t *testing.T) {
resp, err := http.Get(addr + "/v1/graph/testchain/health")
if err != nil {
t.Fatalf("GET health: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatalf("health status %d", resp.StatusCode)
}
var health struct {
Status string `json:"status"`
Chain string `json:"chain"`
}
if err := json.NewDecoder(resp.Body).Decode(&health); err != nil {
t.Fatalf("decode health: %v", err)
}
if health.Status != "ok" {
t.Errorf("health status: %s", health.Status)
}
if health.Chain != "testchain" {
t.Errorf("health chain: %s", health.Chain)
}
})
// --- Test 5: Factory query (seeded data) ---
t.Run("factory_query", func(t *testing.T) {
query := `{"query":"{ factory(id: \"uniswap\") { id poolCount totalVolumeUSD } }"}`
resp, err := http.Post(addr+"/v1/graph/testchain/", "application/json", strings.NewReader(query))
if err != nil {
t.Fatalf("POST: %v", err)
}
defer resp.Body.Close()
var gqlResp engine.Response
json.NewDecoder(resp.Body).Decode(&gqlResp)
if len(gqlResp.Errors) > 0 {
t.Fatalf("errors: %v", gqlResp.Errors)
}
data, ok := gqlResp.Data.(map[string]interface{})
if !ok {
t.Fatal("data not a map")
}
factory, ok := data["factory"].(map[string]interface{})
if !ok {
t.Fatalf("factory not a map: %T", data["factory"])
}
if factory["id"] != "uniswap" {
t.Errorf("factory id: %v", factory["id"])
}
})
}
// TestGraphConfigParsing verifies the graph config is correctly parsed from YAML.
func TestGraphConfigParsing(t *testing.T) {
yaml := `
data_dir: /tmp/test
http_addr: :9090
chains:
- slug: testchain
name: Test
chain_id: 1337
type: evm
rpc: http://localhost:8545
coin: ETH
enabled: true
graph:
enabled: true
schema: uniswap-v2
- slug: nograph
name: No Graph
chain_id: 1338
type: evm
rpc: http://localhost:8546
coin: ETH
enabled: true
`
tmpFile := filepath.Join(t.TempDir(), "chains.yaml")
os.WriteFile(tmpFile, []byte(yaml), 0644)
cfg, err := loadConfig(tmpFile)
if err != nil {
t.Fatalf("loadConfig: %v", err)
}
if len(cfg.Chains) != 2 {
t.Fatalf("expected 2 chains, got %d", len(cfg.Chains))
}
// Chain with graph enabled
c := cfg.Chains[0]
if c.Graph == nil {
t.Fatal("expected graph config on testchain")
}
if !c.Graph.Enabled {
t.Error("expected graph.enabled=true")
}
if c.Graph.Schema != "uniswap-v2" {
t.Errorf("expected schema uniswap-v2, got %s", c.Graph.Schema)
}
// Chain without graph
c2 := cfg.Chains[1]
if c2.Graph != nil {
t.Error("expected nil graph config on nograph chain")
}
}
cmd/explorer/main.go
-884
View File
@@ -1,884 +0,0 @@
// Package main provides the unified explorer binary.
//
// Single binary, per-chain SQLite files, zero external dependencies.
// Indexes any number of chains concurrently. Each chain gets isolated storage.
//
// explorer --config=chains.yaml
// explorer --rpc=http://localhost:9650/ext/bc/C/rpc (single chain mode)
//
// /v1/indexer/{chain}/* Per-chain explorer API
// /v1/indexer/* Default chain API
// /* Embedded frontend (go:embed)
// /health Healthcheck (all chains)
//
// Storage layout:
//
// {data_dir}/
// ├── cchain/query/indexer.db ← C-Chain SQLite
// ├── zoo/query/indexer.db ← Zoo chain SQLite
// ├── ethereum/query/indexer.db ← Ethereum SQLite
// ├── solana/query/indexer.db ← Solana SQLite
// └── ... ← one per chain, isolated
package main
import (
"context"
"database/sql"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"regexp"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"github.com/hanzoai/replicate"
"github.com/luxfi/age"
"github.com/luxfi/indexer/evm"
"github.com/luxfi/indexer/explorer"
"github.com/luxfi/indexer/storage"
"github.com/luxfi/graph/engine"
graphindexer "github.com/luxfi/graph/indexer"
graphstorage "github.com/luxfi/graph/storage"
"gopkg.in/yaml.v3"
)
// jsonMarshalImpl — tiny wrapper so we only import encoding/json once.
func jsonMarshalImpl(v any) ([]byte, error) { return json.Marshal(v) }
var version = "dev"
// GraphConfig controls the per-chain graph (subgraph) engine.
type GraphConfig struct {
Enabled bool `yaml:"enabled"` // Start graph indexer + GraphQL for this chain
Schema string `yaml:"schema"` // Built-in schema name (e.g., "uniswap-v2", "amm", "all") or path to .graphql
}
// ChainConfig defines a single chain to index.
type ChainConfig struct {
Slug string `yaml:"slug"` // URL-safe identifier (e.g., "cchain", "ethereum", "solana")
Name string `yaml:"name"` // Display name
ChainID int64 `yaml:"chain_id"` // EVM chain ID (0 for non-EVM)
Type string `yaml:"type"` // evm, dag, linear, solana, bitcoin, cosmos
RPC string `yaml:"rpc"` // RPC endpoint
WS string `yaml:"ws"` // WebSocket endpoint (optional)
CoinSymbol string `yaml:"coin"` // Native coin symbol
Enabled bool `yaml:"enabled"` // Enable indexing
Default bool `yaml:"default"` // Default chain for /v1/indexer/* (no slug prefix)
Graph *GraphConfig `yaml:"graph"` // Graph engine config (nil = disabled)
}
// Brand holds runtime-configurable frontend branding.
// All fields override the built-in defaults baked into the Vite bundle.
type Brand struct {
Name string `yaml:"name"` // VITE_CHAIN_NAME (e.g. "Liquidity Explorer")
Coin string `yaml:"coin"` // VITE_COIN (e.g. "")
IconURL string `yaml:"icon_url"` // VITE_ICON_URL (relative path, default /icon.svg)
LogoURL string `yaml:"logo_url"` // VITE_LOGO_URL (wordmark)
AccentColor string `yaml:"accent_color"` // VITE_ACCENT_COLOR (hex)
IconFile string `yaml:"icon_file"` // On-disk path to SVG served at /icon.svg
LogoFile string `yaml:"logo_file"` // On-disk path to SVG served at /logo.svg
}
// Network is a cross-explorer link (for network switcher).
type Network struct {
Label string `yaml:"label" json:"label"` // Display label
Domain string `yaml:"domain" json:"domain"` // Hostname
ChainID int64 `yaml:"chain_id" json:"chainId"` // Native chain id for display
}
// Config is the top-level multi-chain configuration.
type Config struct {
DataDir string `yaml:"data_dir"`
HTTPAddr string `yaml:"http_addr"`
Brand Brand `yaml:"brand"`
Networks []Network `yaml:"networks"`
Chains []ChainConfig `yaml:"chains"`
}
func main() {
var (
configFile = flag.String("config", "", "Path to chains.yaml")
rpcEndpoint = flag.String("rpc", "", "Single-chain mode: RPC endpoint")
dataDir = flag.String("data", "", "Data directory (default: ~/.explorer/data)")
httpAddr = flag.String("http", ":8090", "HTTP listen address")
chainName = flag.String("chain-name", "", "Single-chain mode: display name")
coinSymbol = flag.String("coin", "ETH", "Single-chain mode: coin symbol")
chainID = flag.Int64("chain-id", 0, "Single-chain mode: chain ID")
showVersion = flag.Bool("version", false, "Show version and exit")
listChains = flag.Bool("list", false, "List chains and exit")
)
flag.Parse()
if *showVersion {
fmt.Printf("explorer %s\n", version)
os.Exit(0)
}
// Defaults from env
if *dataDir == "" {
*dataDir = envOr("DATA_DIR", filepath.Join(homeDir(), ".explorer", "data"))
}
if *httpAddr == "" {
// HTTP_ADDR wins; otherwise honour the K8s-style PORT env (just a number).
if addr := os.Getenv("HTTP_ADDR"); addr != "" {
*httpAddr = addr
} else if p := os.Getenv("PORT"); p != "" {
*httpAddr = ":" + p
} else {
*httpAddr = ":8090"
}
}
// Load config: either YAML multi-chain or CLI single-chain
var cfg Config
if *configFile != "" {
var err error
cfg, err = loadConfig(*configFile)
if err != nil {
log.Fatalf("Failed to load config: %v", err)
}
} else if rpc := func() string {
// RPC source priority: --rpc flag, RPC_ENDPOINT, CHAIN_RPC_URL (LiquidExplorer operator).
if *rpcEndpoint != "" {
return *rpcEndpoint
}
if v := os.Getenv("RPC_ENDPOINT"); v != "" {
return v
}
return os.Getenv("CHAIN_RPC_URL")
}(); rpc != "" {
// Single-chain mode
if *chainName == "" {
*chainName = envOr("CHAIN_NAME", "EVM Chain")
}
if *coinSymbol == "ETH" {
*coinSymbol = envOr("COIN_SYMBOL", "ETH")
}
if *chainID == 0 {
if env := os.Getenv("CHAIN_ID"); env != "" {
fmt.Sscanf(env, "%d", chainID)
}
}
cfg = Config{
Chains: []ChainConfig{{
Slug: "default",
Name: *chainName,
ChainID: *chainID,
Type: "evm",
RPC: rpc,
CoinSymbol: *coinSymbol,
Enabled: true,
Default: true,
}},
}
} else {
// Try default config locations
for _, p := range []string{
filepath.Join(*dataDir, "chains.yaml"),
"chains.yaml",
"/etc/explorer/chains.yaml",
} {
if _, err := os.Stat(p); err == nil {
cfg, _ = loadConfig(p)
log.Printf("Loaded config from %s", p)
break
}
}
if len(cfg.Chains) == 0 {
fmt.Println("Usage:")
fmt.Println(" explorer --rpc=http://localhost:9650/ext/bc/C/rpc (single chain)")
fmt.Println(" explorer --config=chains.yaml (multi-chain)")
fmt.Println()
fmt.Println("Environment: RPC_ENDPOINT, DATA_DIR, CHAIN_NAME, COIN_SYMBOL, CHAIN_ID")
os.Exit(1)
}
}
if cfg.DataDir == "" {
cfg.DataDir = *dataDir
}
if cfg.HTTPAddr == "" {
cfg.HTTPAddr = *httpAddr
}
// Count enabled
var enabled []ChainConfig
for _, c := range cfg.Chains {
if c.Enabled {
enabled = append(enabled, c)
}
}
if *listChains {
for _, c := range cfg.Chains {
status := "disabled"
if c.Enabled {
status = "enabled"
}
fmt.Printf(" %-20s %-10s %-8s %s\n", c.Slug, c.Type, status, c.RPC)
}
os.Exit(0)
}
if len(enabled) == 0 {
log.Fatal("No enabled chains")
}
log.Printf("explorer %s — %d chains", version, len(enabled))
// Setup context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
log.Println("shutdown signal received")
cancel()
}()
// Start all chains concurrently, each with its own SQLite
var wg sync.WaitGroup
for _, chain := range enabled {
chain := chain
wg.Add(1)
go func() {
defer wg.Done()
runChain(ctx, cfg.DataDir, chain)
}()
}
// Start HTTP server with explorer API
mux := http.NewServeMux()
healthHandler := func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Write([]byte(`{"status":"ok"}`))
}
mux.HandleFunc("GET /health", healthHandler)
mux.HandleFunc("GET /healthz", healthHandler) // K8s-style alias for operator probes
// Mount explorer API for each chain — wait for DB and schema before starting server.
// Retries until ctx is done (no fixed timeout) and logs every failure reason
// so misconfigurations (bad RPC, missing dir, schema init failure) are visible.
log.Printf(" http: %s", cfg.HTTPAddr)
var apiReady sync.WaitGroup
for _, c := range enabled {
dbPath := filepath.Join(cfg.DataDir, c.Slug, "query", "indexer.db")
apiReady.Add(1)
go func(chain ChainConfig, path string) {
defer apiReady.Done()
apiPrefix := "/v1/indexer"
if !chain.Default {
apiPrefix = fmt.Sprintf("/v1/indexer/%s", chain.Slug)
}
var lastErr error
deadline := time.Now().Add(5 * time.Minute)
for attempt := 0; ; attempt++ {
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
// Wait for DB file to exist.
if _, err := os.Stat(path); err != nil {
lastErr = fmt.Errorf("db file not yet created: %w", err)
if time.Now().After(deadline) {
log.Printf("[%s] API mount waiting on DB file %s (attempt %d): %v", chain.Slug, path, attempt, lastErr)
deadline = time.Now().Add(5 * time.Minute)
}
continue
}
// Wait for evm_blocks table to exist before mounting — the
// standalone server caches the table name on construction
// and won't see later schema changes. EVM-typed chains use
// evm_blocks; non-EVM chains skip this check.
if chain.Type == "evm" {
db, openErr := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=ro", path))
if openErr == nil {
var c int
err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='evm_blocks'").Scan(&c)
db.Close()
if err != nil || c == 0 {
lastErr = fmt.Errorf("evm_blocks table not yet created (count=%d, err=%v)", c, err)
if time.Now().After(deadline) {
log.Printf("[%s] API mount waiting on evm_blocks table (attempt %d): %v", chain.Slug, attempt, lastErr)
deadline = time.Now().Add(5 * time.Minute)
}
continue
}
}
}
apiSrv, err := explorer.NewStandaloneServer(explorer.Config{
IndexerDBPath: path,
ChainID: chain.ChainID,
ChainName: chain.Name,
CoinSymbol: chain.CoinSymbol,
APIPrefix: apiPrefix,
})
if err != nil {
lastErr = err
if time.Now().After(deadline) {
log.Printf("[%s] API mount failed (attempt %d): %v", chain.Slug, attempt, err)
deadline = time.Now().Add(5 * time.Minute)
}
continue
}
// Mount the configured prefix.
mux.Handle(apiPrefix+"/", apiSrv.Handler())
// Backward-compat: also serve at /v1/explorer/* (legacy prefix
// used by older clients and docs). For non-default chains the
// alias is /v1/explorer/{slug}/*.
legacyPrefix := strings.Replace(apiPrefix, "/v1/indexer", "/v1/explorer", 1)
if legacyPrefix != apiPrefix {
legacySrv, _ := explorer.NewStandaloneServer(explorer.Config{
IndexerDBPath: path,
ChainID: chain.ChainID,
ChainName: chain.Name,
CoinSymbol: chain.CoinSymbol,
APIPrefix: legacyPrefix,
})
if legacySrv != nil {
mux.Handle(legacyPrefix+"/", legacySrv.Handler())
}
}
// Default chain also mounts under its slug so /v1/indexer/cchain/*
// works alongside /v1/indexer/*. Hostname-based routing in the
// ingress can then pin a frontend to a specific chain.
if chain.Default {
slugSrv, _ := explorer.NewStandaloneServer(explorer.Config{
IndexerDBPath: path,
ChainID: chain.ChainID,
ChainName: chain.Name,
CoinSymbol: chain.CoinSymbol,
APIPrefix: fmt.Sprintf("/v1/indexer/%s", chain.Slug),
})
if slugSrv != nil {
mux.Handle(fmt.Sprintf("/v1/indexer/%s/", chain.Slug), slugSrv.Handler())
}
log.Printf("[%s] API mounted at /v1/indexer/* and /v1/indexer/%s/* (default)", chain.Slug, chain.Slug)
} else {
log.Printf("[%s] API mounted at %s/*", chain.Slug, apiPrefix)
}
return
}
}(c, dbPath)
if c.Default {
log.Printf(" %-20s /v1/indexer/* %s", c.Slug, c.RPC)
} else {
log.Printf(" %-20s /v1/indexer/%s/* %s", c.Slug, c.Slug, c.RPC)
}
}
// Start graph engines for chains with graph.enabled: true
var graphInstances []*graphInstance
for _, c := range enabled {
if c.Graph == nil || !c.Graph.Enabled {
continue
}
gi, err := newGraphInstance(ctx, cfg.DataDir, c)
if err != nil {
log.Printf("[%s][graph] failed to start: %v", c.Slug, err)
continue
}
graphInstances = append(graphInstances, gi)
mountGraphHandlers(mux, c.Slug, gi)
log.Printf("[%s][graph] mounted at /v1/graph/%s/* (schema=%s)", c.Slug, c.Slug, c.Graph.Schema)
}
// Wait for API handlers to mount before starting server
apiReady.Wait()
// Serve /envs.js — runtime frontend config. The SPA reads window.ENV
// before falling back to compile-time import.meta.env defaults.
mux.HandleFunc("GET /envs.js", envsHandler(cfg))
// Per-brand logo and icon — read from disk on each request so brands
// can be updated without a rebuild. Falls back to a generic SVG.
mux.HandleFunc("GET /icon.svg", svgHandler(cfg.Brand.IconFile, genericIcon(cfg.Brand.AccentColor)))
mux.HandleFunc("GET /logo.svg", svgHandler(cfg.Brand.LogoFile, genericLogo(cfg.Brand.Name, cfg.Brand.AccentColor)))
// Serve embedded frontend at /
mux.Handle("/", frontendHandler())
srv := &http.Server{
Addr: cfg.HTTPAddr,
Handler: mux,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 120 * time.Second,
MaxHeaderBytes: 1 << 16, // 64KB
}
go func() {
log.Printf("HTTP server listening on %s", cfg.HTTPAddr)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Printf("HTTP server error: %v", err)
}
}()
go func() {
<-ctx.Done()
srv.Shutdown(context.Background())
}()
wg.Wait()
// Drain graph indexers after chain indexers stop
for _, gi := range graphInstances {
gi.Close()
}
log.Println("explorer stopped")
}
// runChain starts indexing a single chain with its own isolated SQLite.
func runChain(ctx context.Context, baseDir string, chain ChainConfig) {
chainDir := filepath.Join(baseDir, chain.Slug)
if err := os.MkdirAll(chainDir, 0755); err != nil {
log.Printf("[%s] failed to create data dir: %v", chain.Slug, err)
return
}
// Create per-chain unified storage (SQLite + ZapDB KV)
store, err := storage.NewUnified(storage.DefaultUnifiedConfig(chainDir))
if err != nil {
log.Printf("[%s] failed to create storage: %v", chain.Slug, err)
return
}
defer store.Close()
// Start per-chain WAL replication to S3
sqliteDB := filepath.Join(chainDir, "query", "indexer.db")
stopReplicate := startReplicate(sqliteDB, chain.Slug)
defer stopReplicate()
// Initialize
if err := store.Init(ctx); err != nil {
log.Printf("[%s] failed to init storage: %v", chain.Slug, err)
return
}
log.Printf("[%s] indexing %s (%s) → %s", chain.Slug, chain.Name, chain.CoinSymbol, sqliteDB)
switch chain.Type {
case "evm":
evmCfg := evm.Config{
ChainName: chain.Name,
RPCEndpoint: chain.RPC,
HTTPPort: 0,
PollInterval: 30 * time.Second,
}
idx, err := evm.NewIndexer(evmCfg, store)
if err != nil {
log.Printf("[%s] failed to create indexer: %v", chain.Slug, err)
return
}
// Initialize the EVM schema *before* the indexing loop runs. The API
// server polls for the schema and won't mount handlers until tables
// exist — without this pre-init, the API for non-default chains gets
// stuck behind a slow first RPC call and the frontend falls through
// to the SPA placeholder.
if err := idx.Init(ctx); err != nil {
log.Printf("[%s] failed to init evm schema: %v", chain.Slug, err)
return
}
log.Printf("[%s] evm schema initialized", chain.Slug)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Printf("[%s] indexer error: %v", chain.Slug, err)
}
case "dag":
// DAG chains use the dag package — import the specific adapter
log.Printf("[%s] DAG indexer running", chain.Slug)
<-ctx.Done()
case "linear":
// Linear chains (P-Chain) use the chain package
log.Printf("[%s] linear indexer running", chain.Slug)
<-ctx.Done()
case "solana", "bitcoin", "cosmos", "move", "near", "tron", "ton", "substrate":
// External chains handled by multichain package
log.Printf("[%s] multichain indexer running", chain.Slug)
<-ctx.Done()
default:
log.Printf("[%s] unknown chain type: %s", chain.Slug, chain.Type)
}
}
// startReplicate streams a chain's SQLite WAL to S3, PQ-encrypted.
// S3 path includes the chain slug for isolation: {prefix}/{slug}/
func startReplicate(dbPath, slug string) func() {
endpoint := os.Getenv("REPLICATE_S3_ENDPOINT")
if endpoint == "" {
return func() {}
}
bucket := envOr("REPLICATE_S3_BUCKET", "replicate")
prefix := os.Getenv("REPLICATE_S3_PATH")
if prefix == "" {
prefix, _ = os.Hostname()
}
// Per-chain isolation in S3
prefix = prefix + "/" + slug
region := envOr("REPLICATE_S3_REGION", "us-central1")
replicaURL := fmt.Sprintf("s3://%s/%s?endpoint=%s&region=%s&force-path-style=true",
url.PathEscape(bucket),
url.PathEscape(prefix),
url.QueryEscape(endpoint),
url.QueryEscape(region),
)
if ak := os.Getenv("REPLICATE_S3_ACCESS_KEY"); ak != "" {
replicaURL += "&access_key=" + url.QueryEscape(ak)
}
if sk := os.Getenv("REPLICATE_S3_SECRET_KEY"); sk != "" {
replicaURL += "&secret_key=" + url.QueryEscape(sk)
}
client, err := replicate.NewReplicaClientFromURL(replicaURL)
if err != nil {
log.Printf("[%s][replicate] invalid config: %v", slug, err)
return func() {}
}
db := replicate.NewDB(dbPath)
replica := replicate.NewReplicaWithClient(db, client)
if v := os.Getenv("REPLICATE_SYNC_INTERVAL"); v != "" {
if d, err := time.ParseDuration(v); err == nil {
replica.SyncInterval = d
}
}
if recipientStr := os.Getenv("REPLICATE_AGE_RECIPIENT"); recipientStr != "" {
if rcs, err := age.ParseRecipients(strings.NewReader(recipientStr)); err == nil {
replica.AgeRecipients = rcs
}
}
if identityStr := os.Getenv("REPLICATE_AGE_IDENTITY"); identityStr != "" {
if ids, err := age.ParseIdentities(strings.NewReader(identityStr)); err == nil {
replica.AgeIdentities = ids
}
}
db.Replica = replica
if err := db.Open(); err != nil {
log.Printf("[%s][replicate] failed: %v", slug, err)
return func() {}
}
log.Printf("[%s][replicate] streaming → s3://%s/%s (pq=%v)",
slug, bucket, prefix, len(replica.AgeRecipients) > 0)
return func() { _ = db.Close(context.Background()) }
}
// graphInstance holds the per-chain graph engine, indexer, and storage.
type graphInstance struct {
slug string
store *graphstorage.Store
indexer *graphindexer.Indexer
eng *engine.Engine
cancel context.CancelFunc
stopRep func()
}
// newGraphInstance creates and starts a graph indexer + engine for one chain.
// Storage is isolated at {dataDir}/{slug}/graph/subgraph.db.
func newGraphInstance(parent context.Context, dataDir string, chain ChainConfig) (*graphInstance, error) {
graphDir := filepath.Join(dataDir, chain.Slug, "graph")
if err := os.MkdirAll(graphDir, 0755); err != nil {
return nil, fmt.Errorf("mkdir %s: %w", graphDir, err)
}
store, err := graphstorage.New(graphDir)
if err != nil {
return nil, fmt.Errorf("graph storage: %w", err)
}
ctx, cancel := context.WithCancel(parent)
if err := store.Init(ctx); err != nil {
cancel()
store.Close()
return nil, fmt.Errorf("graph storage init: %w", err)
}
stopRep := graphstorage.StartReplicate(filepath.Join(graphDir, "graph.db"))
idx := graphindexer.New(chain.RPC, store)
go func() {
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Printf("[%s][graph] indexer error: %v", chain.Slug, err)
}
}()
eng := engine.New(store, &engine.Config{
MaxQueryDepth: 10,
MaxResultSize: 1 << 20, // 1MB
QueryTimeoutMs: 30000,
})
schema := chain.Graph.Schema
if schema == "" {
schema = "amm"
}
if err := eng.LoadBuiltin(schema); err != nil {
log.Printf("[%s][graph] schema %q not built-in, using amm", chain.Slug, schema)
eng.LoadBuiltin("amm")
}
return &graphInstance{
slug: chain.Slug,
store: store,
indexer: idx,
eng: eng,
cancel: cancel,
stopRep: stopRep,
}, nil
}
// Close drains the graph indexer and closes storage.
func (gi *graphInstance) Close() {
gi.cancel()
gi.stopRep()
gi.store.Close()
log.Printf("[%s][graph] stopped", gi.slug)
}
// mountGraphHandlers registers per-chain GraphQL routes on the mux.
//
// POST /v1/graph/{slug}/ — GraphQL query
// GET /v1/graph/{slug}/schema — introspection schema
// GET /v1/graph/{slug}/playground — GraphiQL IDE
// GET /v1/graph/{slug}/health — graph indexer status
func mountGraphHandlers(mux *http.ServeMux, slug string, gi *graphInstance) {
prefix := fmt.Sprintf("/v1/graph/%s", slug)
// GraphQL POST endpoint with body size limit
mux.HandleFunc(fmt.Sprintf("POST %s/", prefix), func(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, 10<<20) // 10MB
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
w.Header().Set("X-Content-Type-Options", "nosniff")
if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent)
return
}
gi.eng.HandleGraphQL(w, r)
})
// Schema introspection — returns the list of registered resolvers
mux.HandleFunc(fmt.Sprintf("GET %s/schema", prefix), func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Content-Type-Options", "nosniff")
// Execute the __schema introspection query
resp := gi.eng.Execute(r.Context(), &engine.Request{
Query: `{ __schema { types { name } } }`,
})
// If introspection is not supported, return resolver list
if len(resp.Errors) > 0 {
w.Write([]byte(`{"schema":"graph","status":"ok","note":"introspection query not supported; use POST for queries"}`))
return
}
json.NewEncoder(w).Encode(resp)
})
// GraphiQL playground — HTML UI pointing at the chain's GraphQL endpoint
mux.HandleFunc(fmt.Sprintf("GET %s/playground", prefix), func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.Header().Set("X-Content-Type-Options", "nosniff")
fmt.Fprintf(w, graphiQLTemplate, prefix+"/")
})
// Per-chain graph health
mux.HandleFunc(fmt.Sprintf("GET %s/health", prefix), func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Content-Type-Options", "nosniff")
status := gi.indexer.Status()
fmt.Fprintf(w, `{"status":"ok","chain":%q,"block":%d,"indexed":%d}`,
gi.slug, status.LatestBlock, status.IndexedEvents)
})
}
// graphiQLTemplate is the GraphiQL HTML with a %s placeholder for the POST endpoint.
const graphiQLTemplate = `<!DOCTYPE html>
<html><head><title>Graph Explorer</title>
<link rel="stylesheet" href="https://unpkg.com/graphiql@3/graphiql.min.css"/>
</head><body style="margin:0;overflow:hidden">
<div id="graphiql" style="height:100vh"></div>
<script src="https://unpkg.com/react@18/umd/react.production.min.js" crossorigin></script>
<script src="https://unpkg.com/react-dom@18/umd/react-dom.production.min.js" crossorigin></script>
<script src="https://unpkg.com/graphiql@3/graphiql.min.js" crossorigin></script>
<script>
ReactDOM.createRoot(document.getElementById('graphiql')).render(
React.createElement(GraphiQL, {
fetcher: GraphiQL.createFetcher({url: '%s'}),
defaultEditorToolsVisibility: true,
})
);
</script></body></html>`
func loadConfig(path string) (Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return Config{}, err
}
var cfg Config
if err := yaml.Unmarshal(data, &cfg); err != nil {
return Config{}, err
}
// Expand env vars only on RPC/WS fields that legitimately need it.
for i := range cfg.Chains {
cfg.Chains[i].RPC = expandKnownEnv(cfg.Chains[i].RPC)
cfg.Chains[i].WS = expandKnownEnv(cfg.Chains[i].WS)
if err := validateSlug(cfg.Chains[i].Slug); err != nil {
return Config{}, fmt.Errorf("chain[%d]: %w", i, err)
}
}
return cfg, nil
}
// slugPattern is the set of characters allowed in a chain slug. Slugs
// feed directly into filesystem paths (per-chain SQLite dir) and URL
// route prefixes (/v1/graph/{slug}/…). Restricting them to
// lowercase-alphanumeric + hyphen keeps both surfaces safe from path
// traversal and URL-encoding surprises.
var slugPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`)
func validateSlug(slug string) error {
if !slugPattern.MatchString(slug) {
return fmt.Errorf("invalid slug %q: must match ^[a-z0-9][a-z0-9-]{0,63}$", slug)
}
return nil
}
// expandKnownEnv expands $VAR references in a string using os.Getenv,
// but only for the value itself (no shell injection via config fields).
func expandKnownEnv(s string) string {
return os.Expand(s, os.Getenv)
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
// svgHandler returns a handler that serves the configured SVG file, or the
// given fallback SVG if the file is empty or unreadable. This keeps the
// single-binary model for many brands while letting each brand ship a real
// logo on disk.
func svgHandler(path, fallback string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "image/svg+xml; charset=utf-8")
w.Header().Set("Cache-Control", "public, max-age=300")
w.Header().Set("X-Content-Type-Options", "nosniff")
if path != "" {
if data, err := os.ReadFile(path); err == nil && len(data) > 0 {
_, _ = w.Write(data)
return
}
}
_, _ = io.WriteString(w, fallback)
}
}
// genericIcon returns a minimal monochrome square icon with the accent color.
// Used when the brand didn't supply an icon_file.
func genericIcon(accent string) string {
if accent == "" {
accent = "#ffffff"
}
return fmt.Sprintf(`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none">`+
`<rect x="2" y="2" width="20" height="20" rx="4" fill="%s"/>`+
`<rect x="6" y="6" width="5" height="12" fill="#0a0a0a"/>`+
`<rect x="13" y="6" width="5" height="5" fill="#0a0a0a"/>`+
`</svg>`, accent)
}
// genericLogo returns a text wordmark for brands that didn't ship a logo SVG.
func genericLogo(name, accent string) string {
if name == "" {
name = "Explorer"
}
if accent == "" {
accent = "#ffffff"
}
return fmt.Sprintf(`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 240 40">`+
`<text x="0" y="28" font-family="Inter, system-ui, sans-serif" font-size="24" font-weight="700" fill="%s">%s</text>`+
`</svg>`, accent, name)
}
// envsHandler serves /envs.js — runtime config that overrides compile-time
// VITE_* defaults. Keeps one binary, many brands.
func envsHandler(cfg Config) http.HandlerFunc {
// Build exposed chain list (label + slug only — RPC stays server-side).
type chainExpose struct {
Label string `json:"label"`
Slug string `json:"slug"`
}
chains := make([]chainExpose, 0, len(cfg.Chains))
for _, c := range cfg.Chains {
if !c.Enabled {
continue
}
chains = append(chains, chainExpose{Label: c.Name, Slug: c.Slug})
}
chainsJSON, _ := jsonMarshal(chains)
netsJSON, _ := jsonMarshal(cfg.Networks)
// Build the JS once; it never changes at runtime.
body := fmt.Sprintf(
"window.ENV = Object.assign(window.ENV||{}, {"+
"VITE_CHAIN_NAME:%q,"+
"VITE_COIN:%q,"+
"VITE_ICON_URL:%q,"+
"VITE_LOGO_URL:%q,"+
"VITE_ACCENT_COLOR:%q,"+
"VITE_CHAINS:%q,"+
"VITE_NETWORKS:%q"+
"});\n",
cfg.Brand.Name,
cfg.Brand.Coin,
cfg.Brand.IconURL,
cfg.Brand.LogoURL,
cfg.Brand.AccentColor,
string(chainsJSON),
string(netsJSON),
)
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
_, _ = w.Write([]byte(body))
}
}
// jsonMarshal is a tiny local helper so we don't pull encoding/json into the top imports twice.
func jsonMarshal(v any) ([]byte, error) {
return jsonMarshalImpl(v)
}
func homeDir() string {
h, _ := os.UserHomeDir()
return h
}
cmd/explorer/static/assets/index-B95-SidW.js
-74
View File
File diff suppressed because one or more lines are too long
cmd/explorer/static/index.html
-19
View File
@@ -1,19 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Block Explorer</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
html, body, #root { height: 100%; background: #0a0a0a; color: #e5e5e5; }
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; }
</style>
<script type="module" crossorigin src="/assets/index-B95-SidW.js"></script>
</head>
<body>
<div id="root"></div>
<!-- Runtime config — sets window.ENV before the bundle loads. -->
<script src="/envs.js"></script>
</body>
</html>
cmd/indexer/main.go
-503
View File
@@ -1,503 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
// Package main provides the CLI for running LUX chain indexers.
package main
import (
"context"
"flag"
"fmt"
"log"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
"github.com/hanzoai/replicate"
"github.com/luxfi/age"
"github.com/luxfi/indexer/achain"
"github.com/luxfi/indexer/bchain"
"github.com/luxfi/indexer/chain"
"github.com/luxfi/indexer/dag"
"github.com/luxfi/indexer/evm"
"github.com/luxfi/indexer/kchain"
"github.com/luxfi/indexer/pchain"
"github.com/luxfi/indexer/qchain"
"github.com/luxfi/indexer/storage"
"github.com/luxfi/indexer/tchain"
"github.com/luxfi/indexer/xchain"
"github.com/luxfi/indexer/zchain"
)
var version = "dev"
func main() {
var (
chainType = flag.String("chain", "", "Chain to index: xchain, achain, bchain, qchain, tchain, zchain (DAG), pchain (linear), or cchain (EVM)")
rpcEndpoint = flag.String("rpc", "", "RPC endpoint (e.g., http://localhost:9630/ext/bc/X)")
dataDir = flag.String("data", "", "Data directory for storage (default: ~/.lux/indexer/<chain>)")
httpPort = flag.Int("port", 0, "HTTP server port (default: chain-specific)")
pollInterval = flag.Duration("poll", 30*time.Second, "Poll interval for stats updates")
showVersion = flag.Bool("version", false, "Show version and exit")
listChains = flag.Bool("list", false, "List available chains and exit")
)
flag.Parse()
if *showVersion {
fmt.Printf("lux-indexer %s\n", version)
os.Exit(0)
}
if *listChains {
printChainList()
os.Exit(0)
}
if *chainType == "" {
flag.Usage()
fmt.Println("\nAvailable chains:")
printChainList()
os.Exit(1)
}
// Set defaults based on chain type, checking env vars first
if *rpcEndpoint == "" {
if env := os.Getenv("RPC_ENDPOINT"); env != "" {
*rpcEndpoint = env
} else {
*rpcEndpoint = defaultRPC(*chainType)
}
}
if *dataDir == "" {
if env := os.Getenv("DATA_DIR"); env != "" {
*dataDir = env
} else {
home, _ := os.UserHomeDir()
*dataDir = filepath.Join(home, ".lux", "indexer", *chainType)
}
}
if *httpPort == 0 {
if env := os.Getenv("HTTP_PORT"); env != "" {
if p, err := fmt.Sscanf(env, "%d", httpPort); err == nil && p == 1 {
// port set from env
} else {
*httpPort = defaultPort(*chainType)
}
} else {
*httpPort = defaultPort(*chainType)
}
}
// Create data directory if needed
if err := os.MkdirAll(*dataDir, 0755); err != nil {
log.Fatalf("Failed to create data directory: %v", err)
}
// Create unified storage
store, err := storage.NewUnified(storage.DefaultUnifiedConfig(*dataDir))
if err != nil {
log.Fatalf("Failed to create storage: %v", err)
}
defer store.Close()
// Start SQLite WAL replication to S3 (no-op if REPLICATE_S3_ENDPOINT is unset).
// PQ-encrypted via luxfi/age when REPLICATE_AGE_RECIPIENT is set.
stopReplicate := startReplicate(filepath.Join(*dataDir, "query", "indexer.db"))
defer stopReplicate()
// Initialize storage
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := store.Init(ctx); err != nil {
log.Fatalf("Failed to initialize storage: %v", err)
}
// Handle shutdown signals
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
log.Println("Shutdown signal received")
cancel()
}()
switch *chainType {
// DAG chains
case "xchain":
runXChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
case "achain":
runAChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
case "bchain":
runBChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
case "qchain":
runQChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
case "tchain":
runTChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
case "zchain":
runZChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
case "kchain":
runKChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
// Linear chains
case "pchain":
runPChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
// EVM chain
case "cchain":
runCChain(ctx, store, *rpcEndpoint, *httpPort, *pollInterval)
default:
log.Fatalf("Unknown chain type: %s", *chainType)
}
}
func runXChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := xchain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainX,
ChainName: "X-Chain (Exchange)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "xvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[xchain] Failed to create indexer: %v", err)
}
log.Printf("[xchain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[xchain] Indexer error: %v", err)
}
log.Println("[xchain] Indexer stopped")
}
func runAChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := achain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainA,
ChainName: "A-Chain (AI)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "xvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[achain] Failed to create indexer: %v", err)
}
log.Printf("[achain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[achain] Indexer error: %v", err)
}
log.Println("[achain] Indexer stopped")
}
func runBChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := bchain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainB,
ChainName: "B-Chain (Bridge)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "bvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[bchain] Failed to create indexer: %v", err)
}
log.Printf("[bchain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[bchain] Indexer error: %v", err)
}
log.Println("[bchain] Indexer stopped")
}
func runQChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := qchain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainQ,
ChainName: "Q-Chain (Quantum)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "qvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[qchain] Failed to create indexer: %v", err)
}
log.Printf("[qchain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[qchain] Indexer error: %v", err)
}
log.Println("[qchain] Indexer stopped")
}
func runTChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := tchain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainT,
ChainName: "T-Chain (Teleport)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "tvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[tchain] Failed to create indexer: %v", err)
}
log.Printf("[tchain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[tchain] Indexer error: %v", err)
}
log.Println("[tchain] Indexer stopped")
}
func runZChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := zchain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainZ,
ChainName: "Z-Chain (Privacy)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "zvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[zchain] Failed to create indexer: %v", err)
}
log.Printf("[zchain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[zchain] Indexer error: %v", err)
}
log.Println("[zchain] Indexer stopped")
}
func runPChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := pchain.New(rpcEndpoint)
cfg := chain.Config{
ChainType: chain.ChainP,
ChainName: "P-Chain (Platform)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "pvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := chain.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[pchain] Failed to create indexer: %v", err)
}
log.Printf("[pchain] Starting linear chain indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[pchain] Indexer error: %v", err)
}
log.Println("[pchain] Indexer stopped")
}
func runCChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
cfg := evm.Config{
ChainName: "C-Chain (Contract)",
RPCEndpoint: rpcEndpoint,
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := evm.NewIndexer(cfg, store)
if err != nil {
log.Fatalf("[cchain] Failed to create indexer: %v", err)
}
log.Printf("[cchain] Starting EVM indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[cchain] Indexer error: %v", err)
}
log.Println("[cchain] Indexer stopped")
}
func runKChain(ctx context.Context, store storage.Store, rpcEndpoint string, httpPort int, pollInterval time.Duration) {
adapter := kchain.New(rpcEndpoint)
cfg := dag.Config{
ChainType: dag.ChainK,
ChainName: "K-Chain (Key Management)",
RPCEndpoint: rpcEndpoint,
RPCMethod: "kvm",
HTTPPort: httpPort,
PollInterval: pollInterval,
}
idx, err := dag.New(cfg, store, adapter)
if err != nil {
log.Fatalf("[kchain] Failed to create indexer: %v", err)
}
log.Printf("[kchain] Starting DAG indexer on port %d", httpPort)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Fatalf("[kchain] Indexer error: %v", err)
}
log.Println("[kchain] Indexer stopped")
}
func printChainList() {
fmt.Println(" DAG-based chains (fast consensus):")
fmt.Println(" xchain - X-Chain (Exchange) - Port 4200 - Asset exchange, UTXOs")
fmt.Println(" achain - A-Chain (AI) - Port 4500 - AI compute, attestations")
fmt.Println(" bchain - B-Chain (Bridge) - Port 4600 - Cross-chain bridge")
fmt.Println(" qchain - Q-Chain (Quantum) - Port 4300 - Quantum finality proofs")
fmt.Println(" tchain - T-Chain (Teleport) - Port 4700 - MPC threshold signatures")
fmt.Println(" zchain - Z-Chain (Privacy) - Port 4400 - ZK transactions")
fmt.Println(" kchain - K-Chain (KMS) - Port 4900 - Post-quantum key material")
fmt.Println()
fmt.Println(" Linear chains (strict ordering):")
fmt.Println(" pchain - P-Chain (Platform) - Port 4100 - Validators, staking")
fmt.Println()
fmt.Println(" EVM chain:")
fmt.Println(" cchain - C-Chain (Contract) - Port 4000 - Smart contracts, DeFi")
}
func defaultPort(ct string) int {
ports := map[string]int{
"cchain": 4000,
"pchain": 4100,
"xchain": 4200,
"qchain": 4300,
"zchain": 4400,
"achain": 4500,
"bchain": 4600,
"tchain": 4700,
"kchain": 4900,
}
return ports[ct]
}
func defaultRPC(ct string) string {
endpoints := map[string]string{
"pchain": "http://localhost:9650/ext/bc/P",
"xchain": "http://localhost:9650/ext/bc/X",
"cchain": "http://localhost:9650/ext/bc/C/rpc",
"qchain": "http://localhost:9650/ext/bc/Q",
"zchain": "http://localhost:9650/ext/bc/Z",
"achain": "http://localhost:9650/ext/bc/A",
"bchain": "http://localhost:9650/ext/bc/B",
"tchain": "http://localhost:9650/ext/bc/T",
"kchain": "http://localhost:9650/ext/bc/K",
}
return endpoints[ct]
}
// startReplicate streams SQLite WAL changes to S3, PQ-encrypted via luxfi/age.
// Returns a no-op stop function when REPLICATE_S3_ENDPOINT is not set.
//
// Env vars:
//
// REPLICATE_S3_ENDPOINT — S3 endpoint (required, no-op if empty)
// REPLICATE_S3_BUCKET — bucket name (default: "replicate")
// REPLICATE_S3_PATH — key prefix (default: hostname)
// REPLICATE_S3_REGION — S3 region (default: "us-central1")
// REPLICATE_S3_ACCESS_KEY — S3 access key
// REPLICATE_S3_SECRET_KEY — S3 secret key
// REPLICATE_AGE_RECIPIENT — age public key for PQ encryption
// REPLICATE_AGE_IDENTITY — age private key for restore/decrypt
// REPLICATE_SYNC_INTERVAL — WAL sync interval (default: "1s")
func startReplicate(dbPath string) func() {
endpoint := os.Getenv("REPLICATE_S3_ENDPOINT")
if endpoint == "" {
return func() {}
}
bucket := envOr("REPLICATE_S3_BUCKET", "replicate")
prefix := os.Getenv("REPLICATE_S3_PATH")
if prefix == "" {
prefix, _ = os.Hostname()
}
region := envOr("REPLICATE_S3_REGION", "us-central1")
// Propagate to AWS SDK env vars.
if ak := os.Getenv("REPLICATE_S3_ACCESS_KEY"); ak != "" {
os.Setenv("AWS_ACCESS_KEY_ID", ak)
}
if sk := os.Getenv("REPLICATE_S3_SECRET_KEY"); sk != "" {
os.Setenv("AWS_SECRET_ACCESS_KEY", sk)
}
replicaURL := fmt.Sprintf("s3://%s/%s?endpoint=%s&region=%s&force-path-style=true",
url.PathEscape(bucket),
url.PathEscape(prefix),
url.QueryEscape(endpoint),
url.QueryEscape(region),
)
client, err := replicate.NewReplicaClientFromURL(replicaURL)
if err != nil {
log.Printf("[replicate] invalid S3 config: %v", err)
return func() {}
}
db := replicate.NewDB(dbPath)
replica := replicate.NewReplicaWithClient(db, client)
// Sync interval.
if v := os.Getenv("REPLICATE_SYNC_INTERVAL"); v != "" {
if d, err := time.ParseDuration(v); err == nil {
replica.SyncInterval = d
}
}
// PQ encryption via luxfi/age.
if recipientStr := os.Getenv("REPLICATE_AGE_RECIPIENT"); recipientStr != "" {
rcs, err := age.ParseRecipients(strings.NewReader(recipientStr))
if err != nil {
log.Printf("[replicate] invalid AGE_RECIPIENT, encryption disabled: %v", err)
} else {
replica.AgeRecipients = rcs
}
}
if identityStr := os.Getenv("REPLICATE_AGE_IDENTITY"); identityStr != "" {
ids, err := age.ParseIdentities(strings.NewReader(identityStr))
if err != nil {
log.Printf("[replicate] invalid AGE_IDENTITY, decryption disabled: %v", err)
} else {
replica.AgeIdentities = ids
}
}
db.Replica = replica
if err := db.Open(); err != nil {
log.Printf("[replicate] failed to open: %v", err)
return func() {}
}
log.Printf("[replicate] streaming db=%s s3=%s/%s encrypted=%v",
dbPath, bucket, prefix, len(replica.AgeRecipients) > 0)
return func() {
_ = db.Close(context.Background())
}
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
cmd/indexerd/main.go
+141
View File
@@ -0,0 +1,141 @@
// Package main provides the standalone chain-indexer daemon.
//
// Indexes one or more chains (EVM and others) into per-chain SQLite stores
// and serves only the /v1/indexer/* HTTP API. No graph engine, no embedded
// frontend.
//
// indexerd --config=chains.yaml
// indexerd --rpc=http://localhost:9650/ext/bc/C/rpc (single chain mode)
package main
import (
"context"
"flag"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"syscall"
"github.com/luxfi/indexer/daemon"
)
var version = "dev"
func main() {
var (
configFile = flag.String("config", "", "Path to chains.yaml")
rpcEndpoint = flag.String("rpc", "", "Single-chain mode: RPC endpoint")
dataDir = flag.String("data", "", "Data directory (default: ~/.explorer/data)")
httpAddr = flag.String("http", ":8091", "HTTP listen address")
chainName = flag.String("chain-name", "", "Single-chain mode: display name")
coinSymbol = flag.String("coin", "ETH", "Single-chain mode: coin symbol")
chainID = flag.Int64("chain-id", 0, "Single-chain mode: chain ID")
showVersion = flag.Bool("version", false, "Show version and exit")
)
flag.Parse()
if *showVersion {
fmt.Printf("indexerd %s\n", version)
os.Exit(0)
}
if *dataDir == "" {
*dataDir = daemon.EnvOr("DATA_DIR", filepath.Join(daemon.HomeDir(), ".explorer", "data"))
}
if *httpAddr == ":8091" {
if addr := os.Getenv("HTTP_ADDR"); addr != "" {
*httpAddr = addr
} else if p := os.Getenv("PORT"); p != "" {
*httpAddr = ":" + p
}
}
cfg := resolveConfig(*configFile, *rpcEndpoint, *chainName, *coinSymbol, *chainID, *dataDir)
if cfg.DataDir == "" {
cfg.DataDir = *dataDir
}
if cfg.HTTPAddr == "" {
cfg.HTTPAddr = *httpAddr
}
enabled := cfg.EnabledChains()
if len(enabled) == 0 {
log.Fatal("No enabled chains")
}
log.Printf("indexerd %s — %d chains", version, len(enabled))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
cancel()
}()
wg := daemon.RunIndexers(ctx, cfg)
mux := http.NewServeMux()
daemon.MountHealth(mux)
daemon.MountIndexerAPI(ctx, cfg, mux)
go daemon.ListenAndServe(ctx, cfg.HTTPAddr, mux)
wg.Wait()
log.Println("indexerd stopped")
}
func resolveConfig(configFile, rpcFlag, chainName, coinSymbol string, chainID int64, dataDir string) daemon.Config {
if configFile != "" {
cfg, err := daemon.LoadConfig(configFile)
if err != nil {
log.Fatalf("Failed to load config: %v", err)
}
return cfg
}
rpc := rpcFlag
if rpc == "" {
rpc = os.Getenv("RPC_ENDPOINT")
}
if rpc != "" {
if chainName == "" {
chainName = daemon.EnvOr("CHAIN_NAME", "EVM Chain")
}
if chainID == 0 {
if env := os.Getenv("CHAIN_ID"); env != "" {
fmt.Sscanf(env, "%d", &chainID)
}
}
return daemon.Config{
Chains: []daemon.ChainConfig{{
Slug: "default",
Name: chainName,
ChainID: chainID,
Type: "evm",
RPC: rpc,
CoinSymbol: coinSymbol,
Enabled: true,
Default: true,
}},
}
}
if path := daemon.FindConfig(dataDir); path != "" {
cfg, err := daemon.LoadConfig(path)
if err != nil {
log.Fatalf("Failed to load config %s: %v", path, err)
}
return cfg
}
fmt.Println("Usage:")
fmt.Println(" indexerd --rpc=http://localhost:9650/ext/bc/C/rpc (single chain)")
fmt.Println(" indexerd --config=chains.yaml (multi-chain)")
os.Exit(1)
return daemon.Config{}
}
cmd/multichain/main.go
-286
View File
@@ -1,286 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
// Package main provides the CLI for the multi-chain parallel indexer.
// This indexer can index thousands of chains concurrently using Go routines.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/luxfi/indexer/multichain"
)
var version = "dev"
func main() {
var (
configFile = flag.String("config", "", "Path to chains.yaml config file")
dataDir = flag.String("data", "", "Data directory (default: ~/.lux/indexer/multichain)")
httpPort = flag.Int("port", 5000, "HTTP API port for stats and management")
maxChains = flag.Int("max-chains", 100, "Maximum concurrent chains to index")
showVersion = flag.Bool("version", false, "Show version and exit")
listChains = flag.Bool("list", false, "List configured chains and exit")
dbURL = flag.String("db", "", "PostgreSQL connection URL")
redisURL = flag.String("redis", "", "Redis URL for caching")
)
flag.Parse()
if *showVersion {
fmt.Printf("lux-multichain-indexer %s\n", version)
os.Exit(0)
}
// Set defaults
if *dataDir == "" {
if env := os.Getenv("DATA_DIR"); env != "" {
*dataDir = env
} else {
home, _ := os.UserHomeDir()
*dataDir = filepath.Join(home, ".lux", "indexer", "multichain")
}
}
// Create data directory
if err := os.MkdirAll(*dataDir, 0755); err != nil {
log.Fatalf("Failed to create data directory: %v", err)
}
// Load configuration
var config *multichain.Config
var err error
if *configFile != "" {
config, err = multichain.LoadChainsConfig(*configFile)
if err != nil {
log.Fatalf("Failed to load config: %v", err)
}
} else {
// Try default locations
defaultPaths := []string{
filepath.Join(*dataDir, "chains.yaml"),
"config/chains.yaml",
"/etc/lux/indexer/chains.yaml",
}
for _, p := range defaultPaths {
if _, err := os.Stat(p); err == nil {
config, err = multichain.LoadChainsConfig(p)
if err != nil {
log.Printf("Warning: Failed to load config from %s: %v", p, err)
continue
}
log.Printf("Loaded config from %s", p)
break
}
}
if config == nil {
log.Println("No config file found, using default configuration")
config = multichain.DefaultConfig()
}
}
// Override config with CLI flags
config.MaxConcurrentChains = *maxChains
if *dbURL != "" {
config.DatabaseURL = *dbURL
}
if *redisURL != "" {
config.RedisURL = *redisURL
}
if *listChains {
printChainList(config)
os.Exit(0)
}
// Create manager
manager, err := multichain.NewManager(config)
if err != nil {
log.Fatalf("Failed to create manager: %v", err)
}
// Setup context and signal handling
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
log.Println("Shutdown signal received, stopping indexers...")
cancel()
}()
// Start HTTP server for stats
go startHTTPServer(*httpPort, manager)
// Start indexing
log.Printf("Starting multi-chain indexer with %d chains", len(config.Chains))
log.Printf("HTTP API available at http://localhost:%d", *httpPort)
if err := manager.Start(); err != nil {
log.Fatalf("Failed to start manager: %v", err)
}
// Wait for shutdown
<-ctx.Done()
// Graceful shutdown
log.Println("Stopping all indexers...")
if err := manager.Stop(); err != nil {
log.Printf("Error during shutdown: %v", err)
}
log.Println("Multi-chain indexer stopped")
}
func printChainList(config *multichain.Config) {
fmt.Printf("Configured chains (%d total):\n\n", len(config.Chains))
byType := make(map[multichain.ChainType][]multichain.ChainConfig)
for _, chain := range config.Chains {
byType[chain.Type] = append(byType[chain.Type], chain)
}
typeNames := map[multichain.ChainType]string{
multichain.ChainTypeEVM: "EVM Chains",
multichain.ChainTypeSolana: "Solana",
multichain.ChainTypeBitcoin: "Bitcoin",
multichain.ChainTypeCosmos: "Cosmos SDK",
multichain.ChainTypeMove: "Move (Aptos/Sui)",
multichain.ChainTypeNear: "NEAR",
multichain.ChainTypeTron: "Tron",
multichain.ChainTypeTon: "TON",
multichain.ChainTypeSubstrate: "Substrate (Polkadot)",
// Purpose-built native chains
multichain.ChainTypePlatform: "P-Chain (Platform)",
multichain.ChainTypeUTXO: "X-Chain (UTXO)",
multichain.ChainTypeAI: "A-Chain (AI)",
multichain.ChainTypeBridge: "B-Chain (Bridge)",
multichain.ChainTypeDEX: "D-Chain (DEX)",
multichain.ChainTypeGraph: "G-Chain (Graph)",
multichain.ChainTypeIdentity: "I-Chain (Identity)",
multichain.ChainTypeKey: "K-Chain (Key)",
multichain.ChainTypeMPC: "M-Chain (MPC)",
multichain.ChainTypeOracle: "O-Chain (Oracle)",
multichain.ChainTypeQuantum: "Q-Chain (Quantum)",
multichain.ChainTypeRelay: "R-Chain (Relay)",
multichain.ChainTypeThreshold: "T-Chain (Threshold)",
multichain.ChainTypeZK: "Z-Chain (ZK)",
}
for chainType, name := range typeNames {
chains := byType[chainType]
if len(chains) == 0 {
continue
}
fmt.Printf("%s:\n", name)
for _, c := range chains {
status := "enabled"
if !c.Enabled {
status = "disabled"
}
fmt.Printf(" %-20s ChainID: %-8d %s\n", c.ID, c.ChainID, status)
}
fmt.Println()
}
}
func startHTTPServer(port int, manager *multichain.Manager) {
mux := http.NewServeMux()
// Stats endpoint
mux.HandleFunc("/stats", func(w http.ResponseWriter, r *http.Request) {
stats := manager.Stats()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(stats)
})
// Health endpoint
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
stats := manager.Stats()
health := map[string]interface{}{
"status": "ok",
"activeChains": stats.ActiveChains,
"totalChains": stats.TotalChains,
"uptime": stats.Uptime.String(),
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(health)
})
// Chain stats endpoint
mux.HandleFunc("/chains", func(w http.ResponseWriter, r *http.Request) {
stats := manager.Stats()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(stats.ChainStats)
})
// Individual chain endpoint
mux.HandleFunc("/chain/", func(w http.ResponseWriter, r *http.Request) {
chainID := r.URL.Path[len("/chain/"):]
indexer, ok := manager.GetIndexer(chainID)
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(indexer.Stats())
})
// Metrics endpoint (Prometheus format)
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
stats := manager.Stats()
w.Header().Set("Content-Type", "text/plain")
fmt.Fprintf(w, "# HELP indexer_total_chains Total number of configured chains\n")
fmt.Fprintf(w, "# TYPE indexer_total_chains gauge\n")
fmt.Fprintf(w, "indexer_total_chains %d\n", stats.TotalChains)
fmt.Fprintf(w, "# HELP indexer_active_chains Number of actively indexing chains\n")
fmt.Fprintf(w, "# TYPE indexer_active_chains gauge\n")
fmt.Fprintf(w, "indexer_active_chains %d\n", stats.ActiveChains)
fmt.Fprintf(w, "# HELP indexer_total_blocks_indexed Total blocks indexed across all chains\n")
fmt.Fprintf(w, "# TYPE indexer_total_blocks_indexed counter\n")
fmt.Fprintf(w, "indexer_total_blocks_indexed %d\n", stats.TotalBlocksIndexed)
fmt.Fprintf(w, "# HELP indexer_total_txs_indexed Total transactions indexed\n")
fmt.Fprintf(w, "# TYPE indexer_total_txs_indexed counter\n")
fmt.Fprintf(w, "indexer_total_txs_indexed %d\n", stats.TotalTxsIndexed)
fmt.Fprintf(w, "# HELP indexer_uptime_seconds Indexer uptime in seconds\n")
fmt.Fprintf(w, "# TYPE indexer_uptime_seconds gauge\n")
fmt.Fprintf(w, "indexer_uptime_seconds %.0f\n", stats.Uptime.Seconds())
// Per-chain metrics
for chainID, chainStats := range stats.ChainStats {
fmt.Fprintf(w, "# HELP indexer_chain_blocks_behind{chain=\"%s\"} Blocks behind head\n", chainID)
fmt.Fprintf(w, "indexer_chain_blocks_behind{chain=\"%s\"} %d\n", chainID, chainStats.BlocksBehind)
fmt.Fprintf(w, "indexer_chain_blocks_processed{chain=\"%s\"} %d\n", chainID, chainStats.BlocksProcessed)
fmt.Fprintf(w, "indexer_chain_errors{chain=\"%s\"} %d\n", chainID, chainStats.ErrorCount)
}
})
server := &http.Server{
Addr: fmt.Sprintf(":%d", port),
Handler: mux,
ReadTimeout: 10 * time.Second,
WriteTimeout: 30 * time.Second,
}
log.Printf("Starting HTTP server on port %d", port)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Printf("HTTP server error: %v", err)
}
}
daemon/config.go
+110
View File
@@ -0,0 +1,110 @@
// Package daemon is the chain-indexer bootstrap: it owns the per-chain
// indexers and the /v1/indexer/* HTTP API. The standalone indexerd binary
// (cmd/indexerd) is the only consumer in this repo. The unified
// luxfi/explorer binary imports it directly to compose indexer + graph +
// frontend into a single Go binary.
package daemon
import (
"fmt"
"os"
"path/filepath"
"regexp"
"gopkg.in/yaml.v3"
)
// ChainConfig defines a single chain to index.
type ChainConfig struct {
Slug string `yaml:"slug"` // URL-safe identifier (e.g., "cchain")
Name string `yaml:"name"` // Display name
ChainID int64 `yaml:"chain_id"` // EVM chain ID (0 for non-EVM)
Type string `yaml:"type"` // evm, dag, linear, solana, bitcoin, cosmos
RPC string `yaml:"rpc"` // RPC endpoint
WS string `yaml:"ws"` // WebSocket endpoint (optional)
CoinSymbol string `yaml:"coin"` // Native coin symbol
Enabled bool `yaml:"enabled"` // Enable indexing
Default bool `yaml:"default"` // Default chain for /v1/indexer/* (no slug prefix)
}
// Config is the top-level chain-indexer configuration. Extra YAML fields
// (frontend brand, graph schemas, etc.) are accepted and ignored — the
// unified luxfi/explorer binary owns those concerns.
type Config struct {
DataDir string `yaml:"data_dir"`
HTTPAddr string `yaml:"http_addr"`
Chains []ChainConfig `yaml:"chains"`
}
// EnabledChains returns only the chains with Enabled=true.
func (c Config) EnabledChains() []ChainConfig {
out := make([]ChainConfig, 0, len(c.Chains))
for _, ch := range c.Chains {
if ch.Enabled {
out = append(out, ch)
}
}
return out
}
// LoadConfig reads, parses and validates a YAML config from disk.
func LoadConfig(path string) (Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return Config{}, err
}
var cfg Config
if err := yaml.Unmarshal(data, &cfg); err != nil {
return Config{}, err
}
for i := range cfg.Chains {
cfg.Chains[i].RPC = os.Expand(cfg.Chains[i].RPC, os.Getenv)
cfg.Chains[i].WS = os.Expand(cfg.Chains[i].WS, os.Getenv)
if err := validateSlug(cfg.Chains[i].Slug); err != nil {
return Config{}, fmt.Errorf("chain[%d]: %w", i, err)
}
}
return cfg, nil
}
// FindConfig returns the first existing path among a default search list,
// or "" if none are found. Callers may pass an explicit path to skip the search.
func FindConfig(dataDir string) string {
for _, p := range []string{
filepath.Join(dataDir, "chains.yaml"),
"chains.yaml",
"/etc/explorer/chains.yaml",
} {
if _, err := os.Stat(p); err == nil {
return p
}
}
return ""
}
// slugPattern is the set of characters allowed in a chain slug. Slugs feed
// directly into filesystem paths (per-chain SQLite dir) and URL route prefixes
// (/v1/graph/{slug}/…). Restricting them to lowercase-alphanumeric + hyphen
// keeps both surfaces safe from path traversal and URL-encoding surprises.
var slugPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`)
func validateSlug(slug string) error {
if !slugPattern.MatchString(slug) {
return fmt.Errorf("invalid slug %q: must match ^[a-z0-9][a-z0-9-]{0,63}$", slug)
}
return nil
}
// EnvOr returns the value of an env var, or fallback if unset.
func EnvOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
// HomeDir returns the current user's home directory, or "" if unknown.
func HomeDir() string {
h, _ := os.UserHomeDir()
return h
}
daemon/indexer.go
+295
View File
@@ -0,0 +1,295 @@
package daemon
import (
"context"
"database/sql"
"fmt"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/hanzoai/replicate"
"github.com/luxfi/age"
"github.com/luxfi/indexer/evm"
"github.com/luxfi/indexer/explorer"
"github.com/luxfi/indexer/storage"
)
// RunIndexers spawns one chain-indexer goroutine per enabled chain in cfg.
// Each chain owns an isolated SQLite + ZapDB store rooted at
// {DataDir}/{slug}/. Returns a WaitGroup the caller can wait on for shutdown.
//
// SQLite WAL is replicated to S3 when REPLICATE_S3_ENDPOINT is set.
func RunIndexers(ctx context.Context, cfg Config) *sync.WaitGroup {
var wg sync.WaitGroup
for _, chain := range cfg.EnabledChains() {
chain := chain
wg.Add(1)
go func() {
defer wg.Done()
runChain(ctx, cfg.DataDir, chain)
}()
}
return &wg
}
// runChain starts indexing a single chain with its own isolated SQLite.
func runChain(ctx context.Context, baseDir string, chain ChainConfig) {
chainDir := filepath.Join(baseDir, chain.Slug)
if err := os.MkdirAll(chainDir, 0755); err != nil {
log.Printf("[%s] failed to create data dir: %v", chain.Slug, err)
return
}
store, err := storage.NewUnified(storage.DefaultUnifiedConfig(chainDir))
if err != nil {
log.Printf("[%s] failed to create storage: %v", chain.Slug, err)
return
}
defer store.Close()
sqliteDB := filepath.Join(chainDir, "query", "indexer.db")
stopReplicate := startReplicate(sqliteDB, chain.Slug)
defer stopReplicate()
if err := store.Init(ctx); err != nil {
log.Printf("[%s] failed to init storage: %v", chain.Slug, err)
return
}
log.Printf("[%s] indexing %s (%s) → %s", chain.Slug, chain.Name, chain.CoinSymbol, sqliteDB)
switch chain.Type {
case "evm":
evmCfg := evm.Config{
ChainName: chain.Name,
RPCEndpoint: chain.RPC,
HTTPPort: 0,
PollInterval: 30 * time.Second,
}
idx, err := evm.NewIndexer(evmCfg, store)
if err != nil {
log.Printf("[%s] failed to create indexer: %v", chain.Slug, err)
return
}
// Initialize the EVM schema *before* the indexing loop runs. The API
// server polls for the schema and won't mount handlers until tables
// exist — without this pre-init, the API for non-default chains gets
// stuck behind a slow first RPC call and the frontend falls through
// to the SPA placeholder.
if err := idx.Init(ctx); err != nil {
log.Printf("[%s] failed to init evm schema: %v", chain.Slug, err)
return
}
log.Printf("[%s] evm schema initialized", chain.Slug)
if err := idx.Run(ctx); err != nil && ctx.Err() == nil {
log.Printf("[%s] indexer error: %v", chain.Slug, err)
}
case "dag", "linear", "solana", "bitcoin", "cosmos", "move", "near", "tron", "ton", "substrate":
log.Printf("[%s] %s indexer running", chain.Slug, chain.Type)
<-ctx.Done()
default:
log.Printf("[%s] unknown chain type: %s", chain.Slug, chain.Type)
}
}
// startReplicate streams a chain's SQLite WAL to S3, PQ-encrypted via luxfi/age.
// S3 path includes the chain slug for isolation: {prefix}/{slug}/.
// Returns a no-op stop function when REPLICATE_S3_ENDPOINT is not set.
func startReplicate(dbPath, slug string) func() {
endpoint := os.Getenv("REPLICATE_S3_ENDPOINT")
if endpoint == "" {
return func() {}
}
bucket := EnvOr("REPLICATE_S3_BUCKET", "replicate")
prefix := os.Getenv("REPLICATE_S3_PATH")
if prefix == "" {
prefix, _ = os.Hostname()
}
prefix = prefix + "/" + slug
region := EnvOr("REPLICATE_S3_REGION", "us-central1")
replicaURL := fmt.Sprintf("s3://%s/%s?endpoint=%s&region=%s&force-path-style=true",
url.PathEscape(bucket),
url.PathEscape(prefix),
url.QueryEscape(endpoint),
url.QueryEscape(region),
)
if ak := os.Getenv("REPLICATE_S3_ACCESS_KEY"); ak != "" {
replicaURL += "&access_key=" + url.QueryEscape(ak)
}
if sk := os.Getenv("REPLICATE_S3_SECRET_KEY"); sk != "" {
replicaURL += "&secret_key=" + url.QueryEscape(sk)
}
client, err := replicate.NewReplicaClientFromURL(replicaURL)
if err != nil {
log.Printf("[%s][replicate] invalid config: %v", slug, err)
return func() {}
}
db := replicate.NewDB(dbPath)
replica := replicate.NewReplicaWithClient(db, client)
if v := os.Getenv("REPLICATE_SYNC_INTERVAL"); v != "" {
if d, err := time.ParseDuration(v); err == nil {
replica.SyncInterval = d
}
}
if recipientStr := os.Getenv("REPLICATE_AGE_RECIPIENT"); recipientStr != "" {
if rcs, err := age.ParseRecipients(strings.NewReader(recipientStr)); err == nil {
replica.AgeRecipients = rcs
}
}
if identityStr := os.Getenv("REPLICATE_AGE_IDENTITY"); identityStr != "" {
if ids, err := age.ParseIdentities(strings.NewReader(identityStr)); err == nil {
replica.AgeIdentities = ids
}
}
db.Replica = replica
if err := db.Open(); err != nil {
log.Printf("[%s][replicate] failed: %v", slug, err)
return func() {}
}
log.Printf("[%s][replicate] streaming → s3://%s/%s (pq=%v)",
slug, bucket, prefix, len(replica.AgeRecipients) > 0)
return func() { _ = db.Close(context.Background()) }
}
// MountIndexerAPI mounts /v1/indexer/* HTTP handlers for every enabled chain.
// It blocks until every chain's per-chain SQLite is ready (DB file exists and,
// for EVM chains, the evm_blocks table is created). Concurrent — each chain
// is mounted in its own goroutine. Returns when all are mounted or ctx is done.
//
// Routes:
// - /v1/indexer/* (default chain)
// - /v1/indexer/{slug}/* (every chain, including the default chain alias)
// - /v1/explorer/{slug}/* (legacy alias for non-default chains)
func MountIndexerAPI(ctx context.Context, cfg Config, mux *http.ServeMux) {
var apiReady sync.WaitGroup
for _, c := range cfg.EnabledChains() {
c := c
dbPath := filepath.Join(cfg.DataDir, c.Slug, "query", "indexer.db")
apiReady.Add(1)
go func() {
defer apiReady.Done()
mountChainAPI(ctx, mux, c, dbPath)
}()
if c.Default {
log.Printf(" %-20s /v1/indexer/* %s", c.Slug, c.RPC)
} else {
log.Printf(" %-20s /v1/indexer/%s/* %s", c.Slug, c.Slug, c.RPC)
}
}
apiReady.Wait()
}
// mountChainAPI waits for the chain's DB + EVM schema to be ready, then mounts
// every variant of the explorer API handler the unified deploy expects.
func mountChainAPI(ctx context.Context, mux *http.ServeMux, chain ChainConfig, path string) {
apiPrefix := "/v1/indexer"
if !chain.Default {
apiPrefix = fmt.Sprintf("/v1/indexer/%s", chain.Slug)
}
var lastErr error
deadline := time.Now().Add(5 * time.Minute)
for attempt := 0; ; attempt++ {
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
if _, err := os.Stat(path); err != nil {
lastErr = fmt.Errorf("db file not yet created: %w", err)
if time.Now().After(deadline) {
log.Printf("[%s] API mount waiting on DB file %s (attempt %d): %v", chain.Slug, path, attempt, lastErr)
deadline = time.Now().Add(5 * time.Minute)
}
continue
}
// EVM-typed chains use evm_blocks; non-EVM chains skip this check.
// The standalone server caches table names on construction, so we wait
// for the schema to settle before instantiating it.
if chain.Type == "evm" {
db, openErr := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=ro", path))
if openErr == nil {
var c int
err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='evm_blocks'").Scan(&c)
db.Close()
if err != nil || c == 0 {
lastErr = fmt.Errorf("evm_blocks table not yet created (count=%d, err=%v)", c, err)
if time.Now().After(deadline) {
log.Printf("[%s] API mount waiting on evm_blocks (attempt %d): %v", chain.Slug, attempt, lastErr)
deadline = time.Now().Add(5 * time.Minute)
}
continue
}
}
}
apiSrv, err := explorer.NewStandaloneServer(explorer.Config{
IndexerDBPath: path,
ChainID: chain.ChainID,
ChainName: chain.Name,
CoinSymbol: chain.CoinSymbol,
APIPrefix: apiPrefix,
})
if err != nil {
lastErr = err
if time.Now().After(deadline) {
log.Printf("[%s] API mount failed (attempt %d): %v", chain.Slug, attempt, err)
deadline = time.Now().Add(5 * time.Minute)
}
continue
}
mux.Handle(apiPrefix+"/", apiSrv.Handler())
// Legacy /v1/explorer/{slug}/* alias for older clients.
legacyPrefix := strings.Replace(apiPrefix, "/v1/indexer", "/v1/explorer", 1)
if legacyPrefix != apiPrefix {
legacySrv, _ := explorer.NewStandaloneServer(explorer.Config{
IndexerDBPath: path,
ChainID: chain.ChainID,
ChainName: chain.Name,
CoinSymbol: chain.CoinSymbol,
APIPrefix: legacyPrefix,
})
if legacySrv != nil {
mux.Handle(legacyPrefix+"/", legacySrv.Handler())
}
}
// Default chain also mounts under its slug so /v1/indexer/cchain/*
// works alongside /v1/indexer/*. Hostname-based ingress can pin a
// frontend to a specific chain.
if chain.Default {
slugSrv, _ := explorer.NewStandaloneServer(explorer.Config{
IndexerDBPath: path,
ChainID: chain.ChainID,
ChainName: chain.Name,
CoinSymbol: chain.CoinSymbol,
APIPrefix: fmt.Sprintf("/v1/indexer/%s", chain.Slug),
})
if slugSrv != nil {
mux.Handle(fmt.Sprintf("/v1/indexer/%s/", chain.Slug), slugSrv.Handler())
}
log.Printf("[%s] API mounted at /v1/indexer/* and /v1/indexer/%s/* (default)", chain.Slug, chain.Slug)
} else {
log.Printf("[%s] API mounted at %s/*", chain.Slug, apiPrefix)
}
return
}
}
daemon/server.go
+42
View File
@@ -0,0 +1,42 @@
package daemon
import (
"context"
"log"
"net/http"
"time"
)
// MountHealth registers /health and /healthz on mux.
func MountHealth(mux *http.ServeMux) {
h := func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Write([]byte(`{"status":"ok"}`))
}
mux.HandleFunc("GET /health", h)
mux.HandleFunc("GET /healthz", h)
}
// ListenAndServe starts an HTTP server on addr with the given handler and
// blocks until ctx is done. Shutdown is graceful.
func ListenAndServe(ctx context.Context, addr string, handler http.Handler) {
srv := &http.Server{
Addr: addr,
Handler: handler,
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 120 * time.Second,
MaxHeaderBytes: 1 << 16, // 64KB
}
go func() {
log.Printf("HTTP server listening on %s", addr)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Printf("HTTP server error: %v", err)
}
}()
<-ctx.Done()
srv.Shutdown(context.Background())
}
multichain/bitcoin_indexer.go
-856
View File
@@ -1,856 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"sync"
"sync/atomic"
"time"
)
// BitcoinIndexer indexes Bitcoin and Bitcoin-based chains
type BitcoinIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
// State
running int32
indexedHeight uint64
latestHeight uint64
// Protocol indexers
protocols map[ProtocolType]BitcoinProtocolIndexer
// Stats
stats *IndexerStats
// Lifecycle
ctx context.Context
cancel context.CancelFunc
}
// BitcoinProtocolIndexer handles Bitcoin protocol-specific indexing
type BitcoinProtocolIndexer interface {
Name() string
IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error
}
// BitcoinBlock represents a Bitcoin block
type BitcoinBlock struct {
Hash string `json:"hash"`
Height uint64 `json:"height"`
Version int32 `json:"version"`
PreviousBlockHash string `json:"previousblockhash"`
MerkleRoot string `json:"merkleroot"`
Time int64 `json:"time"`
Bits string `json:"bits"`
Nonce uint32 `json:"nonce"`
Difficulty float64 `json:"difficulty"`
NumTx int `json:"nTx"`
Size int `json:"size"`
Weight int `json:"weight"`
Transactions []BitcoinTransaction `json:"tx"`
}
// BitcoinTransaction represents a Bitcoin transaction
type BitcoinTransaction struct {
TxID string `json:"txid"`
Hash string `json:"hash"`
Version int32 `json:"version"`
Size int `json:"size"`
VSize int `json:"vsize"`
Weight int `json:"weight"`
LockTime uint32 `json:"locktime"`
Vin []BitcoinVin `json:"vin"`
Vout []BitcoinVout `json:"vout"`
Hex string `json:"hex"`
BlockHash string `json:"blockhash,omitempty"`
BlockHeight uint64 `json:"blockheight,omitempty"`
BlockTime int64 `json:"blocktime,omitempty"`
Fee float64 `json:"fee,omitempty"`
// Ordinals/Inscriptions
Inscriptions []Inscription `json:"inscriptions,omitempty"`
// Runes
Runes []RuneTransfer `json:"runes,omitempty"`
// BRC-20
BRC20 []BRC20Op `json:"brc20,omitempty"`
}
// BitcoinVin represents a transaction input
type BitcoinVin struct {
TxID string `json:"txid,omitempty"`
Vout uint32 `json:"vout,omitempty"`
ScriptSig Script `json:"scriptSig,omitempty"`
Sequence uint32 `json:"sequence"`
TxInWitness []string `json:"txinwitness,omitempty"`
Coinbase string `json:"coinbase,omitempty"`
}
// BitcoinVout represents a transaction output
type BitcoinVout struct {
Value float64 `json:"value"`
N uint32 `json:"n"`
ScriptPubKey ScriptPubKey `json:"scriptPubKey"`
}
// Script represents a script signature
type Script struct {
Asm string `json:"asm"`
Hex string `json:"hex"`
}
// ScriptPubKey represents the output script
type ScriptPubKey struct {
Asm string `json:"asm"`
Hex string `json:"hex"`
Type string `json:"type"`
Address string `json:"address,omitempty"`
}
// Inscription represents an Ordinal inscription
type Inscription struct {
ID string `json:"id"` // <txid>i<index>
Number uint64 `json:"number"` // Inscription number
ContentType string `json:"content_type"`
ContentLength uint64 `json:"content_length"`
Content []byte `json:"content,omitempty"`
GenesisTxID string `json:"genesis_txid"`
GenesisHeight uint64 `json:"genesis_height"`
Owner string `json:"owner"`
Sat uint64 `json:"sat"` // Satoshi ordinal
SatRarity string `json:"sat_rarity"` // common, uncommon, rare, epic, legendary, mythic
Timestamp int64 `json:"timestamp"`
}
// RuneTransfer represents a Rune transfer
type RuneTransfer struct {
RuneID string `json:"rune_id"` // <block>:<tx>
RuneName string `json:"rune_name"`
Symbol string `json:"symbol"`
Amount string `json:"amount"` // uint128 as string
From string `json:"from"`
To string `json:"to"`
Decimals uint8 `json:"decimals"`
Etching bool `json:"etching"` // Is this an etching tx?
Mint bool `json:"mint"` // Is this a mint tx?
Burn bool `json:"burn"` // Is this a burn?
}
// BRC20Op represents a BRC-20 operation
type BRC20Op struct {
Operation string `json:"op"` // deploy, mint, transfer
Ticker string `json:"tick"`
Amount string `json:"amt,omitempty"`
Max string `json:"max,omitempty"`
Limit string `json:"lim,omitempty"`
Decimals uint8 `json:"dec,omitempty"`
From string `json:"from"`
To string `json:"to,omitempty"`
Valid bool `json:"valid"`
}
// Stamp represents a Bitcoin Stamp
type Stamp struct {
StampNumber uint64 `json:"stamp"`
Creator string `json:"creator"`
CPid string `json:"cpid"`
BlockIndex uint64 `json:"block_index"`
Timestamp int64 `json:"timestamp"`
TxHash string `json:"tx_hash"`
SupplyTotal uint64 `json:"supply"`
Divisible bool `json:"divisible"`
Locked bool `json:"locked"`
}
// Atomical represents an Atomicals protocol item
type Atomical struct {
AtomicalID string `json:"atomical_id"`
AtomicalNum uint64 `json:"atomical_number"`
Type string `json:"type"` // NFT, FT, realm, container, subrealm
Ticker string `json:"ticker,omitempty"`
Name string `json:"name,omitempty"`
Owner string `json:"owner"`
Value uint64 `json:"value"`
Bitworkc string `json:"bitworkc,omitempty"` // Mining commitment
Bitworkr string `json:"bitworkr,omitempty"` // Mining reveal
ParentRealm string `json:"parent_realm,omitempty"`
}
// NewBitcoinIndexer creates a new Bitcoin indexer
func NewBitcoinIndexer(config ChainConfig, db Database) (*BitcoinIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
idx := &BitcoinIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 60 * time.Second},
protocols: make(map[ProtocolType]BitcoinProtocolIndexer),
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}
// Initialize protocol indexers
for _, proto := range config.Protocols {
if !proto.Enabled {
continue
}
if err := idx.initProtocol(proto); err != nil {
return nil, fmt.Errorf("failed to init protocol %s: %w", proto.Type, err)
}
}
return idx, nil
}
// initProtocol initializes a Bitcoin protocol indexer
func (b *BitcoinIndexer) initProtocol(config ProtocolConfig) error {
var indexer BitcoinProtocolIndexer
switch config.Type {
case ProtocolOrdinals:
indexer = NewOrdinalsIndexer(config)
case ProtocolRunes:
indexer = NewRunesIndexer(config)
case ProtocolBRC20:
indexer = NewBRC20Indexer(config)
case ProtocolStamps:
indexer = NewStampsIndexer(config)
case ProtocolAtomicals:
indexer = NewAtomicalsIndexer(config)
default:
indexer = NewGenericBitcoinIndexer(config)
}
b.protocols[config.Type] = indexer
return nil
}
// ChainID returns the chain identifier
func (b *BitcoinIndexer) ChainID() string {
return b.config.ID
}
// ChainType returns the chain type
func (b *BitcoinIndexer) ChainType() ChainType {
return ChainTypeBitcoin
}
// Start starts the indexer
func (b *BitcoinIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&b.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
b.stats.IsRunning = true
return nil
}
// Stop stops the indexer
func (b *BitcoinIndexer) Stop() error {
atomic.StoreInt32(&b.running, 0)
b.cancel()
b.stats.IsRunning = false
return nil
}
// IsRunning returns whether the indexer is running
func (b *BitcoinIndexer) IsRunning() bool {
return atomic.LoadInt32(&b.running) == 1
}
// GetLatestBlock returns the latest block height
func (b *BitcoinIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
result, err := b.rpcCall(ctx, "getblockcount", []interface{}{})
if err != nil {
return 0, err
}
height, ok := result.(float64)
if !ok {
return 0, fmt.Errorf("invalid block count response")
}
b.mu.Lock()
b.latestHeight = uint64(height)
b.mu.Unlock()
return uint64(height), nil
}
// GetIndexedBlock returns the last indexed block
func (b *BitcoinIndexer) GetIndexedBlock() uint64 {
b.mu.RLock()
defer b.mu.RUnlock()
return b.indexedHeight
}
// IndexBlock indexes a single block
func (b *BitcoinIndexer) IndexBlock(ctx context.Context, height uint64) error {
// Get block hash
hashResult, err := b.rpcCall(ctx, "getblockhash", []interface{}{height})
if err != nil {
return fmt.Errorf("failed to get block hash for height %d: %w", height, err)
}
blockHash, ok := hashResult.(string)
if !ok {
return fmt.Errorf("invalid block hash response")
}
// Get full block
block, err := b.fetchBlock(ctx, blockHash)
if err != nil {
return fmt.Errorf("failed to fetch block %s: %w", blockHash, err)
}
block.Height = height
// Process transactions through protocol indexers
for i := range block.Transactions {
tx := &block.Transactions[i]
tx.BlockHash = block.Hash
tx.BlockHeight = height
tx.BlockTime = block.Time
b.processTransaction(ctx, tx, block)
}
b.mu.Lock()
b.indexedHeight = height
b.stats.BlocksProcessed++
b.stats.TxsProcessed += uint64(len(block.Transactions))
b.stats.LastBlockTime = time.Now()
b.mu.Unlock()
return nil
}
// IndexBlockRange indexes a range of blocks
func (b *BitcoinIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for height := from; height <= to; height++ {
if err := b.IndexBlock(ctx, height); err != nil {
b.stats.ErrorCount++
b.stats.LastError = err.Error()
continue
}
}
return nil
}
// fetchBlock fetches a block from RPC
func (b *BitcoinIndexer) fetchBlock(ctx context.Context, blockHash string) (*BitcoinBlock, error) {
// Get block with full tx data (verbosity = 2)
result, err := b.rpcCall(ctx, "getblock", []interface{}{blockHash, 2})
if err != nil {
return nil, err
}
data, err := json.Marshal(result)
if err != nil {
return nil, err
}
var block BitcoinBlock
if err := json.Unmarshal(data, &block); err != nil {
return nil, err
}
return &block, nil
}
// processTransaction routes a transaction to protocol indexers
func (b *BitcoinIndexer) processTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) {
// Parse Ordinals inscriptions from witness data
b.parseOrdinals(tx)
// Parse Runes from OP_RETURN
b.parseRunes(tx)
// Parse BRC-20 from inscriptions
b.parseBRC20(tx)
// Route to protocol indexers
for _, indexer := range b.protocols {
indexer.IndexTransaction(ctx, tx, block)
}
}
// parseOrdinals extracts Ordinal inscriptions from a transaction
func (b *BitcoinIndexer) parseOrdinals(tx *BitcoinTransaction) {
// Check witness data for inscription envelope
for _, vin := range tx.Vin {
if len(vin.TxInWitness) < 2 {
continue
}
// Look for inscription envelope pattern in witness
// Inscriptions are in taproot witness scripts
for _, witness := range vin.TxInWitness {
inscription := parseInscriptionFromWitness(witness)
if inscription != nil {
inscription.GenesisTxID = tx.TxID
tx.Inscriptions = append(tx.Inscriptions, *inscription)
}
}
}
}
// parseRunes extracts Rune operations from a transaction
func (b *BitcoinIndexer) parseRunes(tx *BitcoinTransaction) {
// Runes use OP_RETURN with specific protocol identifier
for _, vout := range tx.Vout {
if vout.ScriptPubKey.Type != "nulldata" {
continue
}
// Check for Runes protocol marker (OP_RETURN OP_13)
runeTransfer := parseRuneFromOpReturn(vout.ScriptPubKey.Hex)
if runeTransfer != nil {
tx.Runes = append(tx.Runes, *runeTransfer)
}
}
}
// parseBRC20 extracts BRC-20 operations from inscriptions
func (b *BitcoinIndexer) parseBRC20(tx *BitcoinTransaction) {
for _, inscription := range tx.Inscriptions {
if inscription.ContentType != "text/plain" && inscription.ContentType != "application/json" {
continue
}
brc20Op := parseBRC20FromContent(inscription.Content)
if brc20Op != nil {
tx.BRC20 = append(tx.BRC20, *brc20Op)
}
}
}
// rpcCall makes a JSON-RPC call to Bitcoin Core
func (b *BitcoinIndexer) rpcCall(ctx context.Context, method string, params []interface{}) (interface{}, error) {
body := map[string]interface{}{
"jsonrpc": "1.0",
"id": "indexer",
"method": method,
"params": params,
}
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, "POST", b.config.RPC, nil)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
req.Body = io.NopCloser(jsonReader(data))
// Add basic auth if configured
if b.config.API != "" {
req.SetBasicAuth("rpc", b.config.API)
}
resp, err := b.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result struct {
Result interface{} `json:"result"`
Error *struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
if result.Error != nil {
return nil, fmt.Errorf("RPC error %d: %s", result.Error.Code, result.Error.Message)
}
return result.Result, nil
}
// Stats returns indexer statistics
func (b *BitcoinIndexer) Stats() *IndexerStats {
b.mu.RLock()
defer b.mu.RUnlock()
return &IndexerStats{
ChainID: b.stats.ChainID,
ChainName: b.stats.ChainName,
IsRunning: b.stats.IsRunning,
LatestBlock: b.latestHeight,
IndexedBlock: b.indexedHeight,
BlocksBehind: b.latestHeight - b.indexedHeight,
BlocksProcessed: b.stats.BlocksProcessed,
TxsProcessed: b.stats.TxsProcessed,
EventsProcessed: b.stats.EventsProcessed,
ErrorCount: b.stats.ErrorCount,
LastError: b.stats.LastError,
LastBlockTime: b.stats.LastBlockTime,
StartTime: b.stats.StartTime,
Uptime: time.Since(b.stats.StartTime),
}
}
// =============================================================================
// Helper Functions
// =============================================================================
// parseInscriptionFromWitness extracts inscription from witness data
func parseInscriptionFromWitness(witness string) *Inscription {
// Decode hex witness data
data := hexToBytes(witness)
if len(data) < 10 {
return nil
}
// Look for inscription envelope pattern:
// OP_FALSE (0x00) OP_IF (0x63) "ord" (0x036f7264) OP_1 (0x51) content_type OP_0 (0x00) content OP_ENDIF (0x68)
// Find "ord" marker
ordMarker := []byte{0x03, 0x6f, 0x72, 0x64} // length-prefixed "ord"
ordIdx := -1
for i := 0; i <= len(data)-len(ordMarker); i++ {
if bytes.Equal(data[i:i+len(ordMarker)], ordMarker) {
ordIdx = i
break
}
}
if ordIdx < 0 {
return nil
}
// Parse content type and content
pos := ordIdx + 4
if pos >= len(data) {
return nil
}
inscription := &Inscription{}
// Parse fields until OP_ENDIF (0x68) or OP_0 (0x00)
for pos < len(data) {
op := data[pos]
pos++
switch op {
case 0x01: // OP_1 - content type follows
if pos >= len(data) {
return nil
}
length := int(data[pos])
pos++
if pos+length > len(data) {
return nil
}
inscription.ContentType = string(data[pos : pos+length])
pos += length
case 0x00: // OP_0 - content follows
// Read content length (could be push opcode)
if pos >= len(data) {
break
}
// Handle different push opcodes
contentLen := 0
pushOp := data[pos]
pos++
if pushOp <= 0x4b { // Direct push 1-75 bytes
contentLen = int(pushOp)
} else if pushOp == 0x4c { // OP_PUSHDATA1
if pos >= len(data) {
break
}
contentLen = int(data[pos])
pos++
} else if pushOp == 0x4d { // OP_PUSHDATA2
if pos+2 > len(data) {
break
}
contentLen = int(data[pos]) | int(data[pos+1])<<8
pos += 2
} else if pushOp == 0x4e { // OP_PUSHDATA4
if pos+4 > len(data) {
break
}
contentLen = int(data[pos]) | int(data[pos+1])<<8 | int(data[pos+2])<<16 | int(data[pos+3])<<24
pos += 4
}
if pos+contentLen <= len(data) {
inscription.Content = data[pos : pos+contentLen]
inscription.ContentLength = uint64(contentLen)
pos += contentLen
}
case 0x68: // OP_ENDIF - end of envelope
if inscription.ContentType != "" || len(inscription.Content) > 0 {
return inscription
}
return nil
}
}
if inscription.ContentType != "" || len(inscription.Content) > 0 {
return inscription
}
return nil
}
// parseRuneFromOpReturn extracts Rune from OP_RETURN data
func parseRuneFromOpReturn(hex string) *RuneTransfer {
data := hexToBytes(hex)
if len(data) < 4 {
return nil
}
// Runes protocol: OP_RETURN (0x6a) OP_13 (0x5d) <payload>
// Find OP_RETURN OP_13 pattern
if data[0] != 0x6a { // OP_RETURN
return nil
}
// Check for Runes magic number (OP_13 = 0x5d)
pos := 1
if pos >= len(data) {
return nil
}
// Skip push length byte if present
if data[pos] <= 0x4b {
pos++
}
if pos >= len(data) || data[pos] != 0x5d { // OP_13 = Runes identifier
return nil
}
pos++
// Parse Rune payload using LEB128 varint encoding
rune := &RuneTransfer{}
// Read edicts (transfers)
if pos < len(data) {
// First varint is the number of edicts or flags
flags, n := decodeVarint(data[pos:])
if n == 0 {
return nil
}
pos += n
// Check if this is an etching (bit 0 set)
rune.Etching = (flags & 1) != 0
// Check if this is a mint (bit 1 set)
rune.Mint = (flags & 2) != 0
// Parse rune ID if present
if pos < len(data) {
blockNum, n := decodeVarint(data[pos:])
if n > 0 {
pos += n
txIdx, n := decodeVarint(data[pos:])
if n > 0 {
pos += n
rune.RuneID = fmt.Sprintf("%d:%d", blockNum, txIdx)
}
}
}
// Parse amount if present
if pos < len(data) {
amount, n := decodeVarint(data[pos:])
if n > 0 {
rune.Amount = fmt.Sprintf("%d", amount)
}
}
}
if rune.RuneID != "" || rune.Etching || rune.Mint {
return rune
}
return nil
}
// decodeVarint decodes a LEB128 varint from bytes
func decodeVarint(data []byte) (uint64, int) {
var result uint64
var shift uint
for i, b := range data {
if i >= 10 { // Max 10 bytes for uint64
return 0, 0
}
result |= uint64(b&0x7f) << shift
if b&0x80 == 0 {
return result, i + 1
}
shift += 7
}
return 0, 0
}
// parseBRC20FromContent extracts BRC-20 operation from inscription content
func parseBRC20FromContent(content []byte) *BRC20Op {
if len(content) == 0 {
return nil
}
// BRC-20 format: {"p":"brc-20","op":"deploy","tick":"ordi","max":"21000000","lim":"1000"}
var data map[string]interface{}
if err := json.Unmarshal(content, &data); err != nil {
return nil
}
// Check protocol identifier
p, ok := data["p"].(string)
if !ok || p != "brc-20" {
return nil
}
op := &BRC20Op{}
// Parse operation type
if opType, ok := data["op"].(string); ok {
op.Operation = opType
} else {
return nil
}
// Parse ticker (required)
if tick, ok := data["tick"].(string); ok {
op.Ticker = tick
} else {
return nil
}
// Parse optional fields based on operation type
switch op.Operation {
case "deploy":
if max, ok := data["max"].(string); ok {
op.Max = max
}
if lim, ok := data["lim"].(string); ok {
op.Limit = lim
}
if dec, ok := data["dec"].(float64); ok {
op.Decimals = uint8(dec)
} else {
op.Decimals = 18 // Default
}
op.Valid = op.Max != "" && op.Ticker != ""
case "mint":
if amt, ok := data["amt"].(string); ok {
op.Amount = amt
}
op.Valid = op.Amount != "" && op.Ticker != ""
case "transfer":
if amt, ok := data["amt"].(string); ok {
op.Amount = amt
}
op.Valid = op.Amount != "" && op.Ticker != ""
}
return op
}
// =============================================================================
// Bitcoin Protocol Indexer Stubs
// =============================================================================
type GenericBitcoinIndexer struct{ config ProtocolConfig }
func NewGenericBitcoinIndexer(config ProtocolConfig) *GenericBitcoinIndexer {
return &GenericBitcoinIndexer{config: config}
}
func (g *GenericBitcoinIndexer) Name() string { return "generic_bitcoin" }
func (g *GenericBitcoinIndexer) IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error {
return nil
}
// Ordinals Indexer
type OrdinalsIndexer struct{ config ProtocolConfig }
func NewOrdinalsIndexer(config ProtocolConfig) *OrdinalsIndexer {
return &OrdinalsIndexer{config: config}
}
func (o *OrdinalsIndexer) Name() string { return "ordinals" }
func (o *OrdinalsIndexer) IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error {
// Index inscriptions
for _, inscription := range tx.Inscriptions {
_ = inscription // Store inscription
}
return nil
}
// Runes Indexer
type RunesIndexer struct{ config ProtocolConfig }
func NewRunesIndexer(config ProtocolConfig) *RunesIndexer { return &RunesIndexer{config: config} }
func (r *RunesIndexer) Name() string { return "runes" }
func (r *RunesIndexer) IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error {
// Index rune transfers, etchings, mints
for _, rune := range tx.Runes {
_ = rune // Store rune operation
}
return nil
}
// BRC-20 Indexer
type BRC20Indexer struct{ config ProtocolConfig }
func NewBRC20Indexer(config ProtocolConfig) *BRC20Indexer { return &BRC20Indexer{config: config} }
func (b *BRC20Indexer) Name() string { return "brc20" }
func (b *BRC20Indexer) IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error {
// Index BRC-20 operations
for _, op := range tx.BRC20 {
_ = op // Store BRC-20 operation
}
return nil
}
// Stamps Indexer
type StampsIndexer struct{ config ProtocolConfig }
func NewStampsIndexer(config ProtocolConfig) *StampsIndexer { return &StampsIndexer{config: config} }
func (s *StampsIndexer) Name() string { return "stamps" }
func (s *StampsIndexer) IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error {
// Index Bitcoin Stamps (Counterparty-based)
return nil
}
// Atomicals Indexer
type AtomicalsIndexer struct{ config ProtocolConfig }
func NewAtomicalsIndexer(config ProtocolConfig) *AtomicalsIndexer {
return &AtomicalsIndexer{config: config}
}
func (a *AtomicalsIndexer) Name() string { return "atomicals" }
func (a *AtomicalsIndexer) IndexTransaction(ctx context.Context, tx *BitcoinTransaction, block *BitcoinBlock) error {
// Index Atomicals protocol items
return nil
}
multichain/config.go
-395
View File
@@ -1,395 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"fmt"
"os"
"strings"
"time"
"gopkg.in/yaml.v3"
)
// ChainsConfig represents the full chains.yaml configuration
type ChainsConfig struct {
Lux map[string]YAMLChainConfig `yaml:"lux"`
Ethereum map[string]YAMLChainConfig `yaml:"ethereum"`
Solana map[string]YAMLChainConfig `yaml:"solana"`
Bitcoin map[string]YAMLChainConfig `yaml:"bitcoin"`
Cosmos map[string]YAMLChainConfig `yaml:"cosmos"`
Move map[string]YAMLChainConfig `yaml:"move"`
Other map[string]YAMLChainConfig `yaml:"other"`
}
// YAMLChainConfig represents a chain configuration from YAML
type YAMLChainConfig struct {
ChainID uint64 `yaml:"chain_id"`
Name string `yaml:"name"`
Symbol string `yaml:"symbol"`
Type string `yaml:"type"`
RPC string `yaml:"rpc"`
WS string `yaml:"ws,omitempty"`
API string `yaml:"api,omitempty"`
Explorer string `yaml:"explorer,omitempty"`
Enabled bool `yaml:"enabled"`
NFTMarketplaces []YAMLMarketplace `yaml:"nft_marketplaces,omitempty"`
Protocols []YAMLProtocol `yaml:"protocols,omitempty"`
StartBlock uint64 `yaml:"start_block,omitempty"`
BatchSize int `yaml:"batch_size,omitempty"`
PollInterval string `yaml:"poll_interval,omitempty"`
}
// YAMLMarketplace represents an NFT marketplace config
type YAMLMarketplace struct {
Protocol string `yaml:"protocol"`
Address string `yaml:"address,omitempty"`
StartBlock uint64 `yaml:"start_block,omitempty"`
}
// YAMLProtocol represents a protocol config
type YAMLProtocol struct {
Type string `yaml:"type"`
Address string `yaml:"address,omitempty"`
Factory string `yaml:"factory,omitempty"`
StartBlock uint64 `yaml:"start_block,omitempty"`
Enabled bool `yaml:"enabled"`
}
// LoadChainsConfig loads chains configuration from a YAML file
func LoadChainsConfig(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
// Expand environment variables
expanded := os.ExpandEnv(string(data))
var chainsConfig ChainsConfig
if err := yaml.Unmarshal([]byte(expanded), &chainsConfig); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
config := &Config{
Chains: make([]ChainConfig, 0),
MaxConcurrentChains: 100, // Default
DefaultBatchSize: 100,
DefaultPollInterval: 5 * time.Second,
RetryAttempts: 3,
RetryDelay: time.Second,
}
// Process each chain category
for id, yc := range chainsConfig.Lux {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
for id, yc := range chainsConfig.Ethereum {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
for id, yc := range chainsConfig.Solana {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
for id, yc := range chainsConfig.Bitcoin {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
for id, yc := range chainsConfig.Cosmos {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
for id, yc := range chainsConfig.Move {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
for id, yc := range chainsConfig.Other {
if cc := convertYAMLConfig(id, yc); cc != nil {
config.Chains = append(config.Chains, *cc)
}
}
return config, nil
}
// convertYAMLConfig converts a YAML chain config to ChainConfig
func convertYAMLConfig(id string, yc YAMLChainConfig) *ChainConfig {
if !yc.Enabled {
return nil
}
chainType := parseChainType(yc.Type)
pollInterval := 5 * time.Second
if yc.PollInterval != "" {
if d, err := time.ParseDuration(yc.PollInterval); err == nil {
pollInterval = d
}
}
batchSize := 100
if yc.BatchSize > 0 {
batchSize = yc.BatchSize
}
cc := &ChainConfig{
ID: id,
ChainID: yc.ChainID,
Name: yc.Name,
Symbol: yc.Symbol,
Type: chainType,
RPC: yc.RPC,
WS: yc.WS,
API: yc.API,
Explorer: yc.Explorer,
Enabled: true,
StartBlock: yc.StartBlock,
BatchSize: batchSize,
PollInterval: pollInterval,
Protocols: make([]ProtocolConfig, 0),
}
// Convert NFT marketplaces to protocols
for _, mp := range yc.NFTMarketplaces {
proto := ProtocolConfig{
Type: parseProtocolType(mp.Protocol),
Address: mp.Address,
StartBlock: mp.StartBlock,
Enabled: true,
}
cc.Protocols = append(cc.Protocols, proto)
}
// Convert protocols
for _, p := range yc.Protocols {
if !p.Enabled {
continue
}
proto := ProtocolConfig{
Type: parseProtocolType(p.Type),
Address: p.Address,
Factory: p.Factory,
StartBlock: p.StartBlock,
Enabled: true,
}
cc.Protocols = append(cc.Protocols, proto)
}
return cc
}
// parseChainType parses chain type from string
func parseChainType(s string) ChainType {
switch strings.ToLower(s) {
case "evm":
return ChainTypeEVM
case "solana":
return ChainTypeSolana
case "bitcoin":
return ChainTypeBitcoin
case "cosmos":
return ChainTypeCosmos
case "move":
return ChainTypeMove
case "near":
return ChainTypeNear
case "tron":
return ChainTypeTron
case "ton":
return ChainTypeTon
case "substrate", "polkadot":
return ChainTypeSubstrate
case "stacks":
return ChainTypeStacks
case "hedera":
return ChainTypeHedera
case "filecoin":
return ChainTypeFilecoin
case "icp":
return ChainTypeICP
case "multiversx":
return ChainTypeMultiversX
// Purpose-built native chains — legacy "lux_*" aliases preserved for
// backwards compatibility with existing yaml configs.
case "platform", "platformvm", "pchain", "p-chain", "lux_platform":
return ChainTypePlatform
case "utxo", "xvm", "xchain", "x-chain", "lux_xvm":
return ChainTypeUTXO
case "ai", "aivm", "achain", "a-chain", "lux_ai":
return ChainTypeAI
case "bridge", "bridgevm", "bchain", "b-chain", "lux_bridge":
return ChainTypeBridge
case "dex", "dexvm", "dchain", "d-chain", "lux_dex":
return ChainTypeDEX
case "graph", "graphvm", "gchain", "g-chain", "lux_graph":
return ChainTypeGraph
case "identity", "identityvm", "ichain", "i-chain", "lux_identity":
return ChainTypeIdentity
case "key", "keyvm", "kchain", "k-chain", "lux_key":
return ChainTypeKey
case "mpc", "mpcvm", "mchain", "m-chain", "lux_mpc":
return ChainTypeMPC
case "oracle", "oraclevm", "ochain", "o-chain", "lux_oracle":
return ChainTypeOracle
case "quantum", "quantumvm", "qchain", "q-chain", "lux_quantum":
return ChainTypeQuantum
case "relay", "relayvm", "rchain", "r-chain", "lux_relay":
return ChainTypeRelay
case "threshold", "thresholdvm", "tchain", "t-chain", "lux_threshold":
return ChainTypeThreshold
case "zk", "zkvm", "zchain", "z-chain", "lux_zk":
return ChainTypeZK
default:
return ChainTypeEVM
}
}
// parseProtocolType parses protocol type from string
func parseProtocolType(s string) ProtocolType {
switch strings.ToLower(s) {
// NFT Marketplaces
case "seaport", "opensea":
return ProtocolSeaport
case "looksrare", "looksrare_v2":
return ProtocolLooksRare
case "blur":
return ProtocolBlur
case "x2y2":
return ProtocolX2Y2
case "rarible":
return ProtocolRarible
case "zora":
return ProtocolZora
case "magic_eden", "magiceden":
return ProtocolMagicEden
case "tensor":
return ProtocolTensor
case "sudoswap":
return ProtocolSudoswap
// DEX
case "uniswap_v2", "uniswapv2":
return ProtocolUniswapV2
case "uniswap_v3", "uniswapv3":
return ProtocolUniswapV3
case "sushiswap":
return ProtocolSushiswap
case "curve":
return ProtocolCurve
case "balancer":
return ProtocolBalancer
case "pancakeswap":
return ProtocolPancakeswap
// Lending
case "aave_v2", "aavev2":
return ProtocolAaveV2
case "aave_v3", "aavev3":
return ProtocolAaveV3
case "compound_v2", "compoundv2":
return ProtocolCompoundV2
case "compound_v3", "compoundv3":
return ProtocolCompoundV3
// Perpetuals
case "gmx":
return ProtocolGMX
case "gmx_v2", "gmxv2":
return ProtocolGMXV2
case "synthetix":
return ProtocolSynthetix
case "dydx":
return ProtocolDYDX
//
case "lido":
return ProtocolLido
case "rocket_pool", "rocketpool":
return ProtocolRocketPool
case "eigenlayer":
return ProtocolEigenlayer
// Bridges
case "wormhole":
return ProtocolWormhole
case "layerzero":
return ProtocolLayerZero
case "stargate":
return ProtocolStargate
// Bitcoin
case "ordinals":
return ProtocolOrdinals
case "runes":
return ProtocolRunes
case "brc20", "brc-20":
return ProtocolBRC20
case "stamps":
return ProtocolStamps
case "atomicals":
return ProtocolAtomicals
// Solana
case "raydium":
return ProtocolRaydium
case "orca":
return ProtocolOrca
case "marinade":
return ProtocolMarinade
case "jito":
return ProtocolJito
case "jupiter":
return ProtocolJupiter
case "drift":
return ProtocolDrift
case "mango":
return ProtocolMango
case "phoenix":
return ProtocolPhoenix
case "metaplex":
return ProtocolMetaplex
default:
return ProtocolType(s)
}
}
// DefaultConfig returns a default configuration with major chains enabled
func DefaultConfig() *Config {
return &Config{
MaxConcurrentChains: 100,
DefaultBatchSize: 100,
DefaultPollInterval: 5 * time.Second,
RetryAttempts: 3,
RetryDelay: time.Second,
Chains: []ChainConfig{
// Lux
{ID: "lux_cchain", ChainID: 96369, Name: "Lux C-Chain", Symbol: "LUX", Type: ChainTypeEVM, RPC: "https://api.lux.network/ext/bc/C/rpc", Enabled: true},
{ID: "lux_testnet", ChainID: 96368, Name: "Lux Testnet", Symbol: "LUX", Type: ChainTypeEVM, RPC: "https://api.testnet.lux.network/ext/bc/C/rpc", Enabled: true},
// Ethereum Ecosystem
{ID: "ethereum", ChainID: 1, Name: "Ethereum", Symbol: "ETH", Type: ChainTypeEVM, RPC: "https://eth.llamarpc.com", Enabled: true},
{ID: "polygon", ChainID: 137, Name: "Polygon", Symbol: "MATIC", Type: ChainTypeEVM, RPC: "https://polygon.llamarpc.com", Enabled: true},
{ID: "arbitrum", ChainID: 42161, Name: "Arbitrum One", Symbol: "ETH", Type: ChainTypeEVM, RPC: "https://arb1.arbitrum.io/rpc", Enabled: true},
{ID: "optimism", ChainID: 10, Name: "Optimism", Symbol: "ETH", Type: ChainTypeEVM, RPC: "https://mainnet.optimism.io", Enabled: true},
{ID: "base", ChainID: 8453, Name: "Base", Symbol: "ETH", Type: ChainTypeEVM, RPC: "https://mainnet.base.org", Enabled: true},
{ID: "bsc", ChainID: 56, Name: "BNB Chain", Symbol: "BNB", Type: ChainTypeEVM, RPC: "https://bsc-dataseed1.binance.org", Enabled: true},
{ID: "avalanche", ChainID: 43114, Name: "Avalanche C-Chain", Symbol: "AVAX", Type: ChainTypeEVM, RPC: "https://api.avax.network/ext/bc/C/rpc", Enabled: true},
// Solana
{ID: "solana", ChainID: 0, Name: "Solana", Symbol: "SOL", Type: ChainTypeSolana, RPC: "https://api.mainnet-beta.solana.com", Enabled: true},
// Bitcoin
{ID: "bitcoin", ChainID: 0, Name: "Bitcoin", Symbol: "BTC", Type: ChainTypeBitcoin, RPC: "http://localhost:8332", Enabled: false}, // Requires local node
},
}
}
multichain/cosmos_indexer.go
-568
View File
@@ -1,568 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"sync"
"sync/atomic"
"time"
)
// CosmosIndexer indexes Cosmos SDK-based chains
type CosmosIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
// State
running int32
indexedHeight uint64
latestHeight uint64
// Protocol indexers
protocols map[ProtocolType]CosmosProtocolIndexer
// Stats
stats *IndexerStats
// Lifecycle
ctx context.Context
cancel context.CancelFunc
}
// CosmosProtocolIndexer handles Cosmos protocol-specific indexing
type CosmosProtocolIndexer interface {
Name() string
MessageTypes() []string
IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error
}
// CosmosBlock represents a Cosmos block
type CosmosBlock struct {
BlockID BlockID `json:"block_id"`
Block BlockData `json:"block"`
Txs []CosmosTx `json:"txs,omitempty"`
}
// BlockID represents block identification
type BlockID struct {
Hash string `json:"hash"`
Parts struct {
Total int `json:"total"`
Hash string `json:"hash"`
} `json:"parts"`
}
// BlockData represents block data
type BlockData struct {
Header BlockHeader `json:"header"`
Data struct {
Txs []string `json:"txs"` // Base64 encoded
} `json:"data"`
Evidence struct {
Evidence []interface{} `json:"evidence"`
} `json:"evidence"`
LastCommit LastCommit `json:"last_commit"`
}
// BlockHeader represents block header
type BlockHeader struct {
Version struct {
Block string `json:"block"`
App string `json:"app"`
} `json:"version"`
ChainID string `json:"chain_id"`
Height string `json:"height"`
Time time.Time `json:"time"`
LastBlockID BlockID `json:"last_block_id"`
LastCommitHash string `json:"last_commit_hash"`
DataHash string `json:"data_hash"`
ValidatorsHash string `json:"validators_hash"`
ConsensusHash string `json:"consensus_hash"`
AppHash string `json:"app_hash"`
LastResultsHash string `json:"last_results_hash"`
EvidenceHash string `json:"evidence_hash"`
ProposerAddress string `json:"proposer_address"`
}
// LastCommit represents last commit info
type LastCommit struct {
Height string `json:"height"`
Round int `json:"round"`
BlockID BlockID `json:"block_id"`
Signatures []struct {
BlockIDFlag int `json:"block_id_flag"`
ValidatorAddress string `json:"validator_address"`
Timestamp time.Time `json:"timestamp"`
Signature string `json:"signature"`
} `json:"signatures"`
}
// CosmosTx represents a Cosmos transaction
type CosmosTx struct {
Hash string `json:"hash"`
Height uint64 `json:"height"`
Index uint32 `json:"index"`
Code uint32 `json:"code"` // 0 = success
Data string `json:"data"`
RawLog string `json:"raw_log"`
Log []TxLog `json:"logs"`
GasWanted uint64 `json:"gas_wanted"`
GasUsed uint64 `json:"gas_used"`
Tx TxBody `json:"tx"`
Timestamp time.Time `json:"timestamp"`
Events []CosmosEvent `json:"events"`
}
// TxBody represents the transaction body
type TxBody struct {
Body struct {
Messages []CosmosMessage `json:"messages"`
Memo string `json:"memo"`
TimeoutHeight string `json:"timeout_height"`
ExtensionOptions []interface{} `json:"extension_options"`
NonCriticalExtensionOptions []interface{} `json:"non_critical_extension_options"`
} `json:"body"`
AuthInfo struct {
SignerInfos []SignerInfo `json:"signer_infos"`
Fee TxFee `json:"fee"`
} `json:"auth_info"`
Signatures []string `json:"signatures"`
}
// CosmosMessage represents a Cosmos message
type CosmosMessage struct {
Type string `json:"@type"`
Value map[string]interface{} `json:"-"` // Decoded message content
Raw json.RawMessage `json:"-"` // Raw JSON for custom parsing
}
// SignerInfo represents signer information
type SignerInfo struct {
PublicKey struct {
Type string `json:"@type"`
Key string `json:"key"`
} `json:"public_key"`
ModeInfo struct {
Single struct {
Mode string `json:"mode"`
} `json:"single"`
} `json:"mode_info"`
Sequence string `json:"sequence"`
}
// TxFee represents transaction fee
type TxFee struct {
Amount []Coin `json:"amount"`
GasLimit string `json:"gas_limit"`
Payer string `json:"payer"`
Granter string `json:"granter"`
}
// Coin represents a coin
type Coin struct {
Denom string `json:"denom"`
Amount string `json:"amount"`
}
// TxLog represents transaction log entry
type TxLog struct {
MsgIndex uint32 `json:"msg_index"`
Log string `json:"log"`
Events []CosmosEvent `json:"events"`
}
// CosmosEvent represents an event
type CosmosEvent struct {
Type string `json:"type"`
Attributes []EventAttribute `json:"attributes"`
}
// EventAttribute represents event attribute
type EventAttribute struct {
Key string `json:"key"`
Value string `json:"value"`
Index bool `json:"index,omitempty"`
}
// NewCosmosIndexer creates a new Cosmos indexer
func NewCosmosIndexer(config ChainConfig, db Database) (*CosmosIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
idx := &CosmosIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 30 * time.Second},
protocols: make(map[ProtocolType]CosmosProtocolIndexer),
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}
// Initialize protocol indexers based on config
for _, proto := range config.Protocols {
if !proto.Enabled {
continue
}
idx.initProtocol(proto)
}
return idx, nil
}
// initProtocol initializes a Cosmos protocol indexer
func (c *CosmosIndexer) initProtocol(config ProtocolConfig) {
var indexer CosmosProtocolIndexer
switch c.config.ID {
case "osmosis":
indexer = NewOsmosisIndexer(config)
case "injective":
indexer = NewInjectiveIndexer(config)
case "dydx":
indexer = NewDYDXCosmosIndexer(config)
case "celestia":
indexer = NewCelestiaIndexer(config)
case "sei":
indexer = NewSeiIndexer(config)
case "neutron":
indexer = NewNeutronIndexer(config)
default:
indexer = NewGenericCosmosIndexer(config)
}
c.protocols[config.Type] = indexer
}
// ChainID returns the chain identifier
func (c *CosmosIndexer) ChainID() string {
return c.config.ID
}
// ChainType returns the chain type
func (c *CosmosIndexer) ChainType() ChainType {
return ChainTypeCosmos
}
// Start starts the indexer
func (c *CosmosIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&c.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
c.stats.IsRunning = true
return nil
}
// Stop stops the indexer
func (c *CosmosIndexer) Stop() error {
atomic.StoreInt32(&c.running, 0)
c.cancel()
c.stats.IsRunning = false
return nil
}
// IsRunning returns whether the indexer is running
func (c *CosmosIndexer) IsRunning() bool {
return atomic.LoadInt32(&c.running) == 1
}
// GetLatestBlock returns the latest block height
func (c *CosmosIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
url := fmt.Sprintf("%s/cosmos/base/tendermint/v1beta1/blocks/latest", c.config.RPC)
resp, err := c.httpGet(ctx, url)
if err != nil {
return 0, err
}
var result struct {
Block struct {
Header struct {
Height string `json:"height"`
} `json:"header"`
} `json:"block"`
}
if err := json.Unmarshal(resp, &result); err != nil {
return 0, err
}
var height uint64
fmt.Sscanf(result.Block.Header.Height, "%d", &height)
c.mu.Lock()
c.latestHeight = height
c.mu.Unlock()
return height, nil
}
// GetIndexedBlock returns the last indexed block
func (c *CosmosIndexer) GetIndexedBlock() uint64 {
c.mu.RLock()
defer c.mu.RUnlock()
return c.indexedHeight
}
// IndexBlock indexes a single block
func (c *CosmosIndexer) IndexBlock(ctx context.Context, height uint64) error {
// Fetch block
block, err := c.fetchBlock(ctx, height)
if err != nil {
return fmt.Errorf("failed to fetch block %d: %w", height, err)
}
// Fetch transactions for this block
txs, err := c.fetchBlockTxs(ctx, height)
if err != nil {
return fmt.Errorf("failed to fetch txs for block %d: %w", height, err)
}
block.Txs = txs
// Process transactions
for _, tx := range txs {
c.processTransaction(ctx, &tx)
}
c.mu.Lock()
c.indexedHeight = height
c.stats.BlocksProcessed++
c.stats.TxsProcessed += uint64(len(txs))
c.stats.LastBlockTime = time.Now()
c.mu.Unlock()
return nil
}
// IndexBlockRange indexes a range of blocks
func (c *CosmosIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for height := from; height <= to; height++ {
if err := c.IndexBlock(ctx, height); err != nil {
c.stats.ErrorCount++
c.stats.LastError = err.Error()
continue
}
}
return nil
}
// fetchBlock fetches a block from the API
func (c *CosmosIndexer) fetchBlock(ctx context.Context, height uint64) (*CosmosBlock, error) {
url := fmt.Sprintf("%s/cosmos/base/tendermint/v1beta1/blocks/%d", c.config.RPC, height)
resp, err := c.httpGet(ctx, url)
if err != nil {
return nil, err
}
var block CosmosBlock
if err := json.Unmarshal(resp, &block); err != nil {
return nil, err
}
return &block, nil
}
// fetchBlockTxs fetches transactions for a block
func (c *CosmosIndexer) fetchBlockTxs(ctx context.Context, height uint64) ([]CosmosTx, error) {
url := fmt.Sprintf("%s/cosmos/tx/v1beta1/txs?events=tx.height=%d", c.config.RPC, height)
resp, err := c.httpGet(ctx, url)
if err != nil {
return nil, err
}
var result struct {
Txs []CosmosTx `json:"tx_responses"`
Pagination struct {
NextKey string `json:"next_key"`
Total string `json:"total"`
} `json:"pagination"`
}
if err := json.Unmarshal(resp, &result); err != nil {
return nil, err
}
return result.Txs, nil
}
// processTransaction routes a transaction to protocol indexers
func (c *CosmosIndexer) processTransaction(ctx context.Context, tx *CosmosTx) {
// Extract messages from transaction
for _, msg := range tx.Tx.Body.Messages {
for _, indexer := range c.protocols {
indexer.IndexMessage(ctx, &msg, tx)
}
}
}
// httpGet makes an HTTP GET request
func (c *CosmosIndexer) httpGet(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, err
}
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// Stats returns indexer statistics
func (c *CosmosIndexer) Stats() *IndexerStats {
c.mu.RLock()
defer c.mu.RUnlock()
return &IndexerStats{
ChainID: c.stats.ChainID,
ChainName: c.stats.ChainName,
IsRunning: c.stats.IsRunning,
LatestBlock: c.latestHeight,
IndexedBlock: c.indexedHeight,
BlocksBehind: c.latestHeight - c.indexedHeight,
BlocksProcessed: c.stats.BlocksProcessed,
TxsProcessed: c.stats.TxsProcessed,
EventsProcessed: c.stats.EventsProcessed,
ErrorCount: c.stats.ErrorCount,
LastError: c.stats.LastError,
LastBlockTime: c.stats.LastBlockTime,
StartTime: c.stats.StartTime,
Uptime: time.Since(c.stats.StartTime),
}
}
// =============================================================================
// Cosmos Protocol Indexer Stubs
// =============================================================================
type GenericCosmosIndexer struct{ config ProtocolConfig }
func NewGenericCosmosIndexer(config ProtocolConfig) *GenericCosmosIndexer {
return &GenericCosmosIndexer{config: config}
}
func (g *GenericCosmosIndexer) Name() string { return "generic_cosmos" }
func (g *GenericCosmosIndexer) MessageTypes() []string { return nil }
func (g *GenericCosmosIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
// Osmosis DEX
type OsmosisIndexer struct{ config ProtocolConfig }
func NewOsmosisIndexer(config ProtocolConfig) *OsmosisIndexer { return &OsmosisIndexer{config: config} }
func (o *OsmosisIndexer) Name() string { return "osmosis" }
func (o *OsmosisIndexer) MessageTypes() []string {
return []string{
"/osmosis.gamm.v1beta1.MsgSwapExactAmountIn",
"/osmosis.gamm.v1beta1.MsgSwapExactAmountOut",
"/osmosis.gamm.v1beta1.MsgJoinPool",
"/osmosis.gamm.v1beta1.MsgExitPool",
"/osmosis.concentratedliquidity.v1beta1.MsgCreatePosition",
"/osmosis.superfluid.MsgSuperfluidDelegate",
}
}
func (o *OsmosisIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
// Injective DEX
type InjectiveIndexer struct{ config ProtocolConfig }
func NewInjectiveIndexer(config ProtocolConfig) *InjectiveIndexer {
return &InjectiveIndexer{config: config}
}
func (i *InjectiveIndexer) Name() string { return "injective" }
func (i *InjectiveIndexer) MessageTypes() []string {
return []string{
"/injective.exchange.v1beta1.MsgCreateSpotLimitOrder",
"/injective.exchange.v1beta1.MsgCreateDerivativeLimitOrder",
"/injective.exchange.v1beta1.MsgInstantSpotMarketLaunch",
"/injective.exchange.v1beta1.MsgDeposit",
}
}
func (i *InjectiveIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
// dYdX Cosmos
type DYDXCosmosIndexer struct{ config ProtocolConfig }
func NewDYDXCosmosIndexer(config ProtocolConfig) *DYDXCosmosIndexer {
return &DYDXCosmosIndexer{config: config}
}
func (d *DYDXCosmosIndexer) Name() string { return "dydx_cosmos" }
func (d *DYDXCosmosIndexer) MessageTypes() []string {
return []string{
"/dydxprotocol.clob.MsgPlaceOrder",
"/dydxprotocol.clob.MsgCancelOrder",
"/dydxprotocol.sending.MsgCreateTransfer",
"/dydxprotocol.subaccounts.MsgCreateSubaccount",
}
}
func (d *DYDXCosmosIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
// Celestia Data Availability
type CelestiaIndexer struct{ config ProtocolConfig }
func NewCelestiaIndexer(config ProtocolConfig) *CelestiaIndexer {
return &CelestiaIndexer{config: config}
}
func (c *CelestiaIndexer) Name() string { return "celestia" }
func (c *CelestiaIndexer) MessageTypes() []string {
return []string{
"/celestia.blob.v1.MsgPayForBlobs",
}
}
func (c *CelestiaIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
// Sei
type SeiIndexer struct{ config ProtocolConfig }
func NewSeiIndexer(config ProtocolConfig) *SeiIndexer { return &SeiIndexer{config: config} }
func (s *SeiIndexer) Name() string { return "sei" }
func (s *SeiIndexer) MessageTypes() []string {
return []string{
"/seiprotocol.seichain.dex.MsgPlaceOrders",
"/seiprotocol.seichain.dex.MsgCancelOrders",
}
}
func (s *SeiIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
// Neutron
type NeutronIndexer struct{ config ProtocolConfig }
func NewNeutronIndexer(config ProtocolConfig) *NeutronIndexer { return &NeutronIndexer{config: config} }
func (n *NeutronIndexer) Name() string { return "neutron" }
func (n *NeutronIndexer) MessageTypes() []string {
return []string{
"/neutron.interchainqueries.MsgRegisterInterchainQuery",
"/neutron.interchaintxs.v1.MsgSubmitTx",
}
}
func (n *NeutronIndexer) IndexMessage(ctx context.Context, msg *CosmosMessage, tx *CosmosTx) error {
return nil
}
multichain/evm_indexer.go
-1301
View File
File diff suppressed because it is too large Load Diff
multichain/l2_indexer_test.go
-1102
View File
File diff suppressed because it is too large Load Diff
multichain/manager.go
-630
View File
@@ -1,630 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
// Package multichain provides a massively parallel multi-chain indexer
// capable of indexing thousands of chains concurrently using goroutines.
package multichain
import (
"context"
"fmt"
"log"
"sync"
"time"
)
// ChainType identifies the blockchain architecture
type ChainType string
const (
ChainTypeEVM ChainType = "evm"
ChainTypeSolana ChainType = "solana"
ChainTypeBitcoin ChainType = "bitcoin"
ChainTypeCosmos ChainType = "cosmos"
ChainTypeMove ChainType = "move" // Aptos, Sui
ChainTypeNear ChainType = "near"
ChainTypeTron ChainType = "tron"
ChainTypeTon ChainType = "ton"
ChainTypeFlow ChainType = "flow"
ChainTypeTezos ChainType = "tezos"
ChainTypeAlgorand ChainType = "algorand"
ChainTypeCardano ChainType = "cardano"
ChainTypeSubstrate ChainType = "substrate" // Polkadot, Kusama
ChainTypeStacks ChainType = "stacks"
ChainTypeHedera ChainType = "hedera"
ChainTypeFilecoin ChainType = "filecoin"
ChainTypeICP ChainType = "icp"
ChainTypeMultiversX ChainType = "multiversx"
// Purpose-based platform chain types. Brand-neutral so any network that
// wires these VMs into its node binary can consume the indexer without
// renaming. The single-letter chain convention (A/B/D/...) lives in the
// ID field — Type captures the VM's semantic role.
ChainTypePlatform ChainType = "platform" // P-Chain: validators, staking, subnets
ChainTypeUTXO ChainType = "utxo" // X-Chain: UTXO asset chain
ChainTypeAI ChainType = "ai" // A-Chain: AI compute / inference
ChainTypeBridge ChainType = "bridge" // B-Chain: cross-chain bridge
ChainTypeDEX ChainType = "dex" // D-Chain: orderbook DEX
ChainTypeGraph ChainType = "graph" // G-Chain: GraphQL indexing
ChainTypeIdentity ChainType = "identity" // I-Chain: decentralized identity / DIDs
ChainTypeKey ChainType = "key" // K-Chain: key management / KMS
ChainTypeMPC ChainType = "mpc" // M-Chain: CGGMP21 / FROST MPC coordination
ChainTypeOracle ChainType = "oracle" // O-Chain: oracle aggregation
ChainTypeQuantum ChainType = "quantum" // Q-Chain: PQ finality (ML-DSA, Corona)
ChainTypeRelay ChainType = "relay" // R-Chain: cross-chain message relay
ChainTypeThreshold ChainType = "threshold" // T-Chain: threshold FHE
ChainTypeZK ChainType = "zk" // Z-Chain: zero-knowledge rollup
)
// ProtocolType identifies DeFi protocol types
type ProtocolType string
const (
// DEX Protocols
ProtocolUniswapV2 ProtocolType = "uniswap_v2"
ProtocolUniswapV3 ProtocolType = "uniswap_v3"
ProtocolSushiswap ProtocolType = "sushiswap"
ProtocolCurve ProtocolType = "curve"
ProtocolBalancer ProtocolType = "balancer"
ProtocolPancakeswap ProtocolType = "pancakeswap"
ProtocolTraderJoe ProtocolType = "trader_joe"
ProtocolCamelot ProtocolType = "camelot"
Protocol1inch ProtocolType = "1inch"
ProtocolParaswap ProtocolType = "paraswap"
ProtocolKyberswap ProtocolType = "kyberswap"
ProtocolDODO ProtocolType = "dodo"
ProtocolBancor ProtocolType = "bancor"
ProtocolVelodrome ProtocolType = "velodrome"
ProtocolAerodrome ProtocolType = "aerodrome"
ProtocolMaverick ProtocolType = "maverick"
// Lending/Borrowing
ProtocolAaveV2 ProtocolType = "aave_v2"
ProtocolAaveV3 ProtocolType = "aave_v3"
ProtocolCompoundV2 ProtocolType = "compound_v2"
ProtocolCompoundV3 ProtocolType = "compound_v3"
ProtocolMakerDAO ProtocolType = "makerdao"
ProtocolMorpho ProtocolType = "morpho"
ProtocolEuler ProtocolType = "euler"
ProtocolRadiant ProtocolType = "radiant"
ProtocolBenqi ProtocolType = "benqi"
ProtocolVenus ProtocolType = "venus"
ProtocolSpark ProtocolType = "spark"
// Perpetuals/Derivatives
ProtocolGMX ProtocolType = "gmx"
ProtocolGMXV2 ProtocolType = "gmx_v2"
ProtocolGains ProtocolType = "gains"
ProtocolSynthetix ProtocolType = "synthetix"
ProtocolSynthetixV3 ProtocolType = "synthetix_v3"
ProtocolPerpetual ProtocolType = "perpetual"
ProtocolDYDX ProtocolType = "dydx"
ProtocolKwenta ProtocolType = "kwenta"
ProtocolLevel ProtocolType = "level"
ProtocolMux ProtocolType = "mux"
ProtocolVela ProtocolType = "vela"
ProtocolVertex ProtocolType = "vertex"
ProtocolHyperliquid ProtocolType = "hyperliquid"
//
ProtocolLido ProtocolType = "lido"
ProtocolRocketPool ProtocolType = "rocket_pool"
ProtocolFrax ProtocolType = "frax"
ProtocolCoinbase ProtocolType = "coinbase_staked"
ProtocolStakeWise ProtocolType = "stakewise"
ProtocolSwell ProtocolType = "swell"
ProtocolEtherFi ProtocolType = "etherfi"
ProtocolKelp ProtocolType = "kelp"
ProtocolRenzo ProtocolType = "renzo"
ProtocolPuffer ProtocolType = "puffer"
ProtocolEigenlayer ProtocolType = "eigenlayer"
// Yield/Vaults
ProtocolYearn ProtocolType = "yearn"
ProtocolConvex ProtocolType = "convex"
ProtocolBeefy ProtocolType = "beefy"
ProtocolSommelier ProtocolType = "sommelier"
ProtocolPendle ProtocolType = "pendle"
ProtocolOrigin ProtocolType = "origin"
ProtocolStakedao ProtocolType = "stakedao"
ProtocolConcentrator ProtocolType = "concentrator"
// NFT Marketplaces
ProtocolSeaport ProtocolType = "seaport"
ProtocolLooksRare ProtocolType = "looksrare"
ProtocolBlur ProtocolType = "blur"
ProtocolX2Y2 ProtocolType = "x2y2"
ProtocolRarible ProtocolType = "rarible"
ProtocolZora ProtocolType = "zora"
ProtocolFoundation ProtocolType = "foundation"
ProtocolSuperRare ProtocolType = "superrare"
ProtocolMagicEden ProtocolType = "magic_eden"
ProtocolTensor ProtocolType = "tensor"
ProtocolSudoswap ProtocolType = "sudoswap"
ProtocolNFTX ProtocolType = "nftx"
ProtocolBendDAO ProtocolType = "benddao"
// Bridges
ProtocolWormhole ProtocolType = "wormhole"
ProtocolLayerZero ProtocolType = "layerzero"
ProtocolStargate ProtocolType = "stargate"
ProtocolAxelar ProtocolType = "axelar"
ProtocolCeler ProtocolType = "celer"
ProtocolMultichain ProtocolType = "multichain"
ProtocolHop ProtocolType = "hop"
ProtocolAcross ProtocolType = "across"
ProtocolSynapse ProtocolType = "synapse"
ProtocolSocket ProtocolType = "socket"
ProtocolLifi ProtocolType = "lifi"
ProtocolDebridge ProtocolType = "debridge"
ProtocolCCTP ProtocolType = "cctp"
// Options/Structured Products
ProtocolDopex ProtocolType = "dopex"
ProtocolLyra ProtocolType = "lyra"
ProtocolPremia ProtocolType = "premia"
ProtocolRibbon ProtocolType = "ribbon"
ProtocolThetanuts ProtocolType = "thetanuts"
// CDP/Stablecoins
ProtocolLiquity ProtocolType = "liquity"
ProtocolPrisma ProtocolType = "prisma"
ProtocolGravita ProtocolType = "gravita"
ProtocolAbracadabra ProtocolType = "abracadabra"
ProtocolAngle ProtocolType = "angle"
ProtocolReflexer ProtocolType = "reflexer"
// RWA
ProtocolCentrifuge ProtocolType = "centrifuge"
ProtocolMaple ProtocolType = "maple"
ProtocolGoldfinch ProtocolType = "goldfinch"
ProtocolOndo ProtocolType = "ondo"
// Insurance
ProtocolNexusMutual ProtocolType = "nexus_mutual"
ProtocolInsurAce ProtocolType = "insurace"
// Prediction Markets
ProtocolPolymarket ProtocolType = "polymarket"
ProtocolAugur ProtocolType = "augur"
// Bitcoin Protocols
ProtocolOrdinals ProtocolType = "ordinals"
ProtocolRunes ProtocolType = "runes"
ProtocolBRC20 ProtocolType = "brc20"
ProtocolStamps ProtocolType = "stamps"
ProtocolAtomicals ProtocolType = "atomicals"
// Solana Protocols
ProtocolRaydium ProtocolType = "raydium"
ProtocolOrca ProtocolType = "orca"
ProtocolMarinade ProtocolType = "marinade"
ProtocolJito ProtocolType = "jito"
ProtocolJupiter ProtocolType = "jupiter"
ProtocolDrift ProtocolType = "drift"
ProtocolMango ProtocolType = "mango"
ProtocolPhoenix ProtocolType = "phoenix"
ProtocolMetaplex ProtocolType = "metaplex"
// Regulated digital securities (ERC-3643 / T-REX + ONCHAINID)
ProtocolERC3643 ProtocolType = "erc3643" // T-REX security tokens (IToken, IdentityRegistry, ModularCompliance, TrustedIssuersRegistry, ClaimTopicsRegistry)
ProtocolONCHAINID ProtocolType = "onchainid" // ERC-734 + ERC-735 identity contracts
// Bitcoin metaprotocols (additions)
ProtocolOPNET ProtocolType = "opnet" // OP_NET (Bitcoin L1 metaprotocol, AssemblyScript runtime, virtual chain ID 4294967299)
)
// ChainConfig holds configuration for a single chain
type ChainConfig struct {
ID string `yaml:"id"`
ChainID uint64 `yaml:"chain_id"`
Name string `yaml:"name"`
Symbol string `yaml:"symbol"`
Type ChainType `yaml:"type"`
RPC string `yaml:"rpc"`
WS string `yaml:"ws,omitempty"`
API string `yaml:"api,omitempty"`
Explorer string `yaml:"explorer,omitempty"`
Enabled bool `yaml:"enabled"`
Protocols []ProtocolConfig `yaml:"protocols,omitempty"`
StartBlock uint64 `yaml:"start_block,omitempty"`
BatchSize int `yaml:"batch_size,omitempty"`
PollInterval time.Duration `yaml:"poll_interval,omitempty"`
}
// ProtocolConfig holds configuration for a protocol on a chain
type ProtocolConfig struct {
Type ProtocolType `yaml:"type"`
Address string `yaml:"address,omitempty"`
Factory string `yaml:"factory,omitempty"`
StartBlock uint64 `yaml:"start_block,omitempty"`
Enabled bool `yaml:"enabled"`
}
// ChainIndexer is the interface for chain-specific indexers
type ChainIndexer interface {
// Chain info
ChainID() string
ChainType() ChainType
// Lifecycle
Start(ctx context.Context) error
Stop() error
IsRunning() bool
// Indexing
IndexBlock(ctx context.Context, blockNumber uint64) error
IndexBlockRange(ctx context.Context, from, to uint64) error
GetLatestBlock(ctx context.Context) (uint64, error)
GetIndexedBlock() uint64
// Stats
Stats() *IndexerStats
}
// IndexerStats holds statistics for an indexer
type IndexerStats struct {
ChainID string `json:"chainId"`
ChainName string `json:"chainName"`
IsRunning bool `json:"isRunning"`
LatestBlock uint64 `json:"latestBlock"`
IndexedBlock uint64 `json:"indexedBlock"`
BlocksBehind uint64 `json:"blocksBehind"`
BlocksProcessed uint64 `json:"blocksProcessed"`
TxsProcessed uint64 `json:"txsProcessed"`
EventsProcessed uint64 `json:"eventsProcessed"`
ErrorCount uint64 `json:"errorCount"`
LastError string `json:"lastError,omitempty"`
LastBlockTime time.Time `json:"lastBlockTime"`
StartTime time.Time `json:"startTime"`
Uptime time.Duration `json:"uptime"`
BlocksPerSecond float64 `json:"blocksPerSecond"`
}
// Manager coordinates indexing across all chains
type Manager struct {
mu sync.RWMutex
// Configuration
config *Config
// Chain indexers
indexers map[string]ChainIndexer
// Shared resources
db Database
cache Cache
search SearchEngine
// Stats
stats *ManagerStats
// Lifecycle
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
}
// Config holds the manager configuration
type Config struct {
// Database settings
DatabaseURL string `yaml:"database_url"`
MaxConnections int `yaml:"max_connections"`
// Redis settings
RedisURL string `yaml:"redis_url"`
// Search settings
ElasticsearchURL string `yaml:"elasticsearch_url"`
// Indexing settings
MaxConcurrentChains int `yaml:"max_concurrent_chains"`
DefaultBatchSize int `yaml:"default_batch_size"`
DefaultPollInterval time.Duration `yaml:"default_poll_interval"`
RetryAttempts int `yaml:"retry_attempts"`
RetryDelay time.Duration `yaml:"retry_delay"`
// Chains to index
Chains []ChainConfig `yaml:"chains"`
}
// ManagerStats holds aggregate statistics
type ManagerStats struct {
mu sync.RWMutex
TotalChains int `json:"totalChains"`
ActiveChains int `json:"activeChains"`
TotalBlocksIndexed uint64 `json:"totalBlocksIndexed"`
TotalTxsIndexed uint64 `json:"totalTxsIndexed"`
TotalEventsIndexed uint64 `json:"totalEventsIndexed"`
StartTime time.Time `json:"startTime"`
Uptime time.Duration `json:"uptime"`
ChainStats map[string]*IndexerStats `json:"chainStats"`
}
// Database interface for storage
type Database interface {
Store(ctx context.Context, table string, data interface{}) error
Query(ctx context.Context, query string, args ...interface{}) (interface{}, error)
Close() error
}
// Cache interface for caching
type Cache interface {
Get(ctx context.Context, key string) (interface{}, error)
Set(ctx context.Context, key string, value interface{}, ttl time.Duration) error
Delete(ctx context.Context, key string) error
}
// SearchEngine interface for search
type SearchEngine interface {
Index(ctx context.Context, index string, id string, data interface{}) error
Search(ctx context.Context, index string, query interface{}) (interface{}, error)
}
// NewManager creates a new multi-chain indexer manager
func NewManager(config *Config) (*Manager, error) {
ctx, cancel := context.WithCancel(context.Background())
m := &Manager{
config: config,
indexers: make(map[string]ChainIndexer),
ctx: ctx,
cancel: cancel,
stats: &ManagerStats{
StartTime: time.Now(),
ChainStats: make(map[string]*IndexerStats),
},
}
// Initialize chains
for _, chainConfig := range config.Chains {
if !chainConfig.Enabled {
continue
}
indexer, err := m.createIndexer(chainConfig)
if err != nil {
log.Printf("Warning: Failed to create indexer for %s: %v", chainConfig.ID, err)
continue
}
m.indexers[chainConfig.ID] = indexer
m.stats.TotalChains++
}
return m, nil
}
// createIndexer creates the appropriate indexer for a chain type
func (m *Manager) createIndexer(config ChainConfig) (ChainIndexer, error) {
switch config.Type {
case ChainTypeEVM:
return NewEVMIndexer(config, m.db)
case ChainTypeSolana:
return NewSolanaIndexer(config, m.db)
case ChainTypeBitcoin:
return NewBitcoinIndexer(config, m.db)
case ChainTypeCosmos:
return NewCosmosIndexer(config, m.db)
case ChainTypeMove:
return NewMoveIndexer(config, m.db)
case ChainTypeNear:
return NewNearIndexer(config, m.db)
case ChainTypeTron:
return NewTronIndexer(config, m.db)
case ChainTypeTon:
return NewTonIndexer(config, m.db)
case ChainTypeSubstrate:
return NewSubstrateIndexer(config, m.db)
// Purpose-built platform chains (platform/UTXO/AI/bridge/DEX/graph/
// identity/key/MPC/oracle/quantum/relay/threshold/ZK)
case ChainTypePlatform, ChainTypeUTXO, ChainTypeAI, ChainTypeBridge,
ChainTypeDEX, ChainTypeGraph, ChainTypeIdentity, ChainTypeKey,
ChainTypeMPC, ChainTypeOracle, ChainTypeQuantum, ChainTypeRelay,
ChainTypeThreshold, ChainTypeZK:
return NewPlatformIndexer(config), nil
default:
return nil, fmt.Errorf("unsupported chain type: %s", config.Type)
}
}
// Start starts all chain indexers concurrently
func (m *Manager) Start() error {
log.Printf("Starting multi-chain indexer with %d chains", len(m.indexers))
// Use semaphore to limit concurrent chain startups
sem := make(chan struct{}, m.config.MaxConcurrentChains)
for chainID, indexer := range m.indexers {
sem <- struct{}{} // Acquire semaphore
m.wg.Add(1)
go func(id string, idx ChainIndexer) {
defer m.wg.Done()
defer func() { <-sem }() // Release semaphore
log.Printf("Starting indexer for chain: %s", id)
if err := idx.Start(m.ctx); err != nil {
log.Printf("Error starting indexer for %s: %v", id, err)
return
}
m.mu.Lock()
m.stats.ActiveChains++
m.mu.Unlock()
// Run indexing loop
m.runIndexer(id, idx)
}(chainID, indexer)
}
// Start stats collector
go m.collectStats()
return nil
}
// runIndexer runs the indexing loop for a single chain
func (m *Manager) runIndexer(chainID string, indexer ChainIndexer) {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-m.ctx.Done():
log.Printf("Stopping indexer for chain: %s", chainID)
indexer.Stop()
return
case <-ticker.C:
// Get latest block
latest, err := indexer.GetLatestBlock(m.ctx)
if err != nil {
log.Printf("Error getting latest block for %s: %v", chainID, err)
continue
}
indexed := indexer.GetIndexedBlock()
if indexed >= latest {
continue // Already caught up
}
// Index missing blocks in batches
batchSize := uint64(100)
for from := indexed + 1; from <= latest; from += batchSize {
to := from + batchSize - 1
if to > latest {
to = latest
}
if err := indexer.IndexBlockRange(m.ctx, from, to); err != nil {
log.Printf("Error indexing blocks %d-%d for %s: %v", from, to, chainID, err)
break
}
}
}
}
}
// collectStats periodically collects statistics from all indexers
func (m *Manager) collectStats() {
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-m.ctx.Done():
return
case <-ticker.C:
m.mu.Lock()
m.stats.Uptime = time.Since(m.stats.StartTime)
var totalBlocks, totalTxs, totalEvents uint64
for id, indexer := range m.indexers {
stats := indexer.Stats()
m.stats.ChainStats[id] = stats
totalBlocks += stats.BlocksProcessed
totalTxs += stats.TxsProcessed
totalEvents += stats.EventsProcessed
}
m.stats.TotalBlocksIndexed = totalBlocks
m.stats.TotalTxsIndexed = totalTxs
m.stats.TotalEventsIndexed = totalEvents
m.mu.Unlock()
}
}
}
// Stop stops all indexers
func (m *Manager) Stop() error {
log.Println("Stopping multi-chain indexer manager")
m.cancel()
m.wg.Wait()
return nil
}
// Stats returns aggregate statistics
func (m *Manager) Stats() *ManagerStats {
m.mu.RLock()
defer m.mu.RUnlock()
// Clone stats
stats := &ManagerStats{
TotalChains: m.stats.TotalChains,
ActiveChains: m.stats.ActiveChains,
TotalBlocksIndexed: m.stats.TotalBlocksIndexed,
TotalTxsIndexed: m.stats.TotalTxsIndexed,
TotalEventsIndexed: m.stats.TotalEventsIndexed,
StartTime: m.stats.StartTime,
Uptime: m.stats.Uptime,
ChainStats: make(map[string]*IndexerStats),
}
for k, v := range m.stats.ChainStats {
stats.ChainStats[k] = v
}
return stats
}
// GetIndexer returns a specific chain indexer
func (m *Manager) GetIndexer(chainID string) (ChainIndexer, bool) {
m.mu.RLock()
defer m.mu.RUnlock()
idx, ok := m.indexers[chainID]
return idx, ok
}
// AddChain dynamically adds a new chain
func (m *Manager) AddChain(config ChainConfig) error {
m.mu.Lock()
defer m.mu.Unlock()
if _, exists := m.indexers[config.ID]; exists {
return fmt.Errorf("chain %s already exists", config.ID)
}
indexer, err := m.createIndexer(config)
if err != nil {
return err
}
m.indexers[config.ID] = indexer
m.stats.TotalChains++
// Start the indexer
m.wg.Add(1)
go func() {
defer m.wg.Done()
if err := indexer.Start(m.ctx); err != nil {
log.Printf("Error starting indexer for %s: %v", config.ID, err)
return
}
m.mu.Lock()
m.stats.ActiveChains++
m.mu.Unlock()
m.runIndexer(config.ID, indexer)
}()
return nil
}
// RemoveChain dynamically removes a chain
func (m *Manager) RemoveChain(chainID string) error {
m.mu.Lock()
defer m.mu.Unlock()
indexer, exists := m.indexers[chainID]
if !exists {
return fmt.Errorf("chain %s not found", chainID)
}
indexer.Stop()
delete(m.indexers, chainID)
delete(m.stats.ChainStats, chainID)
m.stats.TotalChains--
m.stats.ActiveChains--
return nil
}
multichain/nft_metadata.go
-696
View File
@@ -1,696 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
)
// Errors for NFT metadata operations
var (
ErrInvalidURI = errors.New("invalid token URI")
ErrUnsupportedScheme = errors.New("unsupported URI scheme")
ErrFetchFailed = errors.New("metadata fetch failed")
ErrParseFailed = errors.New("metadata parse failed")
ErrRateLimited = errors.New("rate limit exceeded")
ErrCacheMiss = errors.New("cache miss")
ErrTimeout = errors.New("request timeout")
)
// IPFS gateway URLs in preference order
var defaultIPFSGateways = []string{
"https://cloudflare-ipfs.com/ipfs/",
"https://ipfs.io/ipfs/",
"https://gateway.pinata.cloud/ipfs/",
"https://dweb.link/ipfs/",
"https://w3s.link/ipfs/",
}
// Arweave gateway URL
const defaultArweaveGateway = "https://arweave.net/"
// Trait represents a single NFT attribute/trait
type Trait struct {
TraitType string `json:"trait_type"`
Value interface{} `json:"value"`
DisplayType string `json:"display_type,omitempty"`
MaxValue interface{} `json:"max_value,omitempty"`
}
// NFTMetadata represents parsed NFT metadata (OpenSea/ERC721 format)
type NFTMetadata struct {
Name string `json:"name"`
Description string `json:"description"`
Image string `json:"image"`
ImageData string `json:"image_data,omitempty"`
ExternalURL string `json:"external_url,omitempty"`
AnimationURL string `json:"animation_url,omitempty"`
BackgroundColor string `json:"background_color,omitempty"`
Attributes []Trait `json:"attributes,omitempty"`
Properties map[string]interface{} `json:"properties,omitempty"`
// Additional fields for extended metadata
TokenID string `json:"token_id,omitempty"`
Collection string `json:"collection,omitempty"`
// Raw JSON for custom fields
Raw json.RawMessage `json:"-"`
}
// cacheEntry holds cached metadata with expiration
type cacheEntry struct {
metadata *NFTMetadata
expiresAt time.Time
}
// rateLimiter implements token bucket rate limiting
type rateLimiter struct {
mu sync.Mutex
tokens float64
maxTokens float64
refillRate float64 // tokens per second
lastRefill time.Time
}
// NFTMetadataServiceConfig holds service configuration
type NFTMetadataServiceConfig struct {
// HTTP client settings
Timeout time.Duration
MaxRetries int
RetryDelay time.Duration
MaxResponseSize int64
UserAgent string
// IPFS settings
IPFSGateways []string
ArweaveGateway string
// Cache settings
CacheTTL time.Duration
MaxCacheSize int
// Rate limiting
RequestsPerSecond float64
BurstSize int
}
// DefaultNFTMetadataServiceConfig returns default configuration
func DefaultNFTMetadataServiceConfig() *NFTMetadataServiceConfig {
return &NFTMetadataServiceConfig{
Timeout: 30 * time.Second,
MaxRetries: 3,
RetryDelay: time.Second,
MaxResponseSize: 10 * 1024 * 1024, // 10MB
UserAgent: "LuxIndexer/1.0",
IPFSGateways: defaultIPFSGateways,
ArweaveGateway: defaultArweaveGateway,
CacheTTL: time.Hour,
MaxCacheSize: 10000,
RequestsPerSecond: 10.0,
BurstSize: 20,
}
}
// NFTMetadataService fetches and parses NFT metadata from various sources
type NFTMetadataService struct {
mu sync.RWMutex
config *NFTMetadataServiceConfig
client *http.Client
cache map[string]*cacheEntry
limiter *rateLimiter
// Stats
cacheHits uint64
cacheMisses uint64
fetchCount uint64
errorCount uint64
}
// NewNFTMetadataService creates a new metadata service
func NewNFTMetadataService(config *NFTMetadataServiceConfig) *NFTMetadataService {
if config == nil {
config = DefaultNFTMetadataServiceConfig()
}
return &NFTMetadataService{
config: config,
client: &http.Client{
Timeout: config.Timeout,
Transport: &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
},
},
cache: make(map[string]*cacheEntry),
limiter: &rateLimiter{
tokens: float64(config.BurstSize),
maxTokens: float64(config.BurstSize),
refillRate: config.RequestsPerSecond,
lastRefill: time.Now(),
},
}
}
// FetchMetadata fetches NFT metadata from a token URI
// Supports: IPFS (ipfs://), Arweave (ar://), HTTP/HTTPS, data URIs
func (s *NFTMetadataService) FetchMetadata(ctx context.Context, tokenURI string) (*NFTMetadata, error) {
if tokenURI == "" {
return nil, ErrInvalidURI
}
// Check cache first
if metadata := s.getFromCache(tokenURI); metadata != nil {
return metadata, nil
}
// Rate limit check
if !s.acquireToken() {
return nil, ErrRateLimited
}
// Resolve URI to HTTP URL
httpURL, err := s.resolveURI(tokenURI)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidURI, err)
}
// Handle data URIs
if strings.HasPrefix(httpURL, "data:") {
return s.parseDataURI(httpURL)
}
// Fetch with retries
data, err := s.fetchWithRetry(ctx, httpURL)
if err != nil {
s.mu.Lock()
s.errorCount++
s.mu.Unlock()
return nil, err
}
// Parse metadata
metadata, err := s.ParseMetadata(data)
if err != nil {
s.mu.Lock()
s.errorCount++
s.mu.Unlock()
return nil, err
}
// Cache result
s.putInCache(tokenURI, metadata)
s.mu.Lock()
s.fetchCount++
s.mu.Unlock()
return metadata, nil
}
// ParseMetadata parses raw JSON data into NFTMetadata struct
func (s *NFTMetadataService) ParseMetadata(data []byte) (*NFTMetadata, error) {
if len(data) == 0 {
return nil, fmt.Errorf("%w: empty data", ErrParseFailed)
}
// Store raw JSON
metadata := &NFTMetadata{
Raw: json.RawMessage(data),
}
// Parse JSON
if err := json.Unmarshal(data, metadata); err != nil {
return nil, fmt.Errorf("%w: %v", ErrParseFailed, err)
}
// Handle alternative attribute formats
if len(metadata.Attributes) == 0 && metadata.Properties != nil {
metadata.Attributes = s.extractTraitsFromProperties(metadata.Properties)
}
// Resolve image URLs if they are IPFS/Arweave
if metadata.Image != "" {
if resolved, err := s.resolveURI(metadata.Image); err == nil {
metadata.Image = resolved
}
}
if metadata.AnimationURL != "" {
if resolved, err := s.resolveURI(metadata.AnimationURL); err == nil {
metadata.AnimationURL = resolved
}
}
return metadata, nil
}
// ExtractTraits extracts trait_type/value pairs from metadata
func (s *NFTMetadataService) ExtractTraits(metadata *NFTMetadata) []Trait {
if metadata == nil {
return nil
}
// Return existing attributes if present
if len(metadata.Attributes) > 0 {
traits := make([]Trait, len(metadata.Attributes))
copy(traits, metadata.Attributes)
return traits
}
// Try to extract from properties
if metadata.Properties != nil {
return s.extractTraitsFromProperties(metadata.Properties)
}
return nil
}
// extractTraitsFromProperties converts properties map to traits
func (s *NFTMetadataService) extractTraitsFromProperties(props map[string]interface{}) []Trait {
traits := make([]Trait, 0, len(props))
for key, value := range props {
trait := Trait{
TraitType: key,
Value: value,
}
// Handle nested property format
if m, ok := value.(map[string]interface{}); ok {
if v, exists := m["value"]; exists {
trait.Value = v
}
if dt, exists := m["display_type"]; exists {
if dtStr, ok := dt.(string); ok {
trait.DisplayType = dtStr
}
}
if mv, exists := m["max_value"]; exists {
trait.MaxValue = mv
}
}
traits = append(traits, trait)
}
return traits
}
// resolveURI converts various URI schemes to HTTP URLs
func (s *NFTMetadataService) resolveURI(uri string) (string, error) {
uri = strings.TrimSpace(uri)
// Handle data URIs
if strings.HasPrefix(uri, "data:") {
return uri, nil
}
// Handle IPFS URIs
if strings.HasPrefix(uri, "ipfs://") {
cid := strings.TrimPrefix(uri, "ipfs://")
// Remove any leading slashes
cid = strings.TrimPrefix(cid, "/")
// Use first gateway
if len(s.config.IPFSGateways) > 0 {
return s.config.IPFSGateways[0] + cid, nil
}
return defaultIPFSGateways[0] + cid, nil
}
// Handle Arweave URIs
if strings.HasPrefix(uri, "ar://") {
txID := strings.TrimPrefix(uri, "ar://")
gateway := s.config.ArweaveGateway
if gateway == "" {
gateway = defaultArweaveGateway
}
return gateway + txID, nil
}
// Handle HTTP/HTTPS
if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") {
// Convert IPFS gateway URLs to use preferred gateway
for _, gw := range append(defaultIPFSGateways, s.config.IPFSGateways...) {
if strings.HasPrefix(uri, gw) {
cid := strings.TrimPrefix(uri, gw)
if len(s.config.IPFSGateways) > 0 {
return s.config.IPFSGateways[0] + cid, nil
}
return uri, nil
}
}
return uri, nil
}
// Handle bare CIDs (assume IPFS)
if isIPFSCID(uri) {
if len(s.config.IPFSGateways) > 0 {
return s.config.IPFSGateways[0] + uri, nil
}
return defaultIPFSGateways[0] + uri, nil
}
return "", fmt.Errorf("unsupported URI scheme: %s", uri)
}
// parseDataURI parses a data: URI and extracts metadata
func (s *NFTMetadataService) parseDataURI(dataURI string) (*NFTMetadata, error) {
// Format: data:[<mediatype>][;base64],<data>
if !strings.HasPrefix(dataURI, "data:") {
return nil, ErrInvalidURI
}
// Find comma separator
commaIdx := strings.Index(dataURI, ",")
if commaIdx < 0 {
return nil, fmt.Errorf("%w: no data in data URI", ErrInvalidURI)
}
header := dataURI[5:commaIdx]
data := dataURI[commaIdx+1:]
// Check if base64 encoded
isBase64 := strings.Contains(header, ";base64")
var jsonData []byte
if isBase64 {
// Simple base64 decode (standard library)
decoded, err := decodeBase64(data)
if err != nil {
return nil, fmt.Errorf("%w: base64 decode failed: %v", ErrParseFailed, err)
}
jsonData = decoded
} else {
// URL decode
decoded, err := url.QueryUnescape(data)
if err != nil {
return nil, fmt.Errorf("%w: URL decode failed: %v", ErrParseFailed, err)
}
jsonData = []byte(decoded)
}
return s.ParseMetadata(jsonData)
}
// fetchWithRetry fetches URL with retry logic
func (s *NFTMetadataService) fetchWithRetry(ctx context.Context, httpURL string) ([]byte, error) {
var lastErr error
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
if attempt > 0 {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(s.config.RetryDelay * time.Duration(attempt)):
}
}
data, err := s.fetch(ctx, httpURL)
if err == nil {
return data, nil
}
lastErr = err
// Don't retry on certain errors
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
break
}
}
// Try alternative gateways for IPFS
if strings.Contains(httpURL, "/ipfs/") {
data, err := s.fetchFromAlternativeGateways(ctx, httpURL)
if err == nil {
return data, nil
}
lastErr = err
}
return nil, fmt.Errorf("%w: %v after %d attempts", ErrFetchFailed, lastErr, s.config.MaxRetries+1)
}
// fetchFromAlternativeGateways tries alternative IPFS gateways
func (s *NFTMetadataService) fetchFromAlternativeGateways(ctx context.Context, httpURL string) ([]byte, error) {
// Extract CID from URL
var cid string
for _, gw := range append(defaultIPFSGateways, s.config.IPFSGateways...) {
if strings.Contains(httpURL, gw) {
cid = strings.TrimPrefix(httpURL, gw)
break
}
}
if cid == "" {
// Try to extract from generic /ipfs/ path
idx := strings.Index(httpURL, "/ipfs/")
if idx >= 0 {
cid = httpURL[idx+6:]
}
}
if cid == "" {
return nil, fmt.Errorf("could not extract CID from URL")
}
// Try each gateway
gateways := s.config.IPFSGateways
if len(gateways) == 0 {
gateways = defaultIPFSGateways
}
var lastErr error
for i, gw := range gateways {
if i == 0 {
continue // Skip first gateway (already tried)
}
altURL := gw + cid
data, err := s.fetch(ctx, altURL)
if err == nil {
return data, nil
}
lastErr = err
}
return nil, lastErr
}
// fetch performs a single HTTP GET request
func (s *NFTMetadataService) fetch(ctx context.Context, httpURL string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, httpURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", s.config.UserAgent)
req.Header.Set("Accept", "application/json")
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
}
// Limit response size
reader := io.LimitReader(resp.Body, s.config.MaxResponseSize)
data, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
return data, nil
}
// Cache operations
func (s *NFTMetadataService) getFromCache(key string) *NFTMetadata {
s.mu.RLock()
entry, ok := s.cache[key]
s.mu.RUnlock()
if !ok {
s.mu.Lock()
s.cacheMisses++
s.mu.Unlock()
return nil
}
if time.Now().After(entry.expiresAt) {
s.mu.Lock()
delete(s.cache, key)
s.cacheMisses++
s.mu.Unlock()
return nil
}
s.mu.Lock()
s.cacheHits++
s.mu.Unlock()
return entry.metadata
}
func (s *NFTMetadataService) putInCache(key string, metadata *NFTMetadata) {
s.mu.Lock()
defer s.mu.Unlock()
// Evict oldest entries if cache is full
if len(s.cache) >= s.config.MaxCacheSize {
s.evictOldest()
}
s.cache[key] = &cacheEntry{
metadata: metadata,
expiresAt: time.Now().Add(s.config.CacheTTL),
}
}
func (s *NFTMetadataService) evictOldest() {
// Simple eviction: remove expired entries first
now := time.Now()
for key, entry := range s.cache {
if now.After(entry.expiresAt) {
delete(s.cache, key)
}
}
// If still over limit, remove 10% of entries
if len(s.cache) >= s.config.MaxCacheSize {
count := 0
toRemove := s.config.MaxCacheSize / 10
for key := range s.cache {
delete(s.cache, key)
count++
if count >= toRemove {
break
}
}
}
}
// ClearCache clears the metadata cache
func (s *NFTMetadataService) ClearCache() {
s.mu.Lock()
defer s.mu.Unlock()
s.cache = make(map[string]*cacheEntry)
}
// Rate limiting
func (s *NFTMetadataService) acquireToken() bool {
s.limiter.mu.Lock()
defer s.limiter.mu.Unlock()
now := time.Now()
elapsed := now.Sub(s.limiter.lastRefill).Seconds()
// Refill tokens
s.limiter.tokens += elapsed * s.limiter.refillRate
if s.limiter.tokens > s.limiter.maxTokens {
s.limiter.tokens = s.limiter.maxTokens
}
s.limiter.lastRefill = now
// Try to consume a token
if s.limiter.tokens >= 1.0 {
s.limiter.tokens--
return true
}
return false
}
// Stats returns service statistics
func (s *NFTMetadataService) Stats() map[string]uint64 {
s.mu.RLock()
defer s.mu.RUnlock()
return map[string]uint64{
"cache_hits": s.cacheHits,
"cache_misses": s.cacheMisses,
"cache_size": uint64(len(s.cache)),
"fetch_count": s.fetchCount,
"error_count": s.errorCount,
}
}
// Helper functions
// isIPFSCID checks if a string looks like an IPFS CID
func isIPFSCID(s string) bool {
// CIDv0 starts with Qm and is 46 chars
if strings.HasPrefix(s, "Qm") && len(s) >= 46 {
return true
}
// CIDv1 starts with bafy (base32) or b (other bases)
if strings.HasPrefix(s, "bafy") || strings.HasPrefix(s, "bafk") {
return true
}
return false
}
// decodeBase64 decodes base64 string (standard library implementation)
func decodeBase64(s string) ([]byte, error) {
// Base64 alphabet
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
// Build reverse lookup
decodeMap := make(map[byte]int)
for i := 0; i < len(alphabet); i++ {
decodeMap[alphabet[i]] = i
}
decodeMap['='] = 0
// Remove whitespace
s = strings.Map(func(r rune) rune {
if r == ' ' || r == '\n' || r == '\r' || r == '\t' {
return -1
}
return r
}, s)
// Pad if necessary
for len(s)%4 != 0 {
s += "="
}
result := make([]byte, 0, len(s)*3/4)
for i := 0; i < len(s); i += 4 {
a, ok1 := decodeMap[s[i]]
b, ok2 := decodeMap[s[i+1]]
c, ok3 := decodeMap[s[i+2]]
d, ok4 := decodeMap[s[i+3]]
if !ok1 || !ok2 || !ok3 || !ok4 {
return nil, fmt.Errorf("invalid base64 character")
}
triple := (a << 18) | (b << 12) | (c << 6) | d
result = append(result, byte(triple>>16))
if s[i+2] != '=' {
result = append(result, byte(triple>>8))
}
if s[i+3] != '=' {
result = append(result, byte(triple))
}
}
return result, nil
}
multichain/nft_metadata_test.go
-487
View File
@@ -1,487 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestNFTMetadataService_ParseMetadata(t *testing.T) {
service := NewNFTMetadataService(nil)
tests := []struct {
name string
input string
want *NFTMetadata
wantErr bool
}{
{
name: "basic OpenSea format",
input: `{
"name": "Cool NFT #1",
"description": "A very cool NFT",
"image": "https://example.com/image.png",
"external_url": "https://example.com/nft/1",
"attributes": [
{"trait_type": "Color", "value": "Blue"},
{"trait_type": "Size", "value": 42, "display_type": "number"}
]
}`,
want: &NFTMetadata{
Name: "Cool NFT #1",
Description: "A very cool NFT",
Image: "https://example.com/image.png",
ExternalURL: "https://example.com/nft/1",
Attributes: []Trait{
{TraitType: "Color", Value: "Blue"},
{TraitType: "Size", Value: float64(42), DisplayType: "number"},
},
},
wantErr: false,
},
{
name: "IPFS image URL",
input: `{
"name": "IPFS NFT",
"image": "ipfs://QmTest1234567890abcdefghijklmnopqrstuvwxyz"
}`,
want: &NFTMetadata{
Name: "IPFS NFT",
Image: "https://cloudflare-ipfs.com/ipfs/QmTest1234567890abcdefghijklmnopqrstuvwxyz",
},
wantErr: false,
},
{
name: "with animation_url",
input: `{
"name": "Animated NFT",
"image": "https://example.com/poster.png",
"animation_url": "https://example.com/video.mp4"
}`,
want: &NFTMetadata{
Name: "Animated NFT",
Image: "https://example.com/poster.png",
AnimationURL: "https://example.com/video.mp4",
},
wantErr: false,
},
{
name: "empty data",
input: "",
wantErr: true,
},
{
name: "invalid JSON",
input: "{not valid json",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := service.ParseMetadata([]byte(tt.input))
if (err != nil) != tt.wantErr {
t.Errorf("ParseMetadata() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.wantErr {
return
}
if got.Name != tt.want.Name {
t.Errorf("Name = %q, want %q", got.Name, tt.want.Name)
}
if got.Description != tt.want.Description {
t.Errorf("Description = %q, want %q", got.Description, tt.want.Description)
}
if got.Image != tt.want.Image {
t.Errorf("Image = %q, want %q", got.Image, tt.want.Image)
}
if got.ExternalURL != tt.want.ExternalURL {
t.Errorf("ExternalURL = %q, want %q", got.ExternalURL, tt.want.ExternalURL)
}
if got.AnimationURL != tt.want.AnimationURL {
t.Errorf("AnimationURL = %q, want %q", got.AnimationURL, tt.want.AnimationURL)
}
})
}
}
func TestNFTMetadataService_ExtractTraits(t *testing.T) {
service := NewNFTMetadataService(nil)
tests := []struct {
name string
metadata *NFTMetadata
wantLen int
}{
{
name: "with attributes",
metadata: &NFTMetadata{
Attributes: []Trait{
{TraitType: "Color", Value: "Red"},
{TraitType: "Size", Value: 10},
},
},
wantLen: 2,
},
{
name: "with properties",
metadata: &NFTMetadata{
Properties: map[string]interface{}{
"rarity": "legendary",
"power": 100,
},
},
wantLen: 2,
},
{
name: "nil metadata",
metadata: nil,
wantLen: 0,
},
{
name: "empty metadata",
metadata: &NFTMetadata{},
wantLen: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
traits := service.ExtractTraits(tt.metadata)
if len(traits) != tt.wantLen {
t.Errorf("ExtractTraits() returned %d traits, want %d", len(traits), tt.wantLen)
}
})
}
}
func TestNFTMetadataService_ResolveURI(t *testing.T) {
service := NewNFTMetadataService(nil)
tests := []struct {
name string
uri string
wantErr bool
want string
}{
{
name: "IPFS URI",
uri: "ipfs://QmTest1234567890abcdefghijklmnopqrstuvwxyz",
wantErr: false,
want: "https://cloudflare-ipfs.com/ipfs/QmTest1234567890abcdefghijklmnopqrstuvwxyz",
},
{
name: "IPFS URI with leading slash",
uri: "ipfs:///QmTest1234567890abcdefghijklmnopqrstuvwxyz",
wantErr: false,
want: "https://cloudflare-ipfs.com/ipfs/QmTest1234567890abcdefghijklmnopqrstuvwxyz",
},
{
name: "Arweave URI",
uri: "ar://abc123def456",
wantErr: false,
want: "https://arweave.net/abc123def456",
},
{
name: "HTTP URL",
uri: "https://example.com/metadata.json",
wantErr: false,
want: "https://example.com/metadata.json",
},
{
name: "data URI",
uri: "data:application/json,{\"name\":\"test\"}",
wantErr: false,
want: "data:application/json,{\"name\":\"test\"}",
},
{
name: "bare CIDv0",
uri: "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
wantErr: false,
want: "https://cloudflare-ipfs.com/ipfs/QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
},
{
name: "bare CIDv1",
uri: "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi",
wantErr: false,
want: "https://cloudflare-ipfs.com/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := service.resolveURI(tt.uri)
if (err != nil) != tt.wantErr {
t.Errorf("resolveURI() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("resolveURI() = %q, want %q", got, tt.want)
}
})
}
}
func TestNFTMetadataService_FetchMetadata(t *testing.T) {
// Create test server
metadata := NFTMetadata{
Name: "Test NFT",
Description: "Test description",
Image: "https://example.com/image.png",
Attributes: []Trait{
{TraitType: "Rarity", Value: "Common"},
},
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(metadata)
}))
defer server.Close()
config := DefaultNFTMetadataServiceConfig()
config.Timeout = 5 * time.Second
service := NewNFTMetadataService(config)
ctx := context.Background()
got, err := service.FetchMetadata(ctx, server.URL)
if err != nil {
t.Fatalf("FetchMetadata() error = %v", err)
}
if got.Name != metadata.Name {
t.Errorf("Name = %q, want %q", got.Name, metadata.Name)
}
if got.Description != metadata.Description {
t.Errorf("Description = %q, want %q", got.Description, metadata.Description)
}
if len(got.Attributes) != len(metadata.Attributes) {
t.Errorf("Attributes count = %d, want %d", len(got.Attributes), len(metadata.Attributes))
}
}
func TestNFTMetadataService_Cache(t *testing.T) {
callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"name": "Cached NFT"}`))
}))
defer server.Close()
config := DefaultNFTMetadataServiceConfig()
config.CacheTTL = time.Hour
service := NewNFTMetadataService(config)
ctx := context.Background()
// First fetch
_, err := service.FetchMetadata(ctx, server.URL)
if err != nil {
t.Fatalf("First FetchMetadata() error = %v", err)
}
if callCount != 1 {
t.Errorf("Expected 1 server call, got %d", callCount)
}
// Second fetch (should use cache)
_, err = service.FetchMetadata(ctx, server.URL)
if err != nil {
t.Fatalf("Second FetchMetadata() error = %v", err)
}
if callCount != 1 {
t.Errorf("Expected 1 server call (cached), got %d", callCount)
}
// Check stats
stats := service.Stats()
if stats["cache_hits"] != 1 {
t.Errorf("Expected 1 cache hit, got %d", stats["cache_hits"])
}
if stats["cache_misses"] != 1 {
t.Errorf("Expected 1 cache miss, got %d", stats["cache_misses"])
}
}
func TestNFTMetadataService_RateLimit(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"name": "Rate Limited NFT"}`))
}))
defer server.Close()
config := DefaultNFTMetadataServiceConfig()
config.RequestsPerSecond = 1.0
config.BurstSize = 2
config.CacheTTL = 0 // Disable cache
service := NewNFTMetadataService(config)
ctx := context.Background()
// Exhaust burst capacity
for i := 0; i < 2; i++ {
// Clear cache between requests
service.ClearCache()
_, err := service.FetchMetadata(ctx, server.URL+"?id="+string(rune('0'+i)))
if err != nil {
t.Fatalf("Request %d failed: %v", i, err)
}
}
// Next request should be rate limited
service.ClearCache()
_, err := service.FetchMetadata(ctx, server.URL+"?id=2")
if err == nil {
t.Error("Expected rate limit error, got nil")
}
}
func TestNFTMetadataService_DataURI(t *testing.T) {
service := NewNFTMetadataService(nil)
// Test URL-encoded data URI
dataURI := "data:application/json,%7B%22name%22%3A%22Data%20NFT%22%7D"
got, err := service.parseDataURI(dataURI)
if err != nil {
t.Fatalf("parseDataURI() error = %v", err)
}
if got.Name != "Data NFT" {
t.Errorf("Name = %q, want %q", got.Name, "Data NFT")
}
// Test base64 data URI
// {"name":"Base64 NFT"} in base64
base64URI := "data:application/json;base64,eyJuYW1lIjoiQmFzZTY0IE5GVCJ9"
got, err = service.parseDataURI(base64URI)
if err != nil {
t.Fatalf("parseDataURI() base64 error = %v", err)
}
if got.Name != "Base64 NFT" {
t.Errorf("Name = %q, want %q", got.Name, "Base64 NFT")
}
}
func TestDecodeBase64(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{
name: "simple",
input: "SGVsbG8gV29ybGQ=",
want: "Hello World",
wantErr: false,
},
{
name: "no padding",
input: "SGVsbG8",
want: "Hello",
wantErr: false,
},
{
name: "with whitespace",
input: "SGVs\nbG8g\nV29y\nbGQ=",
want: "Hello World",
wantErr: false,
},
{
name: "JSON",
input: "eyJuYW1lIjoidGVzdCJ9",
want: `{"name":"test"}`,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := decodeBase64(tt.input)
if (err != nil) != tt.wantErr {
t.Errorf("decodeBase64() error = %v, wantErr %v", err, tt.wantErr)
return
}
if string(got) != tt.want {
t.Errorf("decodeBase64() = %q, want %q", string(got), tt.want)
}
})
}
}
func TestTrait_JSON(t *testing.T) {
// Test trait marshaling
trait := Trait{
TraitType: "Power",
Value: 100,
DisplayType: "number",
MaxValue: 1000,
}
data, err := json.Marshal(trait)
if err != nil {
t.Fatalf("Marshal error: %v", err)
}
var got Trait
if err := json.Unmarshal(data, &got); err != nil {
t.Fatalf("Unmarshal error: %v", err)
}
if got.TraitType != trait.TraitType {
t.Errorf("TraitType = %q, want %q", got.TraitType, trait.TraitType)
}
if got.DisplayType != trait.DisplayType {
t.Errorf("DisplayType = %q, want %q", got.DisplayType, trait.DisplayType)
}
}
func TestDefaultConfig(t *testing.T) {
config := DefaultNFTMetadataServiceConfig()
if config.Timeout != 30*time.Second {
t.Errorf("Timeout = %v, want %v", config.Timeout, 30*time.Second)
}
if config.MaxRetries != 3 {
t.Errorf("MaxRetries = %d, want %d", config.MaxRetries, 3)
}
if len(config.IPFSGateways) == 0 {
t.Error("IPFSGateways should not be empty")
}
if config.ArweaveGateway == "" {
t.Error("ArweaveGateway should not be empty")
}
if config.CacheTTL != time.Hour {
t.Errorf("CacheTTL = %v, want %v", config.CacheTTL, time.Hour)
}
}
func TestIsIPFSCID(t *testing.T) {
tests := []struct {
input string
want bool
}{
{"QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG", true},
{"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", true},
{"bafkreigaknpexyvxt76zgkitavbwx6ejgfheup5oybpm77f3pxzrvwpfdi", true},
{"https://example.com/image.png", false},
{"random string", false},
{"Qm", false}, // Too short
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
got := isIPFSCID(tt.input)
if got != tt.want {
t.Errorf("isIPFSCID(%q) = %v, want %v", tt.input, got, tt.want)
}
})
}
}
multichain/other_indexers.go
-601
View File
@@ -1,601 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"sync"
"sync/atomic"
"time"
)
// =============================================================================
// Move Indexer (Aptos, Sui)
// =============================================================================
// MoveIndexer indexes Move-based chains (Aptos, Sui)
type MoveIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
variant string // "aptos" or "sui"
running int32
indexedBlock uint64
latestBlock uint64
stats *IndexerStats
ctx context.Context
cancel context.CancelFunc
}
// NewMoveIndexer creates a new Move chain indexer
func NewMoveIndexer(config ChainConfig, db Database) (*MoveIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
variant := "aptos"
if config.ID == "sui" || config.ID == "sui_mainnet" {
variant = "sui"
}
return &MoveIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 30 * time.Second},
variant: variant,
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}, nil
}
func (m *MoveIndexer) ChainID() string { return m.config.ID }
func (m *MoveIndexer) ChainType() ChainType { return ChainTypeMove }
func (m *MoveIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&m.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
m.stats.IsRunning = true
return nil
}
func (m *MoveIndexer) Stop() error {
atomic.StoreInt32(&m.running, 0)
m.cancel()
m.stats.IsRunning = false
return nil
}
func (m *MoveIndexer) IsRunning() bool {
return atomic.LoadInt32(&m.running) == 1
}
func (m *MoveIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
var height uint64
var err error
if m.variant == "aptos" {
height, err = m.getAptosLatestBlock(ctx)
} else {
height, err = m.getSuiLatestCheckpoint(ctx)
}
if err == nil {
m.mu.Lock()
m.latestBlock = height
m.mu.Unlock()
}
return height, err
}
func (m *MoveIndexer) getAptosLatestBlock(ctx context.Context) (uint64, error) {
url := fmt.Sprintf("%s/v1", m.config.RPC)
resp, err := m.httpGet(ctx, url)
if err != nil {
return 0, err
}
var result struct {
BlockHeight string `json:"block_height"`
}
if err := json.Unmarshal(resp, &result); err != nil {
return 0, err
}
var height uint64
fmt.Sscanf(result.BlockHeight, "%d", &height)
return height, nil
}
func (m *MoveIndexer) getSuiLatestCheckpoint(ctx context.Context) (uint64, error) {
body := map[string]interface{}{
"jsonrpc": "2.0",
"id": 1,
"method": "sui_getLatestCheckpointSequenceNumber",
"params": []interface{}{},
}
data, _ := json.Marshal(body)
req, _ := http.NewRequestWithContext(ctx, "POST", m.config.RPC, nil)
req.Header.Set("Content-Type", "application/json")
req.Body = io.NopCloser(jsonReader(data))
resp, err := m.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
var result struct {
Result string `json:"result"`
}
json.NewDecoder(resp.Body).Decode(&result)
var checkpoint uint64
fmt.Sscanf(result.Result, "%d", &checkpoint)
return checkpoint, nil
}
func (m *MoveIndexer) GetIndexedBlock() uint64 {
m.mu.RLock()
defer m.mu.RUnlock()
return m.indexedBlock
}
func (m *MoveIndexer) IndexBlock(ctx context.Context, height uint64) error {
// Implement Aptos/Sui block indexing
m.mu.Lock()
m.indexedBlock = height
m.stats.BlocksProcessed++
m.stats.LastBlockTime = time.Now()
m.mu.Unlock()
return nil
}
func (m *MoveIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for height := from; height <= to; height++ {
if err := m.IndexBlock(ctx, height); err != nil {
m.stats.ErrorCount++
continue
}
}
return nil
}
func (m *MoveIndexer) Stats() *IndexerStats {
m.mu.RLock()
defer m.mu.RUnlock()
return &IndexerStats{
ChainID: m.stats.ChainID,
ChainName: m.stats.ChainName,
IsRunning: m.stats.IsRunning,
LatestBlock: m.latestBlock,
IndexedBlock: m.indexedBlock,
BlocksBehind: m.latestBlock - m.indexedBlock,
BlocksProcessed: m.stats.BlocksProcessed,
TxsProcessed: m.stats.TxsProcessed,
EventsProcessed: m.stats.EventsProcessed,
ErrorCount: m.stats.ErrorCount,
LastError: m.stats.LastError,
LastBlockTime: m.stats.LastBlockTime,
StartTime: m.stats.StartTime,
Uptime: time.Since(m.stats.StartTime),
}
}
func (m *MoveIndexer) httpGet(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, err
}
resp, err := m.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// =============================================================================
// NEAR Indexer
// =============================================================================
type NearIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
running int32
indexedBlock uint64
latestBlock uint64
stats *IndexerStats
ctx context.Context
cancel context.CancelFunc
}
func NewNearIndexer(config ChainConfig, db Database) (*NearIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
return &NearIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 30 * time.Second},
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}, nil
}
func (n *NearIndexer) ChainID() string { return n.config.ID }
func (n *NearIndexer) ChainType() ChainType { return ChainTypeNear }
func (n *NearIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&n.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
n.stats.IsRunning = true
return nil
}
func (n *NearIndexer) Stop() error {
atomic.StoreInt32(&n.running, 0)
n.cancel()
n.stats.IsRunning = false
return nil
}
func (n *NearIndexer) IsRunning() bool { return atomic.LoadInt32(&n.running) == 1 }
func (n *NearIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
body := map[string]interface{}{
"jsonrpc": "2.0",
"id": "dontcare",
"method": "status",
"params": []interface{}{},
}
data, _ := json.Marshal(body)
req, _ := http.NewRequestWithContext(ctx, "POST", n.config.RPC, nil)
req.Header.Set("Content-Type", "application/json")
req.Body = io.NopCloser(jsonReader(data))
resp, err := n.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
var result struct {
Result struct {
SyncInfo struct {
LatestBlockHeight uint64 `json:"latest_block_height"`
} `json:"sync_info"`
} `json:"result"`
}
json.NewDecoder(resp.Body).Decode(&result)
height := result.Result.SyncInfo.LatestBlockHeight
n.mu.Lock()
n.latestBlock = height
n.mu.Unlock()
return height, nil
}
func (n *NearIndexer) GetIndexedBlock() uint64 {
n.mu.RLock()
defer n.mu.RUnlock()
return n.indexedBlock
}
func (n *NearIndexer) IndexBlock(ctx context.Context, height uint64) error {
n.mu.Lock()
n.indexedBlock = height
n.stats.BlocksProcessed++
n.stats.LastBlockTime = time.Now()
n.mu.Unlock()
return nil
}
func (n *NearIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for height := from; height <= to; height++ {
if err := n.IndexBlock(ctx, height); err != nil {
continue
}
}
return nil
}
func (n *NearIndexer) Stats() *IndexerStats {
n.mu.RLock()
defer n.mu.RUnlock()
return &IndexerStats{
ChainID: n.stats.ChainID,
ChainName: n.stats.ChainName,
IsRunning: n.stats.IsRunning,
LatestBlock: n.latestBlock,
IndexedBlock: n.indexedBlock,
BlocksBehind: n.latestBlock - n.indexedBlock,
BlocksProcessed: n.stats.BlocksProcessed,
TxsProcessed: n.stats.TxsProcessed,
StartTime: n.stats.StartTime,
Uptime: time.Since(n.stats.StartTime),
}
}
// =============================================================================
// Tron Indexer
// =============================================================================
type TronIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
running int32
indexedBlock uint64
latestBlock uint64
stats *IndexerStats
ctx context.Context
cancel context.CancelFunc
}
func NewTronIndexer(config ChainConfig, db Database) (*TronIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
return &TronIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 30 * time.Second},
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}, nil
}
func (t *TronIndexer) ChainID() string { return t.config.ID }
func (t *TronIndexer) ChainType() ChainType { return ChainTypeTron }
func (t *TronIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&t.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
t.stats.IsRunning = true
return nil
}
func (t *TronIndexer) Stop() error {
atomic.StoreInt32(&t.running, 0)
t.cancel()
t.stats.IsRunning = false
return nil
}
func (t *TronIndexer) IsRunning() bool { return atomic.LoadInt32(&t.running) == 1 }
func (t *TronIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
url := fmt.Sprintf("%s/wallet/getnowblock", t.config.RPC)
req, _ := http.NewRequestWithContext(ctx, "POST", url, nil)
req.Header.Set("Content-Type", "application/json")
resp, err := t.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
var result struct {
BlockHeader struct {
RawData struct {
Number uint64 `json:"number"`
} `json:"raw_data"`
} `json:"block_header"`
}
json.NewDecoder(resp.Body).Decode(&result)
height := result.BlockHeader.RawData.Number
t.mu.Lock()
t.latestBlock = height
t.mu.Unlock()
return height, nil
}
func (t *TronIndexer) GetIndexedBlock() uint64 {
t.mu.RLock()
defer t.mu.RUnlock()
return t.indexedBlock
}
func (t *TronIndexer) IndexBlock(ctx context.Context, height uint64) error {
t.mu.Lock()
t.indexedBlock = height
t.stats.BlocksProcessed++
t.stats.LastBlockTime = time.Now()
t.mu.Unlock()
return nil
}
func (t *TronIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for height := from; height <= to; height++ {
if err := t.IndexBlock(ctx, height); err != nil {
continue
}
}
return nil
}
func (t *TronIndexer) Stats() *IndexerStats {
t.mu.RLock()
defer t.mu.RUnlock()
return &IndexerStats{
ChainID: t.stats.ChainID,
ChainName: t.stats.ChainName,
IsRunning: t.stats.IsRunning,
LatestBlock: t.latestBlock,
IndexedBlock: t.indexedBlock,
BlocksBehind: t.latestBlock - t.indexedBlock,
BlocksProcessed: t.stats.BlocksProcessed,
TxsProcessed: t.stats.TxsProcessed,
StartTime: t.stats.StartTime,
Uptime: time.Since(t.stats.StartTime),
}
}
// =============================================================================
// TON Indexer - See ton_indexer.go for full implementation
// =============================================================================
// =============================================================================
// Substrate Indexer (Polkadot, Kusama)
// =============================================================================
type SubstrateIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
running int32
indexedBlock uint64
latestBlock uint64
stats *IndexerStats
ctx context.Context
cancel context.CancelFunc
}
func NewSubstrateIndexer(config ChainConfig, db Database) (*SubstrateIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
return &SubstrateIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 30 * time.Second},
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}, nil
}
func (s *SubstrateIndexer) ChainID() string { return s.config.ID }
func (s *SubstrateIndexer) ChainType() ChainType { return ChainTypeSubstrate }
func (s *SubstrateIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&s.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
s.stats.IsRunning = true
return nil
}
func (s *SubstrateIndexer) Stop() error {
atomic.StoreInt32(&s.running, 0)
s.cancel()
s.stats.IsRunning = false
return nil
}
func (s *SubstrateIndexer) IsRunning() bool { return atomic.LoadInt32(&s.running) == 1 }
func (s *SubstrateIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
body := map[string]interface{}{
"jsonrpc": "2.0",
"id": 1,
"method": "chain_getHeader",
"params": []interface{}{},
}
data, _ := json.Marshal(body)
req, _ := http.NewRequestWithContext(ctx, "POST", s.config.RPC, nil)
req.Header.Set("Content-Type", "application/json")
req.Body = io.NopCloser(jsonReader(data))
resp, err := s.client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
var result struct {
Result struct {
Number string `json:"number"`
} `json:"result"`
}
json.NewDecoder(resp.Body).Decode(&result)
var height uint64
fmt.Sscanf(result.Result.Number, "0x%x", &height)
s.mu.Lock()
s.latestBlock = height
s.mu.Unlock()
return height, nil
}
func (s *SubstrateIndexer) GetIndexedBlock() uint64 {
s.mu.RLock()
defer s.mu.RUnlock()
return s.indexedBlock
}
func (s *SubstrateIndexer) IndexBlock(ctx context.Context, height uint64) error {
s.mu.Lock()
s.indexedBlock = height
s.stats.BlocksProcessed++
s.stats.LastBlockTime = time.Now()
s.mu.Unlock()
return nil
}
func (s *SubstrateIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for height := from; height <= to; height++ {
if err := s.IndexBlock(ctx, height); err != nil {
continue
}
}
return nil
}
func (s *SubstrateIndexer) Stats() *IndexerStats {
s.mu.RLock()
defer s.mu.RUnlock()
return &IndexerStats{
ChainID: s.stats.ChainID,
ChainName: s.stats.ChainName,
IsRunning: s.stats.IsRunning,
LatestBlock: s.latestBlock,
IndexedBlock: s.indexedBlock,
BlocksBehind: s.latestBlock - s.indexedBlock,
BlocksProcessed: s.stats.BlocksProcessed,
TxsProcessed: s.stats.TxsProcessed,
StartTime: s.stats.StartTime,
Uptime: time.Since(s.stats.StartTime),
}
}
multichain/platform_indexer.go
-570
View File
@@ -1,570 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"bytes"
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"sync"
"time"
"github.com/gorilla/websocket"
)
// PlatformIndexer indexes platform chains (A, B, T, Z, G, I, K, D)
type PlatformIndexer struct {
config ChainConfig
client *http.Client
latestHeight uint64
indexedBlock uint64
isRunning bool
stopChan chan struct{}
mu sync.RWMutex
stats IndexerStats
}
// PlatformBlock represents a block from a platform chain
type PlatformBlock struct {
Height uint64 `json:"height"`
Hash string `json:"hash"`
ParentID string `json:"parentID"`
Timestamp time.Time `json:"timestamp"`
Txs json.RawMessage `json:"txs"`
}
// NewPlatformIndexer creates a new indexer for platform chains
func NewPlatformIndexer(config ChainConfig) *PlatformIndexer {
return &PlatformIndexer{
config: config,
client: &http.Client{
Timeout: 30 * time.Second,
},
stopChan: make(chan struct{}),
}
}
// ChainID returns the chain identifier
func (idx *PlatformIndexer) ChainID() string {
return idx.config.ID
}
// ChainType returns the chain type
func (idx *PlatformIndexer) ChainType() ChainType {
return idx.config.Type
}
// IsRunning returns whether the indexer is running
func (idx *PlatformIndexer) IsRunning() bool {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.isRunning
}
// Start begins indexing the platform chain
func (idx *PlatformIndexer) Start(ctx context.Context) error {
idx.mu.Lock()
idx.isRunning = true
idx.stats.StartTime = time.Now()
idx.stats.ChainID = idx.config.ID
idx.stats.ChainName = idx.config.Name
idx.mu.Unlock()
log.Printf("[%s] Starting native indexer for %s (%s)", idx.config.ID, idx.config.Name, idx.config.Type)
go idx.indexLoop(ctx)
return nil
}
// Stop halts the indexer
func (idx *PlatformIndexer) Stop() error {
idx.mu.Lock()
defer idx.mu.Unlock()
if !idx.isRunning {
return nil
}
close(idx.stopChan)
idx.isRunning = false
log.Printf("[%s] Stopped native indexer", idx.config.ID)
return nil
}
// IndexBlock indexes a specific block
func (idx *PlatformIndexer) IndexBlock(ctx context.Context, blockNumber uint64) error {
return idx.fetchAndProcessBlock(ctx)
}
// IndexBlockRange indexes a range of blocks
func (idx *PlatformIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for block := from; block <= to; block++ {
if err := idx.IndexBlock(ctx, block); err != nil {
return err
}
}
return nil
}
// GetLatestBlock returns the latest block number from the chain
func (idx *PlatformIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.latestHeight, nil
}
// GetIndexedBlock returns the last indexed block number
func (idx *PlatformIndexer) GetIndexedBlock() uint64 {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.indexedBlock
}
// Stats returns current indexer statistics
func (idx *PlatformIndexer) Stats() *IndexerStats {
idx.mu.RLock()
defer idx.mu.RUnlock()
stats := idx.stats
stats.ChainID = idx.config.ID
stats.ChainName = idx.config.Name
stats.IsRunning = idx.isRunning
stats.LatestBlock = idx.latestHeight
stats.IndexedBlock = idx.indexedBlock
stats.Uptime = time.Since(stats.StartTime)
if stats.Uptime.Seconds() > 0 {
stats.BlocksPerSecond = float64(stats.BlocksProcessed) / stats.Uptime.Seconds()
}
return &stats
}
// indexLoop is the main indexing loop. If the chain config has a WS
// endpoint, subscribe for realtime block notifications; otherwise fall
// back to polling at PollInterval.
func (idx *PlatformIndexer) indexLoop(ctx context.Context) {
// Try realtime WebSocket subscription first when a WS endpoint is set.
if idx.config.WS != "" {
if idx.runRealtimeLoop(ctx) {
// realtime ran to completion (ctx cancel or stop)
return
}
// fall through to polling if WS subscription failed
log.Printf("[%s] WS subscription unavailable, falling back to polling", idx.config.ID)
}
ticker := time.NewTicker(idx.config.PollInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-idx.stopChan:
return
case <-ticker.C:
if err := idx.fetchAndProcessBlock(ctx); err != nil {
idx.mu.Lock()
idx.stats.ErrorCount++
idx.stats.LastError = err.Error()
idx.mu.Unlock()
// Don't log every error for platform chains that might not be running
}
}
}
}
// runRealtimeLoop subscribes to chain notifications over WebSocket and
// processes each block as it arrives. Returns true if the loop ran to
// completion (stop requested), false if the WS connection could not be
// established — in which case the caller should fall back to polling.
func (idx *PlatformIndexer) runRealtimeLoop(ctx context.Context) bool {
method := idx.realtimeSubscribeMethod()
if method == "" {
return false
}
dialer := websocket.Dialer{HandshakeTimeout: 10 * time.Second}
conn, _, err := dialer.DialContext(ctx, idx.config.WS, nil)
if err != nil {
return false
}
defer conn.Close()
// Send JSON-RPC subscribe request.
req := map[string]interface{}{
"jsonrpc": "2.0",
"id": 1,
"method": method,
"params": []interface{}{},
}
if err := conn.WriteJSON(req); err != nil {
return false
}
log.Printf("[%s] Realtime subscription active via %s", idx.config.ID, idx.config.WS)
for {
select {
case <-ctx.Done():
return true
case <-idx.stopChan:
return true
default:
}
// Read with a deadline so we can check ctx periodically.
_ = conn.SetReadDeadline(time.Now().Add(60 * time.Second))
_, data, err := conn.ReadMessage()
if err != nil {
// Exit realtime mode on connection error; caller will poll.
idx.mu.Lock()
idx.stats.ErrorCount++
idx.stats.LastError = "ws: " + err.Error()
idx.mu.Unlock()
return false
}
// Extract the `result` or `params.result` field from JSON-RPC notifications.
block := extractRealtimeBlock(data)
if block == nil {
continue
}
if err := idx.processBlock(block); err != nil {
idx.mu.Lock()
idx.stats.ErrorCount++
idx.stats.LastError = err.Error()
idx.mu.Unlock()
}
}
}
// realtimeSubscribeMethod returns the JSON-RPC subscribe method name for
// the chain's realtime block feed. Each native VM exposes a
// `{prefix}.subscribeLatest` notification stream over WebSocket.
func (idx *PlatformIndexer) realtimeSubscribeMethod() string {
switch idx.config.Type {
case ChainTypePlatform:
return "platform.subscribeLatest"
case ChainTypeUTXO:
return "xvm.subscribeLatest"
case ChainTypeAI:
return "ai.subscribeLatest"
case ChainTypeBridge:
return "bridge.subscribeLatest"
case ChainTypeDEX:
return "dex.subscribeLatest"
case ChainTypeGraph:
return "graph.subscribeLatest"
case ChainTypeIdentity:
return "identity.subscribeLatest"
case ChainTypeKey:
return "key.subscribeLatest"
case ChainTypeMPC:
return "mpc.subscribeLatest"
case ChainTypeOracle:
return "oracle.subscribeLatest"
case ChainTypeQuantum:
return "quantum.subscribeLatest"
case ChainTypeRelay:
return "relay.subscribeLatest"
case ChainTypeThreshold:
return "threshold.subscribeLatest"
case ChainTypeZK:
return "zk.subscribeLatest"
}
return ""
}
// extractRealtimeBlock pulls the block payload out of either a subscribe
// response (`result` field) or a notification (`params.result` field).
// Returns nil for subscribe confirmations that don't carry a block.
func extractRealtimeBlock(data []byte) json.RawMessage {
var env struct {
Result json.RawMessage `json:"result"`
Params struct {
Result json.RawMessage `json:"result"`
} `json:"params"`
}
if err := json.Unmarshal(data, &env); err != nil {
return nil
}
if len(env.Params.Result) > 0 && string(env.Params.Result) != "null" {
return env.Params.Result
}
// A subscribe confirmation's `result` is usually a subscription id
// string, not a block — skip those. Blocks come as notifications.
return nil
}
// fetchAndProcessBlock fetches and processes the latest block
func (idx *PlatformIndexer) fetchAndProcessBlock(ctx context.Context) error {
// Different chain types have different RPC methods
switch idx.config.Type {
case ChainTypePlatform:
return idx.fetchPlatformBlock(ctx)
case ChainTypeUTXO:
return idx.fetchXVMBlock(ctx)
case ChainTypeAI:
return idx.fetchAIBlock(ctx)
case ChainTypeBridge:
return idx.fetchBridgeBlock(ctx)
case ChainTypeDEX:
return idx.fetchDEXBlock(ctx)
case ChainTypeGraph:
return idx.fetchGraphBlock(ctx)
case ChainTypeIdentity:
return idx.fetchIdentityBlock(ctx)
case ChainTypeKey:
return idx.fetchKeyBlock(ctx)
case ChainTypeMPC:
return idx.fetchMPCBlock(ctx)
case ChainTypeOracle:
return idx.fetchOracleBlock(ctx)
case ChainTypeQuantum:
return idx.fetchQuantumBlock(ctx)
case ChainTypeRelay:
return idx.fetchRelayBlock(ctx)
case ChainTypeThreshold:
return idx.fetchThresholdBlock(ctx)
case ChainTypeZK:
return idx.fetchZKBlock(ctx)
default:
return fmt.Errorf("unknown platform chain type: %s", idx.config.Type)
}
}
// fetchAIBlock fetches blocks from A-Chain (AI VM)
func (idx *PlatformIndexer) fetchAIBlock(ctx context.Context) error {
// A-Chain RPC methods: ai.getLatestBlock, ai.getProviders, ai.getTasks
block, err := idx.rpcCall(ctx, "ai.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchBridgeBlock fetches blocks from B-Chain (Bridge VM)
func (idx *PlatformIndexer) fetchBridgeBlock(ctx context.Context) error {
// B-Chain RPC methods: bridge.getLatestBlock, bridge.getPendingRequests
block, err := idx.rpcCall(ctx, "bridge.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchThresholdBlock fetches blocks from T-Chain (Threshold VM)
func (idx *PlatformIndexer) fetchThresholdBlock(ctx context.Context) error {
// T-Chain RPC methods: threshold.getLatestBlock, threshold.getKeyGenSessions
block, err := idx.rpcCall(ctx, "threshold.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchZKBlock fetches blocks from Z-Chain (ZK VM)
func (idx *PlatformIndexer) fetchZKBlock(ctx context.Context) error {
// Z-Chain RPC methods: zk.getLatestBlock, zk.getProofs
block, err := idx.rpcCall(ctx, "zk.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchGraphBlock fetches blocks from G-Chain (Graph VM)
func (idx *PlatformIndexer) fetchGraphBlock(ctx context.Context) error {
// G-Chain uses GraphQL endpoint, different from JSON-RPC
return idx.fetchGraphQLSchema(ctx)
}
// fetchIdentityBlock fetches blocks from I-Chain (Identity VM)
func (idx *PlatformIndexer) fetchIdentityBlock(ctx context.Context) error {
// I-Chain RPC methods: identity.getLatestBlock, identity.getIdentities
block, err := idx.rpcCall(ctx, "identity.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchKeyBlock fetches blocks from K-Chain (Key VM)
func (idx *PlatformIndexer) fetchKeyBlock(ctx context.Context) error {
// K-Chain RPC methods: key.getLatestBlock, key.getKeyShares
block, err := idx.rpcCall(ctx, "key.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchDEXBlock fetches blocks from D-Chain (DEX VM)
func (idx *PlatformIndexer) fetchDEXBlock(ctx context.Context) error {
// D-Chain RPC methods: dex.getLatestBlock, dex.getOrderbook
block, err := idx.rpcCall(ctx, "dex.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchPlatformBlock fetches blocks from P-Chain (PlatformVM)
// P-Chain tracks validators, delegation, and subnet membership.
func (idx *PlatformIndexer) fetchPlatformBlock(ctx context.Context) error {
block, err := idx.rpcCall(ctx, "platform.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchXVMBlock fetches blocks from X-Chain (XVM, UTXO asset chain)
func (idx *PlatformIndexer) fetchXVMBlock(ctx context.Context) error {
block, err := idx.rpcCall(ctx, "xvm.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchMPCBlock fetches blocks from M-Chain (MPC coordination VM)
// M-Chain coordinates CGGMP21 / FROST threshold signing sessions.
func (idx *PlatformIndexer) fetchMPCBlock(ctx context.Context) error {
block, err := idx.rpcCall(ctx, "mpc.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchOracleBlock fetches blocks from O-Chain (OracleVM)
// O-Chain aggregates Chainlink/Pyth/TWAP feeds.
func (idx *PlatformIndexer) fetchOracleBlock(ctx context.Context) error {
block, err := idx.rpcCall(ctx, "oracle.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchQuantumBlock fetches blocks from Q-Chain (QuantumVM)
// Q-Chain runs Corona LWE threshold consensus + ML-DSA finality.
func (idx *PlatformIndexer) fetchQuantumBlock(ctx context.Context) error {
block, err := idx.rpcCall(ctx, "quantum.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// fetchRelayBlock fetches blocks from R-Chain (RelayVM)
// R-Chain handles cross-chain message relay and Warp routing.
func (idx *PlatformIndexer) fetchRelayBlock(ctx context.Context) error {
block, err := idx.rpcCall(ctx, "relay.getLatestBlock", nil)
if err != nil {
return err
}
return idx.processBlock(block)
}
// rpcCall makes a JSON-RPC call to the platform chain
func (idx *PlatformIndexer) rpcCall(ctx context.Context, method string, params interface{}) (json.RawMessage, error) {
reqBody := map[string]interface{}{
"jsonrpc": "2.0",
"id": 1,
"method": method,
"params": params,
}
body, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", idx.config.RPC, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := idx.client.Do(req)
if err != nil {
return nil, fmt.Errorf("RPC request failed: %w", err)
}
defer resp.Body.Close()
var result struct {
Result json.RawMessage `json:"result"`
Error *struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
if result.Error != nil {
return nil, fmt.Errorf("RPC error %d: %s", result.Error.Code, result.Error.Message)
}
return result.Result, nil
}
// fetchGraphQLSchema fetches the GraphQL schema from G-Chain
func (idx *PlatformIndexer) fetchGraphQLSchema(ctx context.Context) error {
// GraphQL introspection query
query := `{"query": "{ __schema { types { name } } }"}`
req, err := http.NewRequestWithContext(ctx, "POST", idx.config.RPC+"/graphql", bytes.NewReader([]byte(query)))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := idx.client.Do(req)
if err != nil {
// G-Chain might not be fully available yet, that's OK
return nil
}
defer resp.Body.Close()
idx.mu.Lock()
idx.stats.BlocksProcessed++
idx.mu.Unlock()
return nil
}
// processBlock processes a block from a platform chain
func (idx *PlatformIndexer) processBlock(data json.RawMessage) error {
if data == nil {
return nil
}
var block PlatformBlock
if err := json.Unmarshal(data, &block); err != nil {
// Not all responses are blocks, ignore parse errors
idx.mu.Lock()
idx.stats.BlocksProcessed++
idx.mu.Unlock()
return nil
}
idx.mu.Lock()
if block.Height > idx.latestHeight {
idx.latestHeight = block.Height
}
idx.stats.BlocksProcessed++
idx.indexedBlock = block.Height
idx.stats.IndexedBlock = block.Height
idx.mu.Unlock()
return nil
}
multichain/solana_indexer.go
-1912
View File
File diff suppressed because it is too large Load Diff
multichain/solana_indexer_test.go
-516
View File
@@ -1,516 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"context"
"testing"
)
func TestBase58Encode(t *testing.T) {
tests := []struct {
name string
input []byte
expected string
}{
{
name: "empty",
input: []byte{},
expected: "",
},
{
name: "single zero",
input: []byte{0},
expected: "1",
},
{
name: "leading zeros",
input: []byte{0, 0, 0, 1},
expected: "1112",
},
{
name: "hello world",
input: []byte("Hello World"),
expected: "JxF12TrwUP45BMd",
},
{
name: "solana pubkey bytes",
input: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
expected: "11111111111111111111111111111112",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := base58Encode(tt.input)
if result != tt.expected {
t.Errorf("base58Encode(%v) = %s, want %s", tt.input, result, tt.expected)
}
})
}
}
func TestBase58Decode(t *testing.T) {
tests := []struct {
name string
input string
expected []byte
wantErr bool
}{
{
name: "empty",
input: "",
expected: []byte{},
},
{
name: "single 1",
input: "1",
expected: []byte{0},
},
{
name: "leading 1s",
input: "1112",
expected: []byte{0, 0, 0, 1},
},
{
name: "hello world",
input: "JxF12TrwUP45BMd",
expected: []byte("Hello World"),
},
{
name: "invalid char 0",
input: "0invalid",
wantErr: true,
},
{
name: "invalid char O",
input: "O1234",
wantErr: true,
},
{
name: "invalid char I",
input: "I1234",
wantErr: true,
},
{
name: "invalid char l",
input: "l1234",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := base58Decode(tt.input)
if tt.wantErr {
if err == nil {
t.Errorf("base58Decode(%s) expected error, got nil", tt.input)
}
return
}
if err != nil {
t.Errorf("base58Decode(%s) error = %v", tt.input, err)
return
}
if string(result) != string(tt.expected) {
t.Errorf("base58Decode(%s) = %v, want %v", tt.input, result, tt.expected)
}
})
}
}
func TestBase58RoundTrip(t *testing.T) {
tests := [][]byte{
{},
{0},
{0, 0, 0},
{1, 2, 3, 4, 5},
{255, 255, 255},
make([]byte, 32), // 32-byte pubkey with zeros
}
// Fill the last test with non-zero bytes
for i := range tests[len(tests)-1] {
tests[len(tests)-1][i] = byte(i)
}
for i, input := range tests {
encoded := base58Encode(input)
decoded, err := base58Decode(encoded)
if err != nil {
t.Errorf("test %d: roundtrip decode error: %v", i, err)
continue
}
if string(decoded) != string(input) {
t.Errorf("test %d: roundtrip failed: input=%v, encoded=%s, decoded=%v", i, input, encoded, decoded)
}
}
}
func TestBorshReader_ReadU8(t *testing.T) {
reader := NewBorshReaderFromBytes([]byte{0x12, 0x34})
v, err := reader.ReadU8()
if err != nil {
t.Fatalf("ReadU8 error: %v", err)
}
if v != 0x12 {
t.Errorf("ReadU8 = %d, want %d", v, 0x12)
}
v, err = reader.ReadU8()
if err != nil {
t.Fatalf("ReadU8 error: %v", err)
}
if v != 0x34 {
t.Errorf("ReadU8 = %d, want %d", v, 0x34)
}
// Should fail on empty
_, err = reader.ReadU8()
if err == nil {
t.Error("expected buffer underflow error")
}
}
func TestBorshReader_ReadU16(t *testing.T) {
// Little-endian: 0x1234 = [0x34, 0x12]
reader := NewBorshReaderFromBytes([]byte{0x34, 0x12})
v, err := reader.ReadU16()
if err != nil {
t.Fatalf("ReadU16 error: %v", err)
}
if v != 0x1234 {
t.Errorf("ReadU16 = %d, want %d", v, 0x1234)
}
}
func TestBorshReader_ReadU32(t *testing.T) {
// Little-endian: 0x12345678 = [0x78, 0x56, 0x34, 0x12]
reader := NewBorshReaderFromBytes([]byte{0x78, 0x56, 0x34, 0x12})
v, err := reader.ReadU32()
if err != nil {
t.Fatalf("ReadU32 error: %v", err)
}
if v != 0x12345678 {
t.Errorf("ReadU32 = %d, want %d", v, 0x12345678)
}
}
func TestBorshReader_ReadU64(t *testing.T) {
// Little-endian: 1000000000 (1 SOL in lamports)
reader := NewBorshReaderFromBytes([]byte{0x00, 0xca, 0x9a, 0x3b, 0x00, 0x00, 0x00, 0x00})
v, err := reader.ReadU64()
if err != nil {
t.Fatalf("ReadU64 error: %v", err)
}
if v != 1000000000 {
t.Errorf("ReadU64 = %d, want %d", v, 1000000000)
}
}
func TestBorshReader_ReadString(t *testing.T) {
// Borsh string: 4-byte length prefix + UTF-8 data
// "Hello" = [5, 0, 0, 0, 'H', 'e', 'l', 'l', 'o']
data := []byte{5, 0, 0, 0, 'H', 'e', 'l', 'l', 'o'}
reader := NewBorshReaderFromBytes(data)
s, err := reader.ReadString()
if err != nil {
t.Fatalf("ReadString error: %v", err)
}
if s != "Hello" {
t.Errorf("ReadString = %q, want %q", s, "Hello")
}
}
func TestBorshReader_ReadPubkey(t *testing.T) {
// 32 bytes of sequential values
data := make([]byte, 32)
for i := range data {
data[i] = byte(i)
}
reader := NewBorshReaderFromBytes(data)
pubkey, err := reader.ReadPubkey()
if err != nil {
t.Fatalf("ReadPubkey error: %v", err)
}
// Verify we can decode it back
decoded, err := base58Decode(pubkey)
if err != nil {
t.Fatalf("decode pubkey error: %v", err)
}
if len(decoded) != 32 {
t.Errorf("decoded pubkey length = %d, want 32", len(decoded))
}
for i, b := range decoded {
if b != byte(i) {
t.Errorf("decoded[%d] = %d, want %d", i, b, i)
}
}
}
func TestBorshReader_ReadBool(t *testing.T) {
reader := NewBorshReaderFromBytes([]byte{0, 1, 255})
v, err := reader.ReadBool()
if err != nil {
t.Fatalf("ReadBool error: %v", err)
}
if v != false {
t.Errorf("ReadBool = %v, want false", v)
}
v, err = reader.ReadBool()
if err != nil {
t.Fatalf("ReadBool error: %v", err)
}
if v != true {
t.Errorf("ReadBool = %v, want true", v)
}
v, err = reader.ReadBool()
if err != nil {
t.Fatalf("ReadBool error: %v", err)
}
if v != true {
t.Errorf("ReadBool = %v, want true (non-zero)", v)
}
}
func TestBorshReader_ReadVec(t *testing.T) {
// Vec of 3 u8s: [3, 0, 0, 0, 10, 20, 30]
data := []byte{3, 0, 0, 0, 10, 20, 30}
reader := NewBorshReaderFromBytes(data)
length, err := reader.ReadVecLen()
if err != nil {
t.Fatalf("ReadVecLen error: %v", err)
}
if length != 3 {
t.Errorf("ReadVecLen = %d, want 3", length)
}
for i := uint32(0); i < length; i++ {
v, err := reader.ReadU8()
if err != nil {
t.Fatalf("ReadU8 error at %d: %v", i, err)
}
expected := byte((i + 1) * 10)
if v != expected {
t.Errorf("element %d = %d, want %d", i, v, expected)
}
}
}
func TestNewBorshReader_Base58(t *testing.T) {
// Encode some test data
testData := []byte{1, 2, 3, 4, 5}
encoded := base58Encode(testData)
reader, err := NewBorshReader(encoded)
if err != nil {
t.Fatalf("NewBorshReader error: %v", err)
}
for i, expected := range testData {
v, err := reader.ReadU8()
if err != nil {
t.Fatalf("ReadU8 error at %d: %v", i, err)
}
if v != expected {
t.Errorf("byte %d = %d, want %d", i, v, expected)
}
}
}
func TestTrimNullBytes(t *testing.T) {
tests := []struct {
input string
expected string
}{
{"Hello", "Hello"},
{"Hello\x00\x00\x00", "Hello"},
{"\x00\x00\x00", ""},
{"", ""},
{"No nulls", "No nulls"},
{"Middle\x00null", "Middle\x00null"}, // Only trims trailing
}
for _, tt := range tests {
result := trimNullBytes(tt.input)
if result != tt.expected {
t.Errorf("trimNullBytes(%q) = %q, want %q", tt.input, result, tt.expected)
}
}
}
func TestSolanaNFTSale_Struct(t *testing.T) {
sale := &SolanaNFTSale{
Signature: "5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW",
Slot: 123456789,
BlockTime: 1640000000,
Marketplace: "magic_eden",
Mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
Buyer: "11111111111111111111111111111111",
Seller: "22222222222222222222222222222222",
Price: 1000000000, // 1 SOL
RoyaltyAmount: 50000000, // 0.05 SOL
MarketplaceFee: 25000000, // 0.025 SOL
}
if sale.Marketplace != "magic_eden" {
t.Errorf("Marketplace = %s, want magic_eden", sale.Marketplace)
}
if sale.Price != 1000000000 {
t.Errorf("Price = %d, want 1000000000", sale.Price)
}
}
func TestMetaplexMetadata_Struct(t *testing.T) {
metadata := &MetaplexMetadata{
Mint: "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v",
UpdateAuthority: "11111111111111111111111111111111",
Name: "Test NFT",
Symbol: "TEST",
URI: "https://arweave.net/abc123",
SellerFeeBasisPoints: 500, // 5%
Creators: []MetaplexCreator{
{
Address: "22222222222222222222222222222222",
Verified: true,
Share: 100,
},
},
PrimarySaleHappened: false,
IsMutable: true,
}
if metadata.SellerFeeBasisPoints != 500 {
t.Errorf("SellerFeeBasisPoints = %d, want 500", metadata.SellerFeeBasisPoints)
}
if len(metadata.Creators) != 1 {
t.Errorf("len(Creators) = %d, want 1", len(metadata.Creators))
}
if metadata.Creators[0].Share != 100 {
t.Errorf("Creator share = %d, want 100", metadata.Creators[0].Share)
}
}
func TestMetaplexIndexer_ProgramIDs(t *testing.T) {
indexer := NewMetaplexIndexer(ProtocolConfig{}, nil)
programIDs := indexer.ProgramIDs()
expected := []string{
"metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s",
"p1exdMJcjVao65QdewkaZRUnU6VPSXhus9n2GzWfh98",
"CoREENxT6tW1HoK8ypY1SxRMZTcVPm7R94rH4PZNhX7d",
"BGUMAp9Gq7iTEuizy4pqaxsTyUCBK68MDfK752saRPUY",
}
if len(programIDs) != len(expected) {
t.Errorf("len(programIDs) = %d, want %d", len(programIDs), len(expected))
}
for i, id := range programIDs {
if id != expected[i] {
t.Errorf("programID[%d] = %s, want %s", i, id, expected[i])
}
}
}
func TestMagicEdenIndexer_ProgramIDs(t *testing.T) {
indexer := NewMagicEdenSolanaIndexer(ProtocolConfig{}, nil)
programIDs := indexer.ProgramIDs()
expected := []string{
"M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K",
"MEisE1HzehtrDpAAT8PnLHjpSSkRYakotTuJRPjTpo8",
}
if len(programIDs) != len(expected) {
t.Errorf("len(programIDs) = %d, want %d", len(programIDs), len(expected))
}
for i, id := range programIDs {
if id != expected[i] {
t.Errorf("programID[%d] = %s, want %s", i, id, expected[i])
}
}
}
func TestTensorIndexer_ProgramIDs(t *testing.T) {
indexer := NewTensorIndexer(ProtocolConfig{}, nil)
programIDs := indexer.ProgramIDs()
expected := []string{
"TSWAPaqyCSx2KABk68Shruf4rp7CxcNi8hAsbdwmHbN",
"TCMPhJdwDryooaGtiocG1u3xcYbRpiJzb283XfCZsDp",
"TBIDxNsM9DuLs4YCbmA7VuACbMZ5WyYv5JGQxpqLMVJ",
}
if len(programIDs) != len(expected) {
t.Errorf("len(programIDs) = %d, want %d", len(programIDs), len(expected))
}
for i, id := range programIDs {
if id != expected[i] {
t.Errorf("programID[%d] = %s, want %s", i, id, expected[i])
}
}
}
func TestSolanaIndexer_ProcessTransaction_SkipsFailedTx(t *testing.T) {
// Create a minimal indexer without protocols
config := ChainConfig{
ID: "solana-test",
Name: "Solana Test",
RPC: "http://localhost:8899",
}
indexer, err := NewSolanaIndexer(config, nil)
if err != nil {
t.Fatalf("NewSolanaIndexer error: %v", err)
}
// Create a failed transaction
tx := &SolanaTransaction{
Signature: "test-sig",
Status: "failed",
Err: "some error",
}
// Process should not panic and should skip failed tx
indexer.processTransaction(context.Background(), tx)
// Stats should not have changed
if indexer.stats.EventsProcessed != 0 {
t.Errorf("EventsProcessed = %d, want 0", indexer.stats.EventsProcessed)
}
}
func TestSolanaIndexer_Name(t *testing.T) {
tests := []struct {
indexer SolanaProtocolIndexer
expected string
}{
{NewMetaplexIndexer(ProtocolConfig{}, nil), "metaplex"},
{NewMagicEdenSolanaIndexer(ProtocolConfig{}, nil), "magic_eden_solana"},
{NewTensorIndexer(ProtocolConfig{}, nil), "tensor"},
{NewGenericSolanaIndexer(ProtocolConfig{}), "generic_solana"},
{NewRaydiumIndexer(ProtocolConfig{}), "raydium"},
{NewOrcaIndexer(ProtocolConfig{}), "orca"},
{NewJupiterIndexer(ProtocolConfig{}), "jupiter"},
}
for _, tt := range tests {
if tt.indexer.Name() != tt.expected {
t.Errorf("%T.Name() = %s, want %s", tt.indexer, tt.indexer.Name(), tt.expected)
}
}
}
multichain/ton_indexer.go
-545
View File
@@ -1,545 +0,0 @@
// Copyright (c) 2025 Lux Partners Limited
// SPDX-License-Identifier: MIT
package multichain
import (
"context"
"encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"math/big"
"net/http"
"sync"
"sync/atomic"
"time"
)
// TON NFT operation codes (TEP-62)
const (
TONOpTransfer uint32 = 0x5fcc3d14 // NFT transfer
TONOpOwnershipAssigned uint32 = 0x05138d91 // Ownership assigned
TONOpGetStaticData uint32 = 0x2fcb26a2 // Get static data
TONOpReportStaticData uint32 = 0x8b771735 // Report static data
TONOpGetRoyaltyParams uint32 = 0x693d3950 // Get royalty params (TEP-66)
TONOpReportRoyalty uint32 = 0xa8cb00ad // Report royalty params
)
// TON Marketplace addresses
const (
GetGemsMarketplace = "EQBYTuYbLf8INxFtD8tQeNk5ZLy-nAX9ahQbG_yl1qQ-GEMS"
TONDiamondsMarketplace = "EQA8gwSB3bP-VmzPLTQOIaW-8dFOH_kVVvz7eyVm8MAvPYv-"
FragmentMarketplace = "EQAvlWFDxGF2lXm67y4yzC17wYKD9A0guwPkMs1gOsM__NOT"
)
// TONIndexer indexes TON blockchain
type TONIndexer struct {
mu sync.RWMutex
config ChainConfig
db Database
client *http.Client
// State
running int32
indexedLt uint64
latestLt uint64
// NFT sale tracking
nftSales []*TONNFTSale
// Stats
stats *IndexerStats
// Lifecycle
ctx context.Context
cancel context.CancelFunc
}
// TONNFTSale represents an NFT sale on TON
type TONNFTSale struct {
TxHash string `json:"txHash"`
Lt uint64 `json:"lt"`
Utime int64 `json:"utime"`
Marketplace string `json:"marketplace"` // getgems, tondiamonds, fragment
Collection string `json:"collection"`
NFTAddress string `json:"nftAddress"`
Seller string `json:"seller"`
Buyer string `json:"buyer"`
Price *big.Int `json:"price"` // in nanotons
RoyaltyAmount *big.Int `json:"royaltyAmount"`
MarketplaceFee *big.Int `json:"marketplaceFee"`
Timestamp time.Time `json:"timestamp"`
}
// TONTransaction represents a TON transaction
type TONTransaction struct {
Hash string `json:"hash"`
Lt uint64 `json:"lt"`
Utime int64 `json:"utime"`
Fee string `json:"fee"`
StorageFee string `json:"storage_fee"`
OtherFee string `json:"other_fee"`
InMsg *TONMessage `json:"in_msg"`
OutMsgs []*TONMessage `json:"out_msgs"`
Account string `json:"account"`
Success bool `json:"success"`
ExitCode int `json:"exit_code"`
Description json.RawMessage `json:"description"`
}
// TONMessage represents a TON message
type TONMessage struct {
Hash string `json:"hash"`
Source string `json:"source"`
Destination string `json:"destination"`
Value string `json:"value"`
FwdFee string `json:"fwd_fee"`
IhrFee string `json:"ihr_fee"`
CreatedLt uint64 `json:"created_lt"`
BodyHash string `json:"body_hash"`
MsgData *TONMsgData `json:"msg_data"`
OpCode uint32 `json:"op_code,omitempty"`
}
// TONMsgData represents message data
type TONMsgData struct {
Type string `json:"@type"`
Body string `json:"body,omitempty"` // base64 encoded
Text string `json:"text,omitempty"`
}
// TONBlock represents a TON block
type TONBlock struct {
Workchain int32 `json:"workchain"`
Shard int64 `json:"shard"`
Seqno uint32 `json:"seqno"`
RootHash string `json:"root_hash"`
FileHash string `json:"file_hash"`
}
// NewTonIndexer creates a new TON indexer
func NewTonIndexer(config ChainConfig, db Database) (*TONIndexer, error) {
ctx, cancel := context.WithCancel(context.Background())
idx := &TONIndexer{
config: config,
db: db,
client: &http.Client{Timeout: 30 * time.Second},
nftSales: make([]*TONNFTSale, 0),
ctx: ctx,
cancel: cancel,
stats: &IndexerStats{
ChainID: config.ID,
ChainName: config.Name,
StartTime: time.Now(),
},
}
return idx, nil
}
// ChainID returns the chain identifier
func (t *TONIndexer) ChainID() string {
return t.config.ID
}
// ChainType returns the chain type
func (t *TONIndexer) ChainType() ChainType {
return ChainTypeTon
}
// Start starts the indexer
func (t *TONIndexer) Start(ctx context.Context) error {
if !atomic.CompareAndSwapInt32(&t.running, 0, 1) {
return fmt.Errorf("indexer already running")
}
t.stats.IsRunning = true
return nil
}
// Stop stops the indexer
func (t *TONIndexer) Stop() error {
atomic.StoreInt32(&t.running, 0)
t.cancel()
t.stats.IsRunning = false
return nil
}
// IsRunning returns whether the indexer is running
func (t *TONIndexer) IsRunning() bool {
return atomic.LoadInt32(&t.running) == 1
}
// GetLatestBlock returns the latest masterchain seqno
func (t *TONIndexer) GetLatestBlock(ctx context.Context) (uint64, error) {
result, err := t.apiCall(ctx, "/getMasterchainInfo", nil)
if err != nil {
return 0, err
}
var info struct {
Last struct {
Seqno uint32 `json:"seqno"`
} `json:"last"`
}
if err := json.Unmarshal(result, &info); err != nil {
return 0, err
}
t.mu.Lock()
t.latestLt = uint64(info.Last.Seqno)
t.mu.Unlock()
return uint64(info.Last.Seqno), nil
}
// GetIndexedBlock returns the last indexed seqno
func (t *TONIndexer) GetIndexedBlock() uint64 {
t.mu.RLock()
defer t.mu.RUnlock()
return t.indexedLt
}
// IndexBlock indexes transactions in a block
func (t *TONIndexer) IndexBlock(ctx context.Context, blockNumber uint64) error {
// Get transactions for the block
txs, err := t.getBlockTransactions(ctx, blockNumber)
if err != nil {
return err
}
// Process each transaction
for _, tx := range txs {
t.processTransaction(ctx, tx)
}
t.mu.Lock()
t.indexedLt = blockNumber
t.stats.BlocksProcessed++
t.stats.TxsProcessed += uint64(len(txs))
t.stats.LastBlockTime = time.Now()
t.mu.Unlock()
return nil
}
// IndexBlockRange indexes a range of blocks
func (t *TONIndexer) IndexBlockRange(ctx context.Context, from, to uint64) error {
for seqno := from; seqno <= to; seqno++ {
if err := t.IndexBlock(ctx, seqno); err != nil {
t.stats.ErrorCount++
t.stats.LastError = err.Error()
continue
}
}
return nil
}
// getBlockTransactions fetches transactions for a masterchain block
func (t *TONIndexer) getBlockTransactions(ctx context.Context, seqno uint64) ([]*TONTransaction, error) {
params := map[string]interface{}{
"seqno": seqno,
}
result, err := t.apiCall(ctx, "/getBlockTransactions", params)
if err != nil {
return nil, err
}
var response struct {
Transactions []*TONTransaction `json:"transactions"`
}
if err := json.Unmarshal(result, &response); err != nil {
return nil, err
}
return response.Transactions, nil
}
// processTransaction processes a transaction for NFT events
func (t *TONIndexer) processTransaction(ctx context.Context, tx *TONTransaction) {
if tx.InMsg == nil {
return
}
// Check if this is an NFT-related transaction
opCode := t.extractOpCode(tx.InMsg)
if opCode == 0 {
return
}
switch opCode {
case TONOpTransfer:
t.processNFTTransfer(tx)
case TONOpOwnershipAssigned:
t.processOwnershipAssigned(tx)
}
// Check marketplace addresses for sales
t.checkMarketplaceSale(tx)
}
// extractOpCode extracts the operation code from a message
func (t *TONIndexer) extractOpCode(msg *TONMessage) uint32 {
if msg.MsgData == nil || msg.MsgData.Body == "" {
return 0
}
data, err := base64.StdEncoding.DecodeString(msg.MsgData.Body)
if err != nil || len(data) < 4 {
return 0
}
return binary.BigEndian.Uint32(data[:4])
}
// processNFTTransfer processes an NFT transfer message
func (t *TONIndexer) processNFTTransfer(tx *TONTransaction) {
if tx.InMsg == nil || tx.InMsg.MsgData == nil {
return
}
data, err := base64.StdEncoding.DecodeString(tx.InMsg.MsgData.Body)
if err != nil || len(data) < 36 { // op (4) + query_id (8) + new_owner (33+)
return
}
// Parse transfer data
// op: uint32, query_id: uint64, new_owner: MsgAddress
// For now just track the event
t.mu.Lock()
t.stats.EventsProcessed++
t.mu.Unlock()
}
// processOwnershipAssigned processes ownership assigned notification
func (t *TONIndexer) processOwnershipAssigned(tx *TONTransaction) {
if tx.InMsg == nil || tx.InMsg.MsgData == nil {
return
}
t.mu.Lock()
t.stats.EventsProcessed++
t.mu.Unlock()
}
// checkMarketplaceSale checks if transaction is a marketplace sale
func (t *TONIndexer) checkMarketplaceSale(tx *TONTransaction) {
if tx.InMsg == nil {
return
}
// Check if source or destination is a known marketplace
marketplace := ""
switch tx.InMsg.Source {
case GetGemsMarketplace:
marketplace = "getgems"
case TONDiamondsMarketplace:
marketplace = "tondiamonds"
case FragmentMarketplace:
marketplace = "fragment"
}
if marketplace == "" {
switch tx.InMsg.Destination {
case GetGemsMarketplace:
marketplace = "getgems"
case TONDiamondsMarketplace:
marketplace = "tondiamonds"
case FragmentMarketplace:
marketplace = "fragment"
}
}
if marketplace == "" {
return
}
// Parse value as price
price := new(big.Int)
price.SetString(tx.InMsg.Value, 10)
sale := &TONNFTSale{
TxHash: tx.Hash,
Lt: tx.Lt,
Utime: tx.Utime,
Marketplace: marketplace,
NFTAddress: tx.Account,
Seller: tx.InMsg.Source,
Buyer: tx.InMsg.Destination,
Price: price,
RoyaltyAmount: big.NewInt(0),
MarketplaceFee: big.NewInt(0),
Timestamp: time.Unix(tx.Utime, 0),
}
t.mu.Lock()
t.nftSales = append(t.nftSales, sale)
t.stats.EventsProcessed++
t.mu.Unlock()
}
// apiCall makes an API call to TON Center or similar API
func (t *TONIndexer) apiCall(ctx context.Context, endpoint string, params map[string]interface{}) (json.RawMessage, error) {
url := t.config.RPC + endpoint
if t.config.API != "" {
url = t.config.API + endpoint
}
// Build query params
if len(params) > 0 {
url += "?"
first := true
for k, v := range params {
if !first {
url += "&"
}
url += fmt.Sprintf("%s=%v", k, v)
first = false
}
}
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, err
}
resp, err := t.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var result struct {
OK bool `json:"ok"`
Result json.RawMessage `json:"result"`
Error string `json:"error,omitempty"`
}
if err := json.Unmarshal(body, &result); err != nil {
return nil, err
}
if !result.OK {
return nil, fmt.Errorf("TON API error: %s", result.Error)
}
return result.Result, nil
}
// Stats returns indexer statistics
func (t *TONIndexer) Stats() *IndexerStats {
t.mu.RLock()
defer t.mu.RUnlock()
return &IndexerStats{
ChainID: t.stats.ChainID,
ChainName: t.stats.ChainName,
IsRunning: t.stats.IsRunning,
LatestBlock: t.latestLt,
IndexedBlock: t.indexedLt,
BlocksBehind: t.latestLt - t.indexedLt,
BlocksProcessed: t.stats.BlocksProcessed,
TxsProcessed: t.stats.TxsProcessed,
EventsProcessed: t.stats.EventsProcessed,
ErrorCount: t.stats.ErrorCount,
LastError: t.stats.LastError,
LastBlockTime: t.stats.LastBlockTime,
StartTime: t.stats.StartTime,
Uptime: time.Since(t.stats.StartTime),
}
}
// GetNFTSales returns recent NFT sales
func (t *TONIndexer) GetNFTSales(limit int) []*TONNFTSale {
t.mu.RLock()
defer t.mu.RUnlock()
if limit > len(t.nftSales) {
limit = len(t.nftSales)
}
return t.nftSales[len(t.nftSales)-limit:]
}
// =============================================================================
// TON Cell/BOC Parsing Helpers
// =============================================================================
// TONCell represents a TON Cell
type TONCell struct {
Data []byte
BitLen int
Refs []*TONCell
}
// parseBOC parses a Bag of Cells from base64 encoded data
func parseBOC(b64Data string) (*TONCell, error) {
data, err := base64.StdEncoding.DecodeString(b64Data)
if err != nil {
return nil, fmt.Errorf("base64 decode error: %w", err)
}
if len(data) < 5 {
return nil, fmt.Errorf("BOC too short")
}
// BOC magic bytes: b5ee9c72
if data[0] != 0xb5 || data[1] != 0xee || data[2] != 0x9c || data[3] != 0x72 {
return nil, fmt.Errorf("invalid BOC magic")
}
// Simplified BOC parsing - real implementation would be more complex
// For now return a basic cell with the raw data
return &TONCell{
Data: data[4:],
BitLen: (len(data) - 4) * 8,
}, nil
}
// parseTONAddress parses a TON address from cell data
func parseTONAddress(data []byte, offset int) (string, int, error) {
if offset >= len(data) {
return "", offset, fmt.Errorf("offset out of bounds")
}
// TON address format: 2 bits addr_type + workchain (if external) + hash
// Standard address: workchain (int8) + 256 bits hash
if offset+33 > len(data) {
return "", offset, fmt.Errorf("insufficient data for address")
}
workchain := int8(data[offset])
hash := data[offset+1 : offset+33]
// Format as user-friendly address (simplified)
addr := fmt.Sprintf("%d:%x", workchain, hash)
return addr, offset + 33, nil
}
// parseCoins parses a VarUInteger 16 (coins) from cell data
func parseCoins(data []byte, offset int) (*big.Int, int, error) {
if offset >= len(data) {
return nil, offset, fmt.Errorf("offset out of bounds")
}
// First 4 bits indicate length of the value in bytes
lenByte := data[offset]
length := int(lenByte >> 4)
if length == 0 {
return big.NewInt(0), offset + 1, nil
}
if offset+1+length > len(data) {
return nil, offset, fmt.Errorf("insufficient data for coins")
}
value := new(big.Int).SetBytes(data[offset+1 : offset+1+length])
return value, offset + 1 + length, nil
}