mirror of
https://github.com/hanzoai/bootnode.git
synced 2026-09-30 20:46:32 +00:00
feat: add planet-scale caching and omnichain query layer
Cloudflare Edge Caching (near-$0 costs for free tier): - Multi-tier cache: L0 (in-memory) + L1 (Redis) + L2 (Cloudflare edge) - Cache shared across ALL accounts (blockchain data is public) - Aggressive TTLs: immutable data (1yr), finalized blocks (1hr) - Cloudflare Worker script for edge caching - Cache bypass only for state-dependent calls (eth_call, eth_getBalance) Omnichain Query Layer (100+ chains): - Unified query client for blocks, transactions, addresses, tokens - PostgreSQL for metadata, ClickHouse for analytics - Qdrant vector search for semantic queries (similar contracts) - Meilisearch full-text search (addresses, tokens, contracts) - GraphQL API with Strawberry (subscriptions for real-time) - Cross-chain queries (address across all chains) Infrastructure Updates: - Added Qdrant and Meilisearch to compose.yml - ClickHouse schema for omnichain analytics (blocks, txs, DeFi, NFTs) - Materialized views for hourly/daily aggregates Integrates with lux/indexer for 100+ chain data.
This commit is contained in:
@@ -80,6 +80,21 @@ class Settings(BaseSettings):
|
||||
datastore_url: str = "clickhouse://bootnode:bootnode@localhost:8123/bootnode"
|
||||
datastore_pool_size: int = 10
|
||||
|
||||
# Vector Search (Qdrant)
|
||||
qdrant_url: str = "http://localhost:6333"
|
||||
qdrant_api_key: str = ""
|
||||
|
||||
# Full-Text Search (Meilisearch)
|
||||
meilisearch_url: str = "http://localhost:7700"
|
||||
meilisearch_api_key: str = ""
|
||||
|
||||
# Lux Indexer (Omnichain data)
|
||||
indexer_url: str = "http://localhost:5000"
|
||||
|
||||
# Cloudflare (Edge Caching)
|
||||
cloudflare_zone_id: str = ""
|
||||
cloudflare_api_token: str = ""
|
||||
|
||||
# Message Queue (Redis-based, @hanzo/mq compatible)
|
||||
# Uses the same Redis instance as cache - different db index for isolation
|
||||
mq_redis_db: int = 1 # Redis DB index for MQ (separate from cache at db 0)
|
||||
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
"""Cloudflare edge caching for planet-scale RPC infrastructure.
|
||||
|
||||
STRATEGY: Blockchain data is PUBLIC and IDENTICAL for ALL users.
|
||||
Cache aggressively at Cloudflare edge to serve 100+ chains at near-$0 cost.
|
||||
|
||||
Cache Tiers:
|
||||
1. IMMUTABLE (forever): Finalized blocks, transactions, receipts by hash
|
||||
2. LONG (1 hour): Block by number (if finalized), logs by block hash
|
||||
3. MEDIUM (30 seconds): Latest block number, gas prices
|
||||
4. SHORT (5 seconds): Pending transactions, mempool
|
||||
5. NO CACHE: State calls (eth_call), balances at "latest", send transactions
|
||||
|
||||
Key insight: Cache keys do NOT include API key/account - shared across ALL users.
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
class CacheTier(str, Enum):
|
||||
"""Cache duration tiers."""
|
||||
IMMUTABLE = "immutable" # 1 year (effectively forever)
|
||||
LONG = "long" # 1 hour
|
||||
MEDIUM = "medium" # 30 seconds
|
||||
SHORT = "short" # 5 seconds
|
||||
REALTIME = "realtime" # 2 seconds
|
||||
NO_CACHE = "no_cache" # Never cache
|
||||
|
||||
|
||||
# Cache TTLs in seconds
|
||||
CACHE_TTLS = {
|
||||
CacheTier.IMMUTABLE: 31536000, # 1 year
|
||||
CacheTier.LONG: 3600, # 1 hour
|
||||
CacheTier.MEDIUM: 30, # 30 seconds
|
||||
CacheTier.SHORT: 5, # 5 seconds
|
||||
CacheTier.REALTIME: 2, # 2 seconds
|
||||
CacheTier.NO_CACHE: 0, # No cache
|
||||
}
|
||||
|
||||
|
||||
# RPC method -> cache tier mapping
|
||||
# Principle: If the data is immutable (by hash) -> IMMUTABLE
|
||||
# If the data changes rarely -> LONG
|
||||
# If the data changes with blocks -> MEDIUM
|
||||
# If the data changes frequently -> SHORT
|
||||
# If the data is state-dependent or write -> NO_CACHE
|
||||
METHOD_CACHE_TIERS: dict[str, CacheTier] = {
|
||||
# === IMMUTABLE: Data by hash never changes ===
|
||||
"eth_getBlockByHash": CacheTier.IMMUTABLE,
|
||||
"eth_getTransactionByHash": CacheTier.IMMUTABLE,
|
||||
"eth_getTransactionReceipt": CacheTier.IMMUTABLE,
|
||||
"eth_getUncleByBlockHashAndIndex": CacheTier.IMMUTABLE,
|
||||
"eth_getBlockTransactionCountByHash": CacheTier.IMMUTABLE,
|
||||
"eth_getUncleCountByBlockHash": CacheTier.IMMUTABLE,
|
||||
"eth_getTransactionByBlockHashAndIndex": CacheTier.IMMUTABLE,
|
||||
# Code at address is immutable (contracts can't change bytecode)
|
||||
"eth_getCode": CacheTier.IMMUTABLE,
|
||||
|
||||
# === LONG: Static chain info or finalized block data ===
|
||||
"eth_chainId": CacheTier.IMMUTABLE,
|
||||
"net_version": CacheTier.IMMUTABLE,
|
||||
"web3_clientVersion": CacheTier.LONG,
|
||||
"eth_protocolVersion": CacheTier.LONG,
|
||||
"eth_syncing": CacheTier.MEDIUM,
|
||||
# Block by number (for finalized blocks - we check finality)
|
||||
"eth_getBlockByNumber": CacheTier.LONG,
|
||||
"eth_getBlockTransactionCountByNumber": CacheTier.LONG,
|
||||
"eth_getUncleCountByBlockNumber": CacheTier.LONG,
|
||||
"eth_getTransactionByBlockNumberAndIndex": CacheTier.LONG,
|
||||
"eth_getUncleByBlockNumberAndIndex": CacheTier.LONG,
|
||||
# Logs by block hash (immutable) - we'll check in handler
|
||||
"eth_getLogs": CacheTier.MEDIUM,
|
||||
|
||||
# === MEDIUM: Changes with each block ===
|
||||
"eth_blockNumber": CacheTier.REALTIME,
|
||||
"eth_gasPrice": CacheTier.MEDIUM,
|
||||
"eth_maxPriorityFeePerGas": CacheTier.MEDIUM,
|
||||
"eth_feeHistory": CacheTier.MEDIUM,
|
||||
"eth_getStorageAt": CacheTier.MEDIUM,
|
||||
|
||||
# === SHORT: Changes frequently ===
|
||||
"eth_accounts": CacheTier.SHORT,
|
||||
"eth_hashrate": CacheTier.SHORT,
|
||||
"eth_mining": CacheTier.SHORT,
|
||||
|
||||
# === NO CACHE: State-dependent or write operations ===
|
||||
"eth_getBalance": CacheTier.NO_CACHE, # Balance changes with txs
|
||||
"eth_getTransactionCount": CacheTier.NO_CACHE, # Nonce changes
|
||||
"eth_call": CacheTier.NO_CACHE, # State-dependent
|
||||
"eth_estimateGas": CacheTier.NO_CACHE, # State-dependent
|
||||
"eth_sendRawTransaction": CacheTier.NO_CACHE, # Write
|
||||
"eth_sendTransaction": CacheTier.NO_CACHE, # Write
|
||||
"eth_sign": CacheTier.NO_CACHE, # User-specific
|
||||
"eth_signTransaction": CacheTier.NO_CACHE, # User-specific
|
||||
"personal_sign": CacheTier.NO_CACHE, # User-specific
|
||||
"eth_getFilterChanges": CacheTier.NO_CACHE, # Stateful
|
||||
"eth_getFilterLogs": CacheTier.NO_CACHE, # Stateful
|
||||
"eth_newFilter": CacheTier.NO_CACHE, # Stateful
|
||||
"eth_newBlockFilter": CacheTier.NO_CACHE, # Stateful
|
||||
"eth_newPendingTransactionFilter": CacheTier.NO_CACHE, # Stateful
|
||||
"eth_uninstallFilter": CacheTier.NO_CACHE, # Stateful
|
||||
"eth_subscribe": CacheTier.NO_CACHE, # WebSocket
|
||||
"eth_unsubscribe": CacheTier.NO_CACHE, # WebSocket
|
||||
|
||||
# === Debug/Trace - LONG cache (expensive, deterministic) ===
|
||||
"debug_traceTransaction": CacheTier.IMMUTABLE, # By hash, immutable
|
||||
"debug_traceBlockByHash": CacheTier.IMMUTABLE,
|
||||
"debug_traceBlockByNumber": CacheTier.LONG,
|
||||
"trace_transaction": CacheTier.IMMUTABLE,
|
||||
"trace_block": CacheTier.LONG,
|
||||
"trace_replayTransaction": CacheTier.IMMUTABLE,
|
||||
"trace_replayBlockTransactions": CacheTier.LONG,
|
||||
|
||||
# === ERC-4337 Bundler ===
|
||||
"eth_sendUserOperation": CacheTier.NO_CACHE, # Write
|
||||
"eth_estimateUserOperationGas": CacheTier.NO_CACHE, # State-dependent
|
||||
"eth_getUserOperationByHash": CacheTier.IMMUTABLE, # By hash
|
||||
"eth_getUserOperationReceipt": CacheTier.IMMUTABLE, # By hash
|
||||
"eth_supportedEntryPoints": CacheTier.LONG,
|
||||
}
|
||||
|
||||
|
||||
def get_cache_tier(method: str, params: list | None = None) -> CacheTier:
|
||||
"""Determine cache tier for an RPC method + params."""
|
||||
tier = METHOD_CACHE_TIERS.get(method, CacheTier.SHORT)
|
||||
|
||||
# Special handling for block-number-based queries
|
||||
if params and tier in (CacheTier.LONG, CacheTier.MEDIUM):
|
||||
# Check if querying "latest", "pending", or "safe"
|
||||
block_param = _extract_block_param(method, params)
|
||||
if block_param in ("latest", "pending", "safe", "finalized"):
|
||||
# These change, use shorter cache
|
||||
return CacheTier.REALTIME if block_param == "pending" else CacheTier.MEDIUM
|
||||
|
||||
return tier
|
||||
|
||||
|
||||
def _extract_block_param(method: str, params: list) -> str | None:
|
||||
"""Extract block number/tag from params based on method."""
|
||||
if not params:
|
||||
return None
|
||||
|
||||
# Methods where first param is block identifier
|
||||
if method in ("eth_getBlockByNumber", "eth_getBlockByHash"):
|
||||
return str(params[0]) if params else None
|
||||
|
||||
# Methods where second param is block identifier
|
||||
if method in (
|
||||
"eth_getBalance",
|
||||
"eth_getCode",
|
||||
"eth_getTransactionCount",
|
||||
"eth_getStorageAt",
|
||||
"eth_call",
|
||||
):
|
||||
return str(params[1]) if len(params) > 1 else "latest"
|
||||
|
||||
# eth_getLogs uses fromBlock/toBlock in filter object
|
||||
if method == "eth_getLogs" and isinstance(params[0], dict):
|
||||
filter_obj = params[0]
|
||||
if filter_obj.get("blockHash"):
|
||||
return "hash" # By hash, immutable
|
||||
return filter_obj.get("toBlock", "latest")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def generate_cache_key(
|
||||
chain: str,
|
||||
network: str,
|
||||
method: str,
|
||||
params: list | None = None,
|
||||
) -> str:
|
||||
"""Generate a cache key for an RPC request.
|
||||
|
||||
IMPORTANT: No API key or account in the key!
|
||||
Blockchain data is public - same for everyone.
|
||||
"""
|
||||
# Normalize params for consistent hashing
|
||||
params_str = json.dumps(params or [], sort_keys=True, separators=(",", ":"))
|
||||
params_hash = hashlib.sha256(params_str.encode()).hexdigest()[:16]
|
||||
|
||||
# Format: rpc:{chain}:{network}:{method}:{params_hash}
|
||||
return f"rpc:{chain}:{network}:{method}:{params_hash}"
|
||||
|
||||
|
||||
def generate_cloudflare_cache_headers(
|
||||
tier: CacheTier,
|
||||
chain: str,
|
||||
method: str,
|
||||
) -> dict[str, str]:
|
||||
"""Generate HTTP headers for Cloudflare caching.
|
||||
|
||||
Uses:
|
||||
- Cache-Control for TTL
|
||||
- CDN-Cache-Control for Cloudflare-specific behavior
|
||||
- Cache-Tag for targeted purging
|
||||
- Vary header to ensure proper cache keying
|
||||
"""
|
||||
ttl = CACHE_TTLS[tier]
|
||||
|
||||
if tier == CacheTier.NO_CACHE:
|
||||
return {
|
||||
"Cache-Control": "no-store, no-cache, must-revalidate",
|
||||
"CDN-Cache-Control": "no-store",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
|
||||
headers = {
|
||||
# Browser cache (shorter) + CDN cache (longer)
|
||||
"Cache-Control": f"public, max-age={min(ttl, 60)}, s-maxage={ttl}",
|
||||
# Cloudflare-specific: cache for full TTL
|
||||
"CDN-Cache-Control": f"max-age={ttl}",
|
||||
# Cache tags for targeted purging
|
||||
"Cache-Tag": f"rpc,chain-{chain},method-{method},tier-{tier.value}",
|
||||
# Stale-while-revalidate for better UX
|
||||
"Surrogate-Control": f"max-age={ttl}, stale-while-revalidate={ttl // 2}",
|
||||
}
|
||||
|
||||
if tier == CacheTier.IMMUTABLE:
|
||||
headers["Cache-Control"] = f"public, max-age={ttl}, immutable"
|
||||
headers["CDN-Cache-Control"] = f"max-age={ttl}, immutable"
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
class CloudflareCacheConfig:
|
||||
"""Cloudflare cache configuration for workers/pages."""
|
||||
|
||||
# Cloudflare zones/domains
|
||||
ZONE_ID_ENV = "CLOUDFLARE_ZONE_ID"
|
||||
API_TOKEN_ENV = "CLOUDFLARE_API_TOKEN"
|
||||
|
||||
# Cache settings
|
||||
CACHE_EVERYTHING = True
|
||||
EDGE_TTL_DEFAULT = 3600 # 1 hour default edge TTL
|
||||
BROWSER_TTL_DEFAULT = 60 # 1 minute browser TTL
|
||||
|
||||
# Cache bypass conditions
|
||||
BYPASS_METHODS = {
|
||||
"eth_sendRawTransaction",
|
||||
"eth_sendTransaction",
|
||||
"eth_sign",
|
||||
"eth_signTransaction",
|
||||
"personal_sign",
|
||||
"eth_subscribe",
|
||||
"eth_unsubscribe",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_page_rule_config(cls) -> dict:
|
||||
"""Get Cloudflare page rule configuration for RPC endpoints."""
|
||||
return {
|
||||
"targets": [
|
||||
{"target": "url", "constraint": {"operator": "matches", "value": "*rpc.hanzo.ai/*"}}
|
||||
],
|
||||
"actions": [
|
||||
{"id": "cache_level", "value": "cache_everything"},
|
||||
{"id": "edge_cache_ttl", "value": cls.EDGE_TTL_DEFAULT},
|
||||
{"id": "browser_cache_ttl", "value": cls.BROWSER_TTL_DEFAULT},
|
||||
{"id": "cache_deception_armor", "value": "on"},
|
||||
{"id": "origin_cache_control", "value": "on"},
|
||||
],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_cache_rules(cls) -> list[dict]:
|
||||
"""Get Cloudflare cache rules for fine-grained control."""
|
||||
return [
|
||||
# Rule 1: Immutable data (by hash) - cache forever
|
||||
{
|
||||
"name": "RPC Immutable Data",
|
||||
"expression": '(http.request.uri.path contains "/rpc/") and (http.request.body contains "ByHash")',
|
||||
"action": "cache",
|
||||
"action_parameters": {
|
||||
"edge_ttl": {"mode": "override", "default": 31536000},
|
||||
"browser_ttl": {"mode": "override", "default": 86400},
|
||||
"cache_key": {
|
||||
"custom_key": {
|
||||
"query_string": {"include": "*"},
|
||||
"header": {"include": ["Content-Type"]},
|
||||
"host": {"resolved": True},
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
# Rule 2: Chain metadata - cache 1 day
|
||||
{
|
||||
"name": "RPC Chain Metadata",
|
||||
"expression": '(http.request.uri.path contains "/rpc/") and (http.request.body contains "eth_chainId" or http.request.body contains "net_version")',
|
||||
"action": "cache",
|
||||
"action_parameters": {
|
||||
"edge_ttl": {"mode": "override", "default": 86400},
|
||||
"browser_ttl": {"mode": "override", "default": 3600},
|
||||
},
|
||||
},
|
||||
# Rule 3: Write operations - bypass cache
|
||||
{
|
||||
"name": "RPC Write Operations",
|
||||
"expression": '(http.request.uri.path contains "/rpc/") and (http.request.body contains "sendRawTransaction" or http.request.body contains "sendTransaction")',
|
||||
"action": "bypass",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# Cloudflare Worker script for edge caching
|
||||
CLOUDFLARE_WORKER_SCRIPT = '''
|
||||
// Cloudflare Worker for aggressive RPC caching
|
||||
// Shares cache across ALL accounts - blockchain data is public!
|
||||
|
||||
const CACHE_TIERS = {
|
||||
immutable: 31536000, // 1 year
|
||||
long: 3600, // 1 hour
|
||||
medium: 30, // 30 seconds
|
||||
short: 5, // 5 seconds
|
||||
realtime: 2, // 2 seconds
|
||||
};
|
||||
|
||||
const IMMUTABLE_METHODS = new Set([
|
||||
'eth_getBlockByHash',
|
||||
'eth_getTransactionByHash',
|
||||
'eth_getTransactionReceipt',
|
||||
'eth_getCode',
|
||||
'eth_chainId',
|
||||
'net_version',
|
||||
'debug_traceTransaction',
|
||||
'trace_transaction',
|
||||
]);
|
||||
|
||||
const NO_CACHE_METHODS = new Set([
|
||||
'eth_sendRawTransaction',
|
||||
'eth_sendTransaction',
|
||||
'eth_call',
|
||||
'eth_estimateGas',
|
||||
'eth_getBalance',
|
||||
'eth_getTransactionCount',
|
||||
'eth_sign',
|
||||
'eth_subscribe',
|
||||
]);
|
||||
|
||||
async function handleRequest(request) {
|
||||
// Only cache POST requests to RPC endpoints
|
||||
if (request.method !== 'POST') {
|
||||
return fetch(request);
|
||||
}
|
||||
|
||||
const url = new URL(request.url);
|
||||
if (!url.pathname.includes('/rpc/')) {
|
||||
return fetch(request);
|
||||
}
|
||||
|
||||
// Parse RPC request
|
||||
const body = await request.clone().json();
|
||||
const method = body.method;
|
||||
const params = body.params || [];
|
||||
|
||||
// Check if cacheable
|
||||
if (NO_CACHE_METHODS.has(method)) {
|
||||
return fetch(request);
|
||||
}
|
||||
|
||||
// Generate cache key (no API key - shared across all accounts!)
|
||||
const paramsHash = await hashParams(params);
|
||||
const pathParts = url.pathname.split('/');
|
||||
const chain = pathParts[pathParts.indexOf('rpc') + 1];
|
||||
const network = pathParts[pathParts.indexOf('rpc') + 2] || 'mainnet';
|
||||
const cacheKey = `rpc:${chain}:${network}:${method}:${paramsHash}`;
|
||||
|
||||
// Check cache
|
||||
const cache = caches.default;
|
||||
const cacheRequest = new Request(url.origin + '/__cache/' + cacheKey, {
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
let response = await cache.match(cacheRequest);
|
||||
if (response) {
|
||||
// Cache hit! Add header to indicate
|
||||
const newResponse = new Response(response.body, response);
|
||||
newResponse.headers.set('X-Cache', 'HIT');
|
||||
newResponse.headers.set('X-Cache-Key', cacheKey);
|
||||
return newResponse;
|
||||
}
|
||||
|
||||
// Cache miss - fetch from origin
|
||||
response = await fetch(request);
|
||||
|
||||
if (response.ok) {
|
||||
// Determine TTL
|
||||
const tier = IMMUTABLE_METHODS.has(method) ? 'immutable' :
|
||||
method.includes('Block') ? 'long' : 'medium';
|
||||
const ttl = CACHE_TIERS[tier];
|
||||
|
||||
// Clone and cache
|
||||
const responseToCache = new Response(response.clone().body, response);
|
||||
responseToCache.headers.set('Cache-Control', `public, max-age=${ttl}`);
|
||||
responseToCache.headers.set('X-Cache', 'MISS');
|
||||
responseToCache.headers.set('X-Cache-Key', cacheKey);
|
||||
responseToCache.headers.set('X-Cache-TTL', ttl.toString());
|
||||
|
||||
// Store in cache (async, don't wait)
|
||||
const cacheResponse = new Response(responseToCache.clone().body, {
|
||||
headers: {
|
||||
'Cache-Control': `public, max-age=${ttl}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
});
|
||||
event.waitUntil(cache.put(cacheRequest, cacheResponse));
|
||||
|
||||
return responseToCache;
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
async function hashParams(params) {
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(JSON.stringify(params));
|
||||
const hashBuffer = await crypto.subtle.digest('SHA-256', data);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
return hashArray.slice(0, 8).map(b => b.toString(16).padStart(2, '0')).join('');
|
||||
}
|
||||
|
||||
addEventListener('fetch', event => {
|
||||
event.respondWith(handleRequest(event.request));
|
||||
});
|
||||
'''
|
||||
|
||||
|
||||
def get_worker_script() -> str:
|
||||
"""Get Cloudflare Worker script for deployment."""
|
||||
return CLOUDFLARE_WORKER_SCRIPT
|
||||
+339
@@ -0,0 +1,339 @@
|
||||
"""Multi-tier caching: Redis (L1) + Cloudflare (L2) for planet-scale RPC.
|
||||
|
||||
Architecture:
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ Client Request │
|
||||
└─────────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ Cloudflare Edge (L2) │
|
||||
│ - 300+ global PoPs │
|
||||
│ - Cache SHARED across ALL accounts (blockchain data is public) │
|
||||
│ - Serves 99%+ of requests from edge │
|
||||
└─────────────────────────────────┬────────────────────────────────────┘
|
||||
│ (cache miss)
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ Redis Cluster (L1) │
|
||||
│ - Regional caches (us-east, eu-west, ap-northeast) │
|
||||
│ - Sub-millisecond lookups │
|
||||
│ - Reduces origin load │
|
||||
└─────────────────────────────────┬────────────────────────────────────┘
|
||||
│ (cache miss)
|
||||
▼
|
||||
┌──────────────────────────────────────────────────────────────────────┐
|
||||
│ Origin RPC │
|
||||
│ - Load balanced across providers │
|
||||
│ - Fallback to public endpoints │
|
||||
│ - Response cached in L1 + returned to L2 │
|
||||
└──────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
Cost Analysis:
|
||||
- Cloudflare Free/Pro: 100B+ requests/month at edge
|
||||
- Redis: Only receives 1-5% of traffic (cache misses)
|
||||
- Origin RPC: Only receives 0.1-1% of traffic
|
||||
- Result: Can serve ALL 100+ chains essentially for FREE
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from bootnode.core.cache.cloudflare import (
|
||||
CacheTier,
|
||||
get_cache_tier,
|
||||
generate_cache_key,
|
||||
generate_cloudflare_cache_headers,
|
||||
CACHE_TTLS,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheStats:
|
||||
"""Cache hit/miss statistics."""
|
||||
l1_hits: int = 0
|
||||
l1_misses: int = 0
|
||||
l2_hits: int = 0
|
||||
l2_misses: int = 0
|
||||
origin_calls: int = 0
|
||||
total_requests: int = 0
|
||||
|
||||
@property
|
||||
def l1_hit_rate(self) -> float:
|
||||
return self.l1_hits / max(self.total_requests, 1)
|
||||
|
||||
@property
|
||||
def l2_hit_rate(self) -> float:
|
||||
return self.l2_hits / max(self.total_requests, 1)
|
||||
|
||||
@property
|
||||
def total_hit_rate(self) -> float:
|
||||
return (self.l1_hits + self.l2_hits) / max(self.total_requests, 1)
|
||||
|
||||
|
||||
class MultiTierCache:
|
||||
"""Multi-tier cache with Redis L1 and Cloudflare L2."""
|
||||
|
||||
def __init__(self, redis_client: Any = None):
|
||||
self.redis = redis_client
|
||||
self.stats = CacheStats()
|
||||
self._local_cache: dict[str, tuple[Any, float]] = {} # In-memory L0
|
||||
self._local_cache_max = 10000 # Max items in local cache
|
||||
|
||||
async def get(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
method: str,
|
||||
params: list | None = None,
|
||||
) -> tuple[Any | None, CacheTier, dict[str, str]]:
|
||||
"""Get cached RPC response.
|
||||
|
||||
Returns: (cached_value, cache_tier, cloudflare_headers)
|
||||
"""
|
||||
self.stats.total_requests += 1
|
||||
|
||||
tier = get_cache_tier(method, params)
|
||||
headers = generate_cloudflare_cache_headers(tier, chain, method)
|
||||
|
||||
# No cache for this method
|
||||
if tier == CacheTier.NO_CACHE:
|
||||
return None, tier, headers
|
||||
|
||||
cache_key = generate_cache_key(chain, network, method, params)
|
||||
|
||||
# L0: In-memory cache (hot data)
|
||||
if cached := self._get_local(cache_key, tier):
|
||||
self.stats.l1_hits += 1
|
||||
logger.debug("L0 cache hit", key=cache_key)
|
||||
return cached, tier, headers
|
||||
|
||||
# L1: Redis cache
|
||||
if self.redis:
|
||||
try:
|
||||
cached = await self.redis.get(cache_key)
|
||||
if cached:
|
||||
self.stats.l1_hits += 1
|
||||
# Populate L0
|
||||
self._set_local(cache_key, cached, tier)
|
||||
logger.debug("L1 cache hit", key=cache_key)
|
||||
return json.loads(cached), tier, headers
|
||||
except Exception as e:
|
||||
logger.warning("Redis get failed", error=str(e))
|
||||
|
||||
self.stats.l1_misses += 1
|
||||
return None, tier, headers
|
||||
|
||||
async def set(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
method: str,
|
||||
params: list | None,
|
||||
value: Any,
|
||||
) -> None:
|
||||
"""Store RPC response in cache."""
|
||||
tier = get_cache_tier(method, params)
|
||||
|
||||
if tier == CacheTier.NO_CACHE:
|
||||
return
|
||||
|
||||
cache_key = generate_cache_key(chain, network, method, params)
|
||||
ttl = CACHE_TTLS[tier]
|
||||
|
||||
# Set in L0 (local)
|
||||
self._set_local(cache_key, value, tier)
|
||||
|
||||
# Set in L1 (Redis)
|
||||
if self.redis:
|
||||
try:
|
||||
await self.redis.set(
|
||||
cache_key,
|
||||
json.dumps(value),
|
||||
ex=ttl,
|
||||
)
|
||||
logger.debug("Cached in L1", key=cache_key, ttl=ttl)
|
||||
except Exception as e:
|
||||
logger.warning("Redis set failed", error=str(e))
|
||||
|
||||
def _get_local(self, key: str, tier: CacheTier) -> Any | None:
|
||||
"""Get from local in-memory cache."""
|
||||
if key in self._local_cache:
|
||||
value, expires_at = self._local_cache[key]
|
||||
if time.time() < expires_at:
|
||||
return value
|
||||
# Expired, remove
|
||||
del self._local_cache[key]
|
||||
return None
|
||||
|
||||
def _set_local(self, key: str, value: Any, tier: CacheTier) -> None:
|
||||
"""Set in local in-memory cache."""
|
||||
# Evict if at capacity
|
||||
if len(self._local_cache) >= self._local_cache_max:
|
||||
# Remove oldest 10%
|
||||
to_remove = list(self._local_cache.keys())[:self._local_cache_max // 10]
|
||||
for k in to_remove:
|
||||
del self._local_cache[k]
|
||||
|
||||
ttl = min(CACHE_TTLS[tier], 300) # Max 5 min in local cache
|
||||
self._local_cache[key] = (value, time.time() + ttl)
|
||||
|
||||
async def invalidate(
|
||||
self,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
method: str | None = None,
|
||||
) -> int:
|
||||
"""Invalidate cache entries by pattern.
|
||||
|
||||
Returns number of keys invalidated.
|
||||
"""
|
||||
pattern_parts = ["rpc"]
|
||||
if chain:
|
||||
pattern_parts.append(chain)
|
||||
if network:
|
||||
pattern_parts.append(network)
|
||||
if method:
|
||||
pattern_parts.append(method)
|
||||
|
||||
pattern = ":".join(pattern_parts) + ":*"
|
||||
count = 0
|
||||
|
||||
# Clear local cache
|
||||
keys_to_remove = [k for k in self._local_cache if k.startswith(pattern.replace("*", ""))]
|
||||
for k in keys_to_remove:
|
||||
del self._local_cache[k]
|
||||
count += 1
|
||||
|
||||
# Clear Redis
|
||||
if self.redis:
|
||||
try:
|
||||
async for key in self.redis.scan_iter(pattern):
|
||||
await self.redis.delete(key)
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.warning("Redis invalidate failed", error=str(e))
|
||||
|
||||
logger.info("Cache invalidated", pattern=pattern, count=count)
|
||||
return count
|
||||
|
||||
def get_stats(self) -> dict[str, Any]:
|
||||
"""Get cache statistics."""
|
||||
return {
|
||||
"l1_hits": self.stats.l1_hits,
|
||||
"l1_misses": self.stats.l1_misses,
|
||||
"l2_hits": self.stats.l2_hits,
|
||||
"l2_misses": self.stats.l2_misses,
|
||||
"origin_calls": self.stats.origin_calls,
|
||||
"total_requests": self.stats.total_requests,
|
||||
"l1_hit_rate": f"{self.stats.l1_hit_rate:.2%}",
|
||||
"l2_hit_rate": f"{self.stats.l2_hit_rate:.2%}",
|
||||
"total_hit_rate": f"{self.stats.total_hit_rate:.2%}",
|
||||
"local_cache_size": len(self._local_cache),
|
||||
}
|
||||
|
||||
|
||||
class BatchCacheOptimizer:
|
||||
"""Optimize batch RPC requests for caching.
|
||||
|
||||
Strategy:
|
||||
1. Deduplicate requests in batch
|
||||
2. Check cache for each unique request
|
||||
3. Only fetch cache misses from origin
|
||||
4. Reconstruct batch response
|
||||
"""
|
||||
|
||||
def __init__(self, cache: MultiTierCache):
|
||||
self.cache = cache
|
||||
|
||||
async def process_batch(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
requests: list[dict],
|
||||
) -> tuple[list[dict], list[int], dict[str, str]]:
|
||||
"""Process a batch of RPC requests with caching.
|
||||
|
||||
Returns:
|
||||
- requests_to_fetch: Requests that need to go to origin
|
||||
- cached_indices: Indices of requests that were cached
|
||||
- headers: Cloudflare cache headers
|
||||
"""
|
||||
cached_responses: dict[int, Any] = {}
|
||||
requests_to_fetch: list[tuple[int, dict]] = []
|
||||
headers: dict[str, str] = {}
|
||||
|
||||
# Check cache for each request
|
||||
for i, req in enumerate(requests):
|
||||
method = req.get("method", "")
|
||||
params = req.get("params", [])
|
||||
|
||||
cached, tier, req_headers = await self.cache.get(
|
||||
chain, network, method, params
|
||||
)
|
||||
|
||||
if cached is not None:
|
||||
# Build JSON-RPC response
|
||||
cached_responses[i] = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": req.get("id"),
|
||||
"result": cached,
|
||||
}
|
||||
else:
|
||||
requests_to_fetch.append((i, req))
|
||||
|
||||
# Merge headers (use most restrictive)
|
||||
headers.update(req_headers)
|
||||
|
||||
return (
|
||||
[req for _, req in requests_to_fetch],
|
||||
cached_responses,
|
||||
[idx for idx, _ in requests_to_fetch],
|
||||
headers,
|
||||
)
|
||||
|
||||
async def merge_responses(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
requests: list[dict],
|
||||
cached_responses: dict[int, Any],
|
||||
fetch_indices: list[int],
|
||||
fetched_responses: list[dict],
|
||||
) -> list[dict]:
|
||||
"""Merge cached and fetched responses, caching new results."""
|
||||
# Cache fetched responses
|
||||
for idx, (fetch_idx, response) in enumerate(zip(fetch_indices, fetched_responses)):
|
||||
if "result" in response:
|
||||
req = requests[fetch_idx]
|
||||
await self.cache.set(
|
||||
chain,
|
||||
network,
|
||||
req.get("method", ""),
|
||||
req.get("params", []),
|
||||
response["result"],
|
||||
)
|
||||
cached_responses[fetch_idx] = response
|
||||
|
||||
# Reconstruct in original order
|
||||
return [cached_responses[i] for i in range(len(requests))]
|
||||
|
||||
|
||||
# Global cache instance
|
||||
_multi_tier_cache: MultiTierCache | None = None
|
||||
|
||||
|
||||
def get_multi_tier_cache(redis_client: Any = None) -> MultiTierCache:
|
||||
"""Get or create multi-tier cache singleton."""
|
||||
global _multi_tier_cache
|
||||
if _multi_tier_cache is None:
|
||||
_multi_tier_cache = MultiTierCache(redis_client)
|
||||
return _multi_tier_cache
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Omnichain Query Layer - Unified access to 100+ chains.
|
||||
|
||||
Storage Architecture:
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ Query Layer │
|
||||
├─────────────┬─────────────┬─────────────┬─────────────┬─────────────────┤
|
||||
│ PostgreSQL │ ClickHouse │ Qdrant │ Meilisearch │ GraphQL │
|
||||
│ (Metadata) │ (Analytics) │ (Vector) │ (Full-Text) │ (API) │
|
||||
├─────────────┴─────────────┴─────────────┴─────────────┴─────────────────┤
|
||||
│ Lux Indexer │
|
||||
│ (100+ chains indexed) │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
Components:
|
||||
- PostgreSQL: Relational metadata, user data, subscriptions
|
||||
- ClickHouse: Time-series analytics, usage tracking, chain metrics
|
||||
- Qdrant: Vector search for semantic queries (contracts, transactions)
|
||||
- Meilisearch: Full-text search for addresses, tokens, contracts
|
||||
- GraphQL: Unified API for all query types
|
||||
"""
|
||||
|
||||
from bootnode.core.query.unified import (
|
||||
UnifiedQueryClient,
|
||||
QueryResult,
|
||||
SearchResult,
|
||||
get_query_client,
|
||||
)
|
||||
from bootnode.core.query.vector import (
|
||||
VectorStore,
|
||||
get_vector_store,
|
||||
)
|
||||
from bootnode.core.query.search import (
|
||||
SearchEngine,
|
||||
get_search_engine,
|
||||
)
|
||||
from bootnode.core.query.analytics import (
|
||||
AnalyticsEngine,
|
||||
get_analytics_engine,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"UnifiedQueryClient",
|
||||
"QueryResult",
|
||||
"SearchResult",
|
||||
"get_query_client",
|
||||
"VectorStore",
|
||||
"get_vector_store",
|
||||
"SearchEngine",
|
||||
"get_search_engine",
|
||||
"AnalyticsEngine",
|
||||
"get_analytics_engine",
|
||||
]
|
||||
@@ -0,0 +1,541 @@
|
||||
"""Analytics Engine with ClickHouse - Time-series analytics for blockchain data.
|
||||
|
||||
Provides:
|
||||
- Chain metrics and statistics
|
||||
- Historical price/volume data
|
||||
- DeFi protocol analytics
|
||||
- User activity analytics
|
||||
- Real-time dashboards
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, date, timedelta
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
from bootnode.core.datastore.client import DataStoreClient, get_datastore
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimeSeriesPoint:
|
||||
"""Single point in time series."""
|
||||
timestamp: datetime
|
||||
value: float
|
||||
metadata: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimeSeriesData:
|
||||
"""Time series data response."""
|
||||
points: list[TimeSeriesPoint]
|
||||
chain: str
|
||||
network: str
|
||||
metric: str
|
||||
interval: str
|
||||
took_ms: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class AggregateMetric:
|
||||
"""Aggregated metric."""
|
||||
name: str
|
||||
value: float
|
||||
change_24h: float | None = None
|
||||
change_7d: float | None = None
|
||||
|
||||
|
||||
class AnalyticsEngine:
|
||||
"""ClickHouse analytics engine for blockchain metrics."""
|
||||
|
||||
def __init__(self, datastore: DataStoreClient | None = None) -> None:
|
||||
self.datastore = datastore or get_datastore()
|
||||
|
||||
# =========================================================================
|
||||
# Chain Metrics
|
||||
# =========================================================================
|
||||
|
||||
async def get_chain_metrics(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
) -> dict[str, AggregateMetric]:
|
||||
"""Get current metrics for a chain."""
|
||||
start = datetime.now()
|
||||
|
||||
# Query latest metrics from ClickHouse
|
||||
query = """
|
||||
SELECT
|
||||
metric_name,
|
||||
metric_value,
|
||||
timestamp
|
||||
FROM chain_metrics
|
||||
WHERE chain = {chain:String}
|
||||
AND network = {network:String}
|
||||
AND timestamp >= now() - INTERVAL 1 DAY
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 100
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(
|
||||
query,
|
||||
params={"chain": chain, "network": network},
|
||||
)
|
||||
|
||||
metrics = {}
|
||||
for row in result:
|
||||
name = row["metric_name"]
|
||||
if name not in metrics:
|
||||
metrics[name] = AggregateMetric(
|
||||
name=name,
|
||||
value=row["metric_value"],
|
||||
)
|
||||
|
||||
return metrics
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get chain metrics", error=str(e))
|
||||
return {}
|
||||
|
||||
async def get_block_time_history(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
days: int = 7,
|
||||
interval: str = "1h",
|
||||
) -> TimeSeriesData:
|
||||
"""Get historical block time data."""
|
||||
start = datetime.now()
|
||||
|
||||
# Interval to ClickHouse format
|
||||
ch_interval = {
|
||||
"1m": "toStartOfMinute",
|
||||
"5m": "toStartOfFiveMinutes",
|
||||
"15m": "toStartOfFifteenMinutes",
|
||||
"1h": "toStartOfHour",
|
||||
"1d": "toStartOfDay",
|
||||
}.get(interval, "toStartOfHour")
|
||||
|
||||
query = f"""
|
||||
SELECT
|
||||
{ch_interval}(timestamp) as ts,
|
||||
avg(block_time) as avg_block_time
|
||||
FROM blocks
|
||||
WHERE chain = {{chain:String}}
|
||||
AND network = {{network:String}}
|
||||
AND timestamp >= now() - INTERVAL {{days:Int32}} DAY
|
||||
GROUP BY ts
|
||||
ORDER BY ts
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(
|
||||
query,
|
||||
params={"chain": chain, "network": network, "days": days},
|
||||
)
|
||||
|
||||
points = [
|
||||
TimeSeriesPoint(
|
||||
timestamp=row["ts"],
|
||||
value=row["avg_block_time"],
|
||||
)
|
||||
for row in result
|
||||
]
|
||||
|
||||
return TimeSeriesData(
|
||||
points=points,
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="block_time",
|
||||
interval=interval,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get block time history", error=str(e))
|
||||
return TimeSeriesData(
|
||||
points=[],
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="block_time",
|
||||
interval=interval,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
|
||||
async def get_transaction_volume(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
days: int = 30,
|
||||
interval: str = "1d",
|
||||
) -> TimeSeriesData:
|
||||
"""Get historical transaction volume."""
|
||||
start = datetime.now()
|
||||
|
||||
ch_interval = {
|
||||
"1h": "toStartOfHour",
|
||||
"1d": "toStartOfDay",
|
||||
"1w": "toStartOfWeek",
|
||||
}.get(interval, "toStartOfDay")
|
||||
|
||||
query = f"""
|
||||
SELECT
|
||||
{ch_interval}(timestamp) as ts,
|
||||
count() as tx_count,
|
||||
sum(value) as total_value,
|
||||
sum(gas_used * gas_price) as total_fees
|
||||
FROM transactions
|
||||
WHERE chain = {{chain:String}}
|
||||
AND network = {{network:String}}
|
||||
AND timestamp >= now() - INTERVAL {{days:Int32}} DAY
|
||||
GROUP BY ts
|
||||
ORDER BY ts
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(
|
||||
query,
|
||||
params={"chain": chain, "network": network, "days": days},
|
||||
)
|
||||
|
||||
points = [
|
||||
TimeSeriesPoint(
|
||||
timestamp=row["ts"],
|
||||
value=row["tx_count"],
|
||||
metadata={
|
||||
"total_value": row["total_value"],
|
||||
"total_fees": row["total_fees"],
|
||||
},
|
||||
)
|
||||
for row in result
|
||||
]
|
||||
|
||||
return TimeSeriesData(
|
||||
points=points,
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="transaction_volume",
|
||||
interval=interval,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get tx volume", error=str(e))
|
||||
return TimeSeriesData(
|
||||
points=[],
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="transaction_volume",
|
||||
interval=interval,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
|
||||
async def get_gas_price_history(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
hours: int = 24,
|
||||
) -> TimeSeriesData:
|
||||
"""Get gas price history."""
|
||||
start = datetime.now()
|
||||
|
||||
query = """
|
||||
SELECT
|
||||
toStartOfFiveMinutes(timestamp) as ts,
|
||||
avg(gas_price) as avg_gas,
|
||||
min(gas_price) as min_gas,
|
||||
max(gas_price) as max_gas,
|
||||
quantile(0.5)(gas_price) as median_gas
|
||||
FROM transactions
|
||||
WHERE chain = {chain:String}
|
||||
AND network = {network:String}
|
||||
AND timestamp >= now() - INTERVAL {hours:Int32} HOUR
|
||||
GROUP BY ts
|
||||
ORDER BY ts
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(
|
||||
query,
|
||||
params={"chain": chain, "network": network, "hours": hours},
|
||||
)
|
||||
|
||||
points = [
|
||||
TimeSeriesPoint(
|
||||
timestamp=row["ts"],
|
||||
value=row["avg_gas"],
|
||||
metadata={
|
||||
"min": row["min_gas"],
|
||||
"max": row["max_gas"],
|
||||
"median": row["median_gas"],
|
||||
},
|
||||
)
|
||||
for row in result
|
||||
]
|
||||
|
||||
return TimeSeriesData(
|
||||
points=points,
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="gas_price",
|
||||
interval="5m",
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get gas price history", error=str(e))
|
||||
return TimeSeriesData(
|
||||
points=[],
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="gas_price",
|
||||
interval="5m",
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Token Analytics
|
||||
# =========================================================================
|
||||
|
||||
async def get_token_transfers(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
token_address: str,
|
||||
days: int = 7,
|
||||
) -> TimeSeriesData:
|
||||
"""Get token transfer volume over time."""
|
||||
start = datetime.now()
|
||||
|
||||
query = """
|
||||
SELECT
|
||||
toStartOfHour(timestamp) as ts,
|
||||
count() as transfer_count,
|
||||
countDistinct(from_address) as unique_senders,
|
||||
countDistinct(to_address) as unique_receivers,
|
||||
sum(value) as total_value
|
||||
FROM token_transfers
|
||||
WHERE chain = {chain:String}
|
||||
AND network = {network:String}
|
||||
AND token_address = {token:String}
|
||||
AND timestamp >= now() - INTERVAL {days:Int32} DAY
|
||||
GROUP BY ts
|
||||
ORDER BY ts
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(
|
||||
query,
|
||||
params={
|
||||
"chain": chain,
|
||||
"network": network,
|
||||
"token": token_address.lower(),
|
||||
"days": days,
|
||||
},
|
||||
)
|
||||
|
||||
points = [
|
||||
TimeSeriesPoint(
|
||||
timestamp=row["ts"],
|
||||
value=row["transfer_count"],
|
||||
metadata={
|
||||
"unique_senders": row["unique_senders"],
|
||||
"unique_receivers": row["unique_receivers"],
|
||||
"total_value": row["total_value"],
|
||||
},
|
||||
)
|
||||
for row in result
|
||||
]
|
||||
|
||||
return TimeSeriesData(
|
||||
points=points,
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric=f"token_transfers:{token_address}",
|
||||
interval="1h",
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get token transfers", error=str(e))
|
||||
return TimeSeriesData(
|
||||
points=[],
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric=f"token_transfers:{token_address}",
|
||||
interval="1h",
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
|
||||
async def get_top_tokens(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
metric: str = "transfers",
|
||||
limit: int = 50,
|
||||
) -> list[dict]:
|
||||
"""Get top tokens by specified metric."""
|
||||
order_by = {
|
||||
"transfers": "transfer_count",
|
||||
"holders": "holder_count",
|
||||
"volume": "total_volume",
|
||||
}.get(metric, "transfer_count")
|
||||
|
||||
query = f"""
|
||||
SELECT
|
||||
token_address,
|
||||
token_name,
|
||||
token_symbol,
|
||||
count() as transfer_count,
|
||||
countDistinct(from_address) + countDistinct(to_address) as holder_count,
|
||||
sum(value) as total_volume
|
||||
FROM token_transfers
|
||||
WHERE chain = {{chain:String}}
|
||||
AND network = {{network:String}}
|
||||
AND timestamp >= now() - INTERVAL 7 DAY
|
||||
GROUP BY token_address, token_name, token_symbol
|
||||
ORDER BY {order_by} DESC
|
||||
LIMIT {{limit:Int32}}
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(
|
||||
query,
|
||||
params={"chain": chain, "network": network, "limit": limit},
|
||||
)
|
||||
return list(result)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get top tokens", error=str(e))
|
||||
return []
|
||||
|
||||
# =========================================================================
|
||||
# DeFi Analytics
|
||||
# =========================================================================
|
||||
|
||||
async def get_defi_tvl(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
protocol: str | None = None,
|
||||
days: int = 30,
|
||||
) -> TimeSeriesData:
|
||||
"""Get DeFi TVL (Total Value Locked) over time."""
|
||||
start = datetime.now()
|
||||
|
||||
protocol_filter = "AND protocol = {protocol:String}" if protocol else ""
|
||||
|
||||
query = f"""
|
||||
SELECT
|
||||
toStartOfDay(timestamp) as ts,
|
||||
sum(tvl_usd) as total_tvl
|
||||
FROM defi_tvl
|
||||
WHERE chain = {{chain:String}}
|
||||
AND network = {{network:String}}
|
||||
{protocol_filter}
|
||||
AND timestamp >= now() - INTERVAL {{days:Int32}} DAY
|
||||
GROUP BY ts
|
||||
ORDER BY ts
|
||||
"""
|
||||
|
||||
params: dict[str, Any] = {
|
||||
"chain": chain,
|
||||
"network": network,
|
||||
"days": days,
|
||||
}
|
||||
if protocol:
|
||||
params["protocol"] = protocol
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(query, params=params)
|
||||
|
||||
points = [
|
||||
TimeSeriesPoint(
|
||||
timestamp=row["ts"],
|
||||
value=row["total_tvl"],
|
||||
)
|
||||
for row in result
|
||||
]
|
||||
|
||||
return TimeSeriesData(
|
||||
points=points,
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="defi_tvl",
|
||||
interval="1d",
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get DeFi TVL", error=str(e))
|
||||
return TimeSeriesData(
|
||||
points=[],
|
||||
chain=chain,
|
||||
network=network,
|
||||
metric="defi_tvl",
|
||||
interval="1d",
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Cross-Chain Analytics
|
||||
# =========================================================================
|
||||
|
||||
async def get_cross_chain_summary(self) -> dict[str, Any]:
|
||||
"""Get summary metrics across all chains."""
|
||||
query = """
|
||||
SELECT
|
||||
chain,
|
||||
network,
|
||||
count() as tx_count_24h,
|
||||
countDistinct(from_address) as unique_addresses_24h,
|
||||
sum(gas_used * gas_price) as total_fees_24h
|
||||
FROM transactions
|
||||
WHERE timestamp >= now() - INTERVAL 1 DAY
|
||||
GROUP BY chain, network
|
||||
ORDER BY tx_count_24h DESC
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(query)
|
||||
return {
|
||||
"chains": list(result),
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get cross-chain summary", error=str(e))
|
||||
return {"chains": [], "generated_at": datetime.now().isoformat()}
|
||||
|
||||
async def get_bridge_activity(
|
||||
self,
|
||||
days: int = 7,
|
||||
) -> list[dict]:
|
||||
"""Get cross-chain bridge activity."""
|
||||
query = """
|
||||
SELECT
|
||||
source_chain,
|
||||
dest_chain,
|
||||
count() as transfer_count,
|
||||
sum(amount_usd) as total_volume_usd,
|
||||
countDistinct(sender) as unique_users
|
||||
FROM bridge_transfers
|
||||
WHERE timestamp >= now() - INTERVAL {days:Int32} DAY
|
||||
GROUP BY source_chain, dest_chain
|
||||
ORDER BY total_volume_usd DESC
|
||||
LIMIT 100
|
||||
"""
|
||||
|
||||
try:
|
||||
result = await self.datastore.query(query, params={"days": days})
|
||||
return list(result)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to get bridge activity", error=str(e))
|
||||
return []
|
||||
|
||||
|
||||
# Global instance
|
||||
_analytics_engine: AnalyticsEngine | None = None
|
||||
|
||||
|
||||
def get_analytics_engine() -> AnalyticsEngine:
|
||||
"""Get analytics engine singleton."""
|
||||
global _analytics_engine
|
||||
if _analytics_engine is None:
|
||||
_analytics_engine = AnalyticsEngine()
|
||||
return _analytics_engine
|
||||
@@ -0,0 +1,747 @@
|
||||
"""GraphQL API for Omnichain Queries.
|
||||
|
||||
Provides a unified GraphQL interface for:
|
||||
- Blocks, transactions, addresses across 100+ chains
|
||||
- Full-text search via Meilisearch
|
||||
- Vector/semantic search via Qdrant
|
||||
- Analytics via ClickHouse
|
||||
- Real-time subscriptions via WebSocket
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import strawberry
|
||||
from strawberry.types import Info
|
||||
|
||||
from bootnode.core.query.unified import get_query_client, QueryResult
|
||||
from bootnode.core.query.search import get_search_engine, SearchResponse
|
||||
from bootnode.core.query.vector import get_vector_store, VectorSearchResponse
|
||||
from bootnode.core.query.analytics import get_analytics_engine, TimeSeriesData
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# GraphQL Types
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Block:
|
||||
"""Blockchain block."""
|
||||
hash: str
|
||||
number: int
|
||||
timestamp: datetime
|
||||
parent_hash: str
|
||||
transactions_count: int
|
||||
gas_used: int
|
||||
gas_limit: int
|
||||
base_fee_per_gas: int | None
|
||||
miner: str
|
||||
chain: str
|
||||
network: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Transaction:
|
||||
"""Blockchain transaction."""
|
||||
hash: str
|
||||
block_number: int
|
||||
block_hash: str
|
||||
from_address: str
|
||||
to_address: str | None
|
||||
value: str
|
||||
gas: int
|
||||
gas_price: str
|
||||
gas_used: int | None
|
||||
nonce: int
|
||||
input: str
|
||||
status: int | None
|
||||
timestamp: datetime
|
||||
chain: str
|
||||
network: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Address:
|
||||
"""Blockchain address with balance and activity."""
|
||||
address: str
|
||||
balance: str
|
||||
tx_count: int
|
||||
is_contract: bool
|
||||
token_count: int
|
||||
chain: str
|
||||
network: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Token:
|
||||
"""ERC20/721/1155 token."""
|
||||
address: str
|
||||
name: str
|
||||
symbol: str
|
||||
decimals: int
|
||||
total_supply: str
|
||||
token_type: str
|
||||
holders: int
|
||||
chain: str
|
||||
network: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Contract:
|
||||
"""Verified smart contract."""
|
||||
address: str
|
||||
name: str | None
|
||||
compiler: str | None
|
||||
optimization: bool
|
||||
verified: bool
|
||||
source_code: str | None
|
||||
abi: str | None
|
||||
chain: str
|
||||
network: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class SearchHit:
|
||||
"""Search result hit."""
|
||||
id: str
|
||||
type: str
|
||||
data: strawberry.scalars.JSON
|
||||
score: float | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class SearchResults:
|
||||
"""Search results."""
|
||||
hits: list[SearchHit]
|
||||
total: int
|
||||
query: str
|
||||
took_ms: float
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class VectorSearchHit:
|
||||
"""Vector search result."""
|
||||
id: str
|
||||
score: float
|
||||
data: strawberry.scalars.JSON
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class VectorResults:
|
||||
"""Vector search results."""
|
||||
hits: list[VectorSearchHit]
|
||||
total: int
|
||||
took_ms: float
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class TimeSeriesPoint:
|
||||
"""Time series data point."""
|
||||
timestamp: datetime
|
||||
value: float
|
||||
metadata: strawberry.scalars.JSON | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class TimeSeries:
|
||||
"""Time series data."""
|
||||
points: list[TimeSeriesPoint]
|
||||
metric: str
|
||||
interval: str
|
||||
chain: str
|
||||
network: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class ChainStats:
|
||||
"""Chain statistics."""
|
||||
chain: str
|
||||
network: str
|
||||
block_height: int
|
||||
tx_count_24h: int
|
||||
unique_addresses_24h: int
|
||||
avg_block_time: float
|
||||
avg_gas_price: str
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class PaginatedBlocks:
|
||||
"""Paginated blocks response."""
|
||||
items: list[Block]
|
||||
total: int
|
||||
cursor: str | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class PaginatedTransactions:
|
||||
"""Paginated transactions response."""
|
||||
items: list[Transaction]
|
||||
total: int
|
||||
cursor: str | None
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class PaginatedTokens:
|
||||
"""Paginated tokens response."""
|
||||
items: list[Token]
|
||||
total: int
|
||||
cursor: str | None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Resolvers
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _result_to_block(data: dict, chain: str, network: str) -> Block:
|
||||
"""Convert query result to Block type."""
|
||||
return Block(
|
||||
hash=data.get("hash", ""),
|
||||
number=data.get("number", 0),
|
||||
timestamp=data.get("timestamp", datetime.now()),
|
||||
parent_hash=data.get("parent_hash", ""),
|
||||
transactions_count=data.get("transactions_count", 0),
|
||||
gas_used=data.get("gas_used", 0),
|
||||
gas_limit=data.get("gas_limit", 0),
|
||||
base_fee_per_gas=data.get("base_fee_per_gas"),
|
||||
miner=data.get("miner", ""),
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
|
||||
def _result_to_transaction(data: dict, chain: str, network: str) -> Transaction:
|
||||
"""Convert query result to Transaction type."""
|
||||
return Transaction(
|
||||
hash=data.get("hash", ""),
|
||||
block_number=data.get("block_number", 0),
|
||||
block_hash=data.get("block_hash", ""),
|
||||
from_address=data.get("from", ""),
|
||||
to_address=data.get("to"),
|
||||
value=str(data.get("value", "0")),
|
||||
gas=data.get("gas", 0),
|
||||
gas_price=str(data.get("gas_price", "0")),
|
||||
gas_used=data.get("gas_used"),
|
||||
nonce=data.get("nonce", 0),
|
||||
input=data.get("input", "0x"),
|
||||
status=data.get("status"),
|
||||
timestamp=data.get("timestamp", datetime.now()),
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
|
||||
def _result_to_token(data: dict, chain: str, network: str) -> Token:
|
||||
"""Convert query result to Token type."""
|
||||
return Token(
|
||||
address=data.get("address", ""),
|
||||
name=data.get("name", ""),
|
||||
symbol=data.get("symbol", ""),
|
||||
decimals=data.get("decimals", 18),
|
||||
total_supply=str(data.get("total_supply", "0")),
|
||||
token_type=data.get("type", "ERC20"),
|
||||
holders=data.get("holders", 0),
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Query Type
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Query:
|
||||
"""Root query type for omnichain data."""
|
||||
|
||||
# =========================================================================
|
||||
# Block Queries
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def block(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
number: int | None = None,
|
||||
hash: str | None = None,
|
||||
) -> Block | None:
|
||||
"""Get a block by number or hash."""
|
||||
client = get_query_client()
|
||||
|
||||
identifier = hash if hash else number
|
||||
if identifier is None:
|
||||
return None
|
||||
|
||||
result = await client.get_block(chain, network, identifier)
|
||||
if result.data:
|
||||
return _result_to_block(result.data[0], chain, network)
|
||||
return None
|
||||
|
||||
@strawberry.field
|
||||
async def blocks(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
) -> PaginatedBlocks:
|
||||
"""Get recent blocks with pagination."""
|
||||
client = get_query_client()
|
||||
result = await client.get_blocks(chain, network, limit, cursor)
|
||||
|
||||
return PaginatedBlocks(
|
||||
items=[_result_to_block(b, chain, network) for b in result.data],
|
||||
total=result.total,
|
||||
cursor=result.cursor,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Transaction Queries
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def transaction(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
hash: str,
|
||||
) -> Transaction | None:
|
||||
"""Get a transaction by hash."""
|
||||
client = get_query_client()
|
||||
result = await client.get_transaction(chain, network, hash)
|
||||
|
||||
if result.data:
|
||||
return _result_to_transaction(result.data[0], chain, network)
|
||||
return None
|
||||
|
||||
@strawberry.field
|
||||
async def transactions(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str | None = None,
|
||||
block_number: int | None = None,
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
) -> PaginatedTransactions:
|
||||
"""Get transactions with optional filters."""
|
||||
client = get_query_client()
|
||||
result = await client.get_transactions(
|
||||
chain, network, address, block_number, limit, cursor
|
||||
)
|
||||
|
||||
return PaginatedTransactions(
|
||||
items=[_result_to_transaction(t, chain, network) for t in result.data],
|
||||
total=result.total,
|
||||
cursor=result.cursor,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Address Queries
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def address(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
) -> Address | None:
|
||||
"""Get address details."""
|
||||
client = get_query_client()
|
||||
result = await client.get_address(chain, network, address)
|
||||
|
||||
if result.data:
|
||||
data = result.data[0]
|
||||
return Address(
|
||||
address=data.get("address", address),
|
||||
balance=str(data.get("balance", "0")),
|
||||
tx_count=data.get("tx_count", 0),
|
||||
is_contract=data.get("is_contract", False),
|
||||
token_count=data.get("token_count", 0),
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
return None
|
||||
|
||||
@strawberry.field
|
||||
async def address_across_chains(
|
||||
self,
|
||||
address: str,
|
||||
chains: list[str] | None = None,
|
||||
) -> list[Address]:
|
||||
"""Get address info across multiple chains."""
|
||||
client = get_query_client()
|
||||
|
||||
# Parse chains input (format: "chain:network")
|
||||
chain_list = None
|
||||
if chains:
|
||||
chain_list = [
|
||||
tuple(c.split(":")) if ":" in c else (c, "mainnet")
|
||||
for c in chains
|
||||
]
|
||||
|
||||
results = await client.get_address_across_chains(address, chain_list)
|
||||
|
||||
addresses = []
|
||||
for key, result in results.items():
|
||||
if result.data:
|
||||
chain, network = key.split(":")
|
||||
data = result.data[0]
|
||||
addresses.append(Address(
|
||||
address=data.get("address", address),
|
||||
balance=str(data.get("balance", "0")),
|
||||
tx_count=data.get("tx_count", 0),
|
||||
is_contract=data.get("is_contract", False),
|
||||
token_count=data.get("token_count", 0),
|
||||
chain=chain,
|
||||
network=network,
|
||||
))
|
||||
|
||||
return addresses
|
||||
|
||||
# =========================================================================
|
||||
# Token Queries
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def token(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
) -> Token | None:
|
||||
"""Get token details."""
|
||||
client = get_query_client()
|
||||
result = await client.get_token(chain, network, address)
|
||||
|
||||
if result.data:
|
||||
return _result_to_token(result.data[0], chain, network)
|
||||
return None
|
||||
|
||||
@strawberry.field
|
||||
async def tokens(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
query: str | None = None,
|
||||
limit: int = 50,
|
||||
cursor: str | None = None,
|
||||
) -> PaginatedTokens:
|
||||
"""Get tokens with optional search."""
|
||||
client = get_query_client()
|
||||
result = await client.get_tokens(chain, network, query, limit, cursor)
|
||||
|
||||
return PaginatedTokens(
|
||||
items=[_result_to_token(t, chain, network) for t in result.data],
|
||||
total=result.total,
|
||||
cursor=result.cursor,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Search (Meilisearch)
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
types: list[str] | None = None,
|
||||
limit: int = 20,
|
||||
) -> SearchResults:
|
||||
"""Full-text search across addresses, tokens, contracts, transactions."""
|
||||
engine = get_search_engine()
|
||||
|
||||
# If types specified, search only those
|
||||
if types:
|
||||
all_hits = []
|
||||
total = 0
|
||||
took = 0.0
|
||||
|
||||
for t in types:
|
||||
if t == "addresses":
|
||||
r = await engine.search_addresses(query, chain, network, limit=limit)
|
||||
elif t == "tokens":
|
||||
r = await engine.search_tokens(query, chain, network, limit=limit)
|
||||
elif t == "contracts":
|
||||
r = await engine.search_contracts(query, chain, network, limit=limit)
|
||||
elif t == "transactions":
|
||||
r = await engine.search_transactions(query, chain, network, limit=limit)
|
||||
else:
|
||||
continue
|
||||
|
||||
all_hits.extend([
|
||||
SearchHit(id=h.id, type=t, data=h.data, score=h.score)
|
||||
for h in r.hits
|
||||
])
|
||||
total += r.total
|
||||
took = max(took, r.took_ms)
|
||||
|
||||
return SearchResults(hits=all_hits, total=total, query=query, took_ms=took)
|
||||
|
||||
# Search all types
|
||||
results = await engine.search_all(query, chain, network, limit_per_type=limit // 4)
|
||||
|
||||
all_hits = []
|
||||
total = 0
|
||||
took = 0.0
|
||||
|
||||
for index, response in results.items():
|
||||
all_hits.extend([
|
||||
SearchHit(id=h.id, type=index, data=h.data, score=h.score)
|
||||
for h in response.hits
|
||||
])
|
||||
total += response.total
|
||||
took = max(took, response.took_ms)
|
||||
|
||||
return SearchResults(hits=all_hits, total=total, query=query, took_ms=took)
|
||||
|
||||
# =========================================================================
|
||||
# Vector Search (Qdrant)
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def similar_contracts(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
limit: int = 10,
|
||||
) -> VectorResults:
|
||||
"""Find contracts similar to a given contract."""
|
||||
store = get_vector_store()
|
||||
result = await store.find_similar_contracts(address, chain, network, limit)
|
||||
|
||||
return VectorResults(
|
||||
hits=[
|
||||
VectorSearchHit(id=r.id, score=r.score, data=r.payload)
|
||||
for r in result.results
|
||||
],
|
||||
total=result.total,
|
||||
took_ms=result.took_ms,
|
||||
)
|
||||
|
||||
@strawberry.field
|
||||
async def semantic_search(
|
||||
self,
|
||||
query: str,
|
||||
collection: str = "contracts",
|
||||
chain: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> VectorResults:
|
||||
"""Semantic search using embeddings."""
|
||||
store = get_vector_store()
|
||||
|
||||
# Generate embedding for query
|
||||
embedding = await store.generate_embedding(query)
|
||||
|
||||
# Build filter
|
||||
filter_dict = None
|
||||
if chain:
|
||||
filter_dict = {"must": [{"key": "chain", "match": {"value": chain}}]}
|
||||
|
||||
result = await store.search(collection, embedding, limit=limit, filter=filter_dict)
|
||||
|
||||
return VectorResults(
|
||||
hits=[
|
||||
VectorSearchHit(id=r.id, score=r.score, data=r.payload)
|
||||
for r in result.results
|
||||
],
|
||||
total=result.total,
|
||||
took_ms=result.took_ms,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Analytics (ClickHouse)
|
||||
# =========================================================================
|
||||
|
||||
@strawberry.field
|
||||
async def chain_stats(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
) -> ChainStats | None:
|
||||
"""Get chain statistics."""
|
||||
client = get_query_client()
|
||||
result = await client.get_chain_stats(chain, network)
|
||||
|
||||
if result.data:
|
||||
data = result.data[0]
|
||||
return ChainStats(
|
||||
chain=chain,
|
||||
network=network,
|
||||
block_height=data.get("block_height", 0),
|
||||
tx_count_24h=data.get("tx_count_24h", 0),
|
||||
unique_addresses_24h=data.get("unique_addresses_24h", 0),
|
||||
avg_block_time=data.get("avg_block_time", 0.0),
|
||||
avg_gas_price=str(data.get("avg_gas_price", "0")),
|
||||
)
|
||||
return None
|
||||
|
||||
@strawberry.field
|
||||
async def gas_price_history(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
hours: int = 24,
|
||||
) -> TimeSeries:
|
||||
"""Get gas price history."""
|
||||
analytics = get_analytics_engine()
|
||||
result = await analytics.get_gas_price_history(chain, network, hours)
|
||||
|
||||
return TimeSeries(
|
||||
points=[
|
||||
TimeSeriesPoint(
|
||||
timestamp=p.timestamp,
|
||||
value=p.value,
|
||||
metadata=p.metadata,
|
||||
)
|
||||
for p in result.points
|
||||
],
|
||||
metric=result.metric,
|
||||
interval=result.interval,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
@strawberry.field
|
||||
async def transaction_volume(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
days: int = 30,
|
||||
interval: str = "1d",
|
||||
) -> TimeSeries:
|
||||
"""Get transaction volume history."""
|
||||
analytics = get_analytics_engine()
|
||||
result = await analytics.get_transaction_volume(chain, network, days, interval)
|
||||
|
||||
return TimeSeries(
|
||||
points=[
|
||||
TimeSeriesPoint(
|
||||
timestamp=p.timestamp,
|
||||
value=p.value,
|
||||
metadata=p.metadata,
|
||||
)
|
||||
for p in result.points
|
||||
],
|
||||
metric=result.metric,
|
||||
interval=result.interval,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
@strawberry.field
|
||||
async def defi_tvl(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
protocol: str | None = None,
|
||||
days: int = 30,
|
||||
) -> TimeSeries:
|
||||
"""Get DeFi TVL history."""
|
||||
analytics = get_analytics_engine()
|
||||
result = await analytics.get_defi_tvl(chain, network, protocol, days)
|
||||
|
||||
return TimeSeries(
|
||||
points=[
|
||||
TimeSeriesPoint(
|
||||
timestamp=p.timestamp,
|
||||
value=p.value,
|
||||
metadata=p.metadata,
|
||||
)
|
||||
for p in result.points
|
||||
],
|
||||
metric=result.metric,
|
||||
interval=result.interval,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Subscription Type (for real-time updates)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@strawberry.type
|
||||
class Subscription:
|
||||
"""Real-time subscriptions via WebSocket."""
|
||||
|
||||
@strawberry.subscription
|
||||
async def new_blocks(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
) -> Block:
|
||||
"""Subscribe to new blocks."""
|
||||
# This would connect to the indexer's WebSocket
|
||||
# For now, return a placeholder
|
||||
import asyncio
|
||||
while True:
|
||||
await asyncio.sleep(12) # ~1 block on Ethereum
|
||||
yield Block(
|
||||
hash="0x...",
|
||||
number=0,
|
||||
timestamp=datetime.now(),
|
||||
parent_hash="0x...",
|
||||
transactions_count=0,
|
||||
gas_used=0,
|
||||
gas_limit=0,
|
||||
base_fee_per_gas=None,
|
||||
miner="0x...",
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
@strawberry.subscription
|
||||
async def new_transactions(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str | None = None,
|
||||
) -> Transaction:
|
||||
"""Subscribe to new transactions."""
|
||||
import asyncio
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
yield Transaction(
|
||||
hash="0x...",
|
||||
block_number=0,
|
||||
block_hash="0x...",
|
||||
from_address="0x...",
|
||||
to_address="0x...",
|
||||
value="0",
|
||||
gas=0,
|
||||
gas_price="0",
|
||||
gas_used=0,
|
||||
nonce=0,
|
||||
input="0x",
|
||||
status=1,
|
||||
timestamp=datetime.now(),
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema
|
||||
# =============================================================================
|
||||
|
||||
|
||||
schema = strawberry.Schema(
|
||||
query=Query,
|
||||
subscription=Subscription,
|
||||
)
|
||||
|
||||
|
||||
def get_graphql_app():
|
||||
"""Get GraphQL ASGI app for FastAPI integration."""
|
||||
from strawberry.fastapi import GraphQLRouter
|
||||
|
||||
return GraphQLRouter(schema, path="/graphql")
|
||||
@@ -0,0 +1,444 @@
|
||||
"""Full-Text Search with Meilisearch - Fast search across blockchain data.
|
||||
|
||||
Indexes:
|
||||
- addresses: Wallet addresses with labels, ENS names
|
||||
- tokens: Token names, symbols, descriptions
|
||||
- contracts: Verified contract names, functions
|
||||
- transactions: Transaction hashes, methods
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import structlog
|
||||
|
||||
from bootnode.config import get_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
"""Search hit with relevance ranking."""
|
||||
id: str
|
||||
data: dict
|
||||
score: float | None = None
|
||||
highlights: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResponse:
|
||||
"""Search response with facets and stats."""
|
||||
hits: list[SearchHit]
|
||||
query: str
|
||||
total: int
|
||||
took_ms: float
|
||||
facets: dict | None = None
|
||||
|
||||
|
||||
class SearchEngine:
|
||||
"""Meilisearch full-text search engine."""
|
||||
|
||||
# Index names
|
||||
ADDRESSES = "addresses"
|
||||
TOKENS = "tokens"
|
||||
CONTRACTS = "contracts"
|
||||
TRANSACTIONS = "transactions"
|
||||
|
||||
def __init__(self) -> None:
|
||||
settings = get_settings()
|
||||
self.meili_url = getattr(settings, "meilisearch_url", "http://localhost:7700")
|
||||
self.meili_key = getattr(settings, "meilisearch_api_key", None)
|
||||
self._timeout = httpx.Timeout(30.0)
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
"""Get request headers."""
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.meili_key:
|
||||
headers["Authorization"] = f"Bearer {self.meili_key}"
|
||||
return headers
|
||||
|
||||
# =========================================================================
|
||||
# Index Management
|
||||
# =========================================================================
|
||||
|
||||
async def create_index(
|
||||
self,
|
||||
name: str,
|
||||
primary_key: str = "id",
|
||||
) -> bool:
|
||||
"""Create a search index."""
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.meili_url}/indexes",
|
||||
json={
|
||||
"uid": name,
|
||||
"primaryKey": primary_key,
|
||||
},
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
if response.status_code in (200, 201, 202):
|
||||
logger.info("Created search index", name=name)
|
||||
return True
|
||||
|
||||
logger.warning(
|
||||
"Failed to create index",
|
||||
name=name,
|
||||
status=response.status_code,
|
||||
)
|
||||
return False
|
||||
|
||||
async def configure_index(
|
||||
self,
|
||||
name: str,
|
||||
searchable_attributes: list[str] | None = None,
|
||||
filterable_attributes: list[str] | None = None,
|
||||
sortable_attributes: list[str] | None = None,
|
||||
ranking_rules: list[str] | None = None,
|
||||
) -> bool:
|
||||
"""Configure index settings."""
|
||||
settings: dict[str, Any] = {}
|
||||
|
||||
if searchable_attributes:
|
||||
settings["searchableAttributes"] = searchable_attributes
|
||||
if filterable_attributes:
|
||||
settings["filterableAttributes"] = filterable_attributes
|
||||
if sortable_attributes:
|
||||
settings["sortableAttributes"] = sortable_attributes
|
||||
if ranking_rules:
|
||||
settings["rankingRules"] = ranking_rules
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.patch(
|
||||
f"{self.meili_url}/indexes/{name}/settings",
|
||||
json=settings,
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
return response.status_code in (200, 202)
|
||||
|
||||
async def ensure_indexes(self) -> None:
|
||||
"""Ensure all required indexes exist with proper configuration."""
|
||||
# Addresses index
|
||||
await self.create_index(self.ADDRESSES)
|
||||
await self.configure_index(
|
||||
self.ADDRESSES,
|
||||
searchable_attributes=["address", "label", "ens", "tags"],
|
||||
filterable_attributes=["chain", "network", "type", "verified"],
|
||||
sortable_attributes=["balance", "tx_count", "created_at"],
|
||||
)
|
||||
|
||||
# Tokens index
|
||||
await self.create_index(self.TOKENS)
|
||||
await self.configure_index(
|
||||
self.TOKENS,
|
||||
searchable_attributes=["name", "symbol", "address", "description"],
|
||||
filterable_attributes=["chain", "network", "type", "decimals"],
|
||||
sortable_attributes=["holders", "market_cap", "volume_24h"],
|
||||
)
|
||||
|
||||
# Contracts index
|
||||
await self.create_index(self.CONTRACTS)
|
||||
await self.configure_index(
|
||||
self.CONTRACTS,
|
||||
searchable_attributes=["name", "address", "functions", "events", "source"],
|
||||
filterable_attributes=["chain", "network", "verified", "compiler"],
|
||||
sortable_attributes=["tx_count", "created_at"],
|
||||
)
|
||||
|
||||
# Transactions index
|
||||
await self.create_index(self.TRANSACTIONS)
|
||||
await self.configure_index(
|
||||
self.TRANSACTIONS,
|
||||
searchable_attributes=["hash", "from", "to", "method", "input_decoded"],
|
||||
filterable_attributes=["chain", "network", "status", "block_number"],
|
||||
sortable_attributes=["block_number", "gas_used", "value"],
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Indexing
|
||||
# =========================================================================
|
||||
|
||||
async def add_documents(
|
||||
self,
|
||||
index: str,
|
||||
documents: list[dict],
|
||||
primary_key: str | None = None,
|
||||
) -> str | None:
|
||||
"""Add documents to an index."""
|
||||
params = {}
|
||||
if primary_key:
|
||||
params["primaryKey"] = primary_key
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.meili_url}/indexes/{index}/documents",
|
||||
json=documents,
|
||||
params=params,
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
if response.status_code in (200, 202):
|
||||
data = response.json()
|
||||
return data.get("taskUid")
|
||||
|
||||
logger.warning(
|
||||
"Failed to add documents",
|
||||
index=index,
|
||||
status=response.status_code,
|
||||
)
|
||||
return None
|
||||
|
||||
async def delete_document(
|
||||
self,
|
||||
index: str,
|
||||
document_id: str,
|
||||
) -> bool:
|
||||
"""Delete a document from an index."""
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.delete(
|
||||
f"{self.meili_url}/indexes/{index}/documents/{document_id}",
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
return response.status_code in (200, 202)
|
||||
|
||||
# =========================================================================
|
||||
# Search
|
||||
# =========================================================================
|
||||
|
||||
async def search(
|
||||
self,
|
||||
index: str,
|
||||
query: str,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
filter: str | None = None,
|
||||
sort: list[str] | None = None,
|
||||
facets: list[str] | None = None,
|
||||
highlight_fields: list[str] | None = None,
|
||||
) -> SearchResponse:
|
||||
"""Search an index."""
|
||||
start = datetime.now()
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"q": query,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
if filter:
|
||||
body["filter"] = filter
|
||||
if sort:
|
||||
body["sort"] = sort
|
||||
if facets:
|
||||
body["facets"] = facets
|
||||
if highlight_fields:
|
||||
body["attributesToHighlight"] = highlight_fields
|
||||
body["highlightPreTag"] = "<mark>"
|
||||
body["highlightPostTag"] = "</mark>"
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.meili_url}/indexes/{index}/search",
|
||||
json=body,
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
took_ms = (datetime.now() - start).total_seconds() * 1000
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
hits = [
|
||||
SearchHit(
|
||||
id=hit.get("id", hit.get("address", "")),
|
||||
data=hit,
|
||||
highlights=hit.get("_formatted"),
|
||||
)
|
||||
for hit in data.get("hits", [])
|
||||
]
|
||||
|
||||
return SearchResponse(
|
||||
hits=hits,
|
||||
query=query,
|
||||
total=data.get("estimatedTotalHits", len(hits)),
|
||||
took_ms=data.get("processingTimeMs", took_ms),
|
||||
facets=data.get("facetDistribution"),
|
||||
)
|
||||
|
||||
return SearchResponse(
|
||||
hits=[],
|
||||
query=query,
|
||||
total=0,
|
||||
took_ms=took_ms,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Specialized Searches
|
||||
# =========================================================================
|
||||
|
||||
async def search_addresses(
|
||||
self,
|
||||
query: str,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
address_type: str | None = None,
|
||||
limit: int = 20,
|
||||
) -> SearchResponse:
|
||||
"""Search addresses by label, ENS, or address."""
|
||||
filters = []
|
||||
if chain:
|
||||
filters.append(f'chain = "{chain}"')
|
||||
if network:
|
||||
filters.append(f'network = "{network}"')
|
||||
if address_type:
|
||||
filters.append(f'type = "{address_type}"')
|
||||
|
||||
filter_str = " AND ".join(filters) if filters else None
|
||||
|
||||
return await self.search(
|
||||
self.ADDRESSES,
|
||||
query,
|
||||
limit=limit,
|
||||
filter=filter_str,
|
||||
highlight_fields=["label", "ens", "address"],
|
||||
)
|
||||
|
||||
async def search_tokens(
|
||||
self,
|
||||
query: str,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
token_type: str | None = None,
|
||||
limit: int = 20,
|
||||
) -> SearchResponse:
|
||||
"""Search tokens by name, symbol, or address."""
|
||||
filters = []
|
||||
if chain:
|
||||
filters.append(f'chain = "{chain}"')
|
||||
if network:
|
||||
filters.append(f'network = "{network}"')
|
||||
if token_type:
|
||||
filters.append(f'type = "{token_type}"')
|
||||
|
||||
filter_str = " AND ".join(filters) if filters else None
|
||||
|
||||
return await self.search(
|
||||
self.TOKENS,
|
||||
query,
|
||||
limit=limit,
|
||||
filter=filter_str,
|
||||
sort=["holders:desc"],
|
||||
facets=["chain", "type"],
|
||||
highlight_fields=["name", "symbol", "description"],
|
||||
)
|
||||
|
||||
async def search_contracts(
|
||||
self,
|
||||
query: str,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
verified_only: bool = False,
|
||||
limit: int = 20,
|
||||
) -> SearchResponse:
|
||||
"""Search contracts by name, function, or address."""
|
||||
filters = []
|
||||
if chain:
|
||||
filters.append(f'chain = "{chain}"')
|
||||
if network:
|
||||
filters.append(f'network = "{network}"')
|
||||
if verified_only:
|
||||
filters.append("verified = true")
|
||||
|
||||
filter_str = " AND ".join(filters) if filters else None
|
||||
|
||||
return await self.search(
|
||||
self.CONTRACTS,
|
||||
query,
|
||||
limit=limit,
|
||||
filter=filter_str,
|
||||
facets=["chain", "verified", "compiler"],
|
||||
highlight_fields=["name", "functions", "events"],
|
||||
)
|
||||
|
||||
async def search_transactions(
|
||||
self,
|
||||
query: str,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 20,
|
||||
) -> SearchResponse:
|
||||
"""Search transactions by hash, method, or address."""
|
||||
filters = []
|
||||
if chain:
|
||||
filters.append(f'chain = "{chain}"')
|
||||
if network:
|
||||
filters.append(f'network = "{network}"')
|
||||
if status:
|
||||
filters.append(f'status = "{status}"')
|
||||
|
||||
filter_str = " AND ".join(filters) if filters else None
|
||||
|
||||
return await self.search(
|
||||
self.TRANSACTIONS,
|
||||
query,
|
||||
limit=limit,
|
||||
filter=filter_str,
|
||||
sort=["block_number:desc"],
|
||||
highlight_fields=["hash", "method", "input_decoded"],
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Global Search
|
||||
# =========================================================================
|
||||
|
||||
async def search_all(
|
||||
self,
|
||||
query: str,
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
limit_per_type: int = 5,
|
||||
) -> dict[str, SearchResponse]:
|
||||
"""Search across all indexes in parallel."""
|
||||
import asyncio
|
||||
|
||||
async def search_index(index: str, search_fn) -> tuple[str, SearchResponse]:
|
||||
result = await search_fn(
|
||||
query,
|
||||
chain=chain,
|
||||
network=network,
|
||||
limit=limit_per_type,
|
||||
)
|
||||
return index, result
|
||||
|
||||
tasks = [
|
||||
search_index(self.ADDRESSES, self.search_addresses),
|
||||
search_index(self.TOKENS, self.search_tokens),
|
||||
search_index(self.CONTRACTS, self.search_contracts),
|
||||
search_index(self.TRANSACTIONS, self.search_transactions),
|
||||
]
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
return {
|
||||
index: result
|
||||
for index, result in results
|
||||
if not isinstance(result, Exception)
|
||||
}
|
||||
|
||||
|
||||
# Global instance
|
||||
_search_engine: SearchEngine | None = None
|
||||
|
||||
|
||||
def get_search_engine() -> SearchEngine:
|
||||
"""Get search engine singleton."""
|
||||
global _search_engine
|
||||
if _search_engine is None:
|
||||
_search_engine = SearchEngine()
|
||||
return _search_engine
|
||||
@@ -0,0 +1,563 @@
|
||||
"""Unified Query Client - Single interface for all storage backends.
|
||||
|
||||
Provides a unified interface to query blockchain data across:
|
||||
- PostgreSQL (metadata, indexes)
|
||||
- ClickHouse (analytics, time-series)
|
||||
- Qdrant (vector/semantic search)
|
||||
- Meilisearch (full-text search)
|
||||
- Lux Indexer (real-time chain data)
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
import structlog
|
||||
|
||||
from bootnode.config import get_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
class QueryType(str, Enum):
|
||||
"""Query type determines which backend to use."""
|
||||
METADATA = "metadata" # PostgreSQL
|
||||
ANALYTICS = "analytics" # ClickHouse
|
||||
SEMANTIC = "semantic" # Qdrant (vector)
|
||||
FULLTEXT = "fulltext" # Meilisearch
|
||||
REALTIME = "realtime" # Lux Indexer
|
||||
HYBRID = "hybrid" # Multiple backends
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueryResult:
|
||||
"""Unified query result."""
|
||||
data: list[dict]
|
||||
total: int
|
||||
query_type: QueryType
|
||||
took_ms: float
|
||||
chain: str | None = None
|
||||
network: str | None = None
|
||||
cursor: str | None = None
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Search result with relevance scoring."""
|
||||
items: list[dict]
|
||||
total: int
|
||||
query: str
|
||||
took_ms: float
|
||||
facets: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class UnifiedQueryClient:
|
||||
"""Unified client for querying all storage backends."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
settings = get_settings()
|
||||
self.indexer_url = getattr(settings, "indexer_url", "http://localhost:5000")
|
||||
self._timeout = httpx.Timeout(30.0)
|
||||
|
||||
# =========================================================================
|
||||
# Block Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_block(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
block_number: int | str,
|
||||
) -> QueryResult:
|
||||
"""Get block by number or hash."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
# Query Lux Indexer
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/blocks/{block_number}"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=[data],
|
||||
total=1,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
async def get_blocks(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
) -> QueryResult:
|
||||
"""Get recent blocks with pagination."""
|
||||
start = datetime.now()
|
||||
|
||||
params = {"limit": limit}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/blocks",
|
||||
params=params,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=data.get("items", []),
|
||||
total=data.get("total", len(data.get("items", []))),
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
cursor=data.get("next_cursor"),
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Transaction Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_transaction(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
tx_hash: str,
|
||||
) -> QueryResult:
|
||||
"""Get transaction by hash."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/transactions/{tx_hash}"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=[data],
|
||||
total=1,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
async def get_transactions(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str | None = None,
|
||||
block_number: int | None = None,
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
) -> QueryResult:
|
||||
"""Get transactions with optional filters."""
|
||||
start = datetime.now()
|
||||
|
||||
params: dict[str, Any] = {"limit": limit}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
|
||||
# Determine endpoint based on filters
|
||||
if address:
|
||||
endpoint = f"{self.indexer_url}/api/v2/{chain}/{network}/addresses/{address}/transactions"
|
||||
elif block_number:
|
||||
endpoint = f"{self.indexer_url}/api/v2/{chain}/{network}/blocks/{block_number}/transactions"
|
||||
else:
|
||||
endpoint = f"{self.indexer_url}/api/v2/{chain}/{network}/transactions"
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(endpoint, params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=data.get("items", []),
|
||||
total=data.get("total", len(data.get("items", []))),
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
cursor=data.get("next_cursor"),
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Address Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_address(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
) -> QueryResult:
|
||||
"""Get address details including balance, token holdings."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/addresses/{address}"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=[data],
|
||||
total=1,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
async def get_address_tokens(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
limit: int = 50,
|
||||
) -> QueryResult:
|
||||
"""Get token balances for an address."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/addresses/{address}/tokens",
|
||||
params={"limit": limit},
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=data.get("items", []),
|
||||
total=data.get("total", 0),
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Token Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_token(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
) -> QueryResult:
|
||||
"""Get token details by contract address."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/tokens/{address}"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=[data],
|
||||
total=1,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
async def get_tokens(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
query: str | None = None,
|
||||
limit: int = 50,
|
||||
cursor: str | None = None,
|
||||
) -> QueryResult:
|
||||
"""Get tokens with optional search query."""
|
||||
start = datetime.now()
|
||||
|
||||
params: dict[str, Any] = {"limit": limit}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
if query:
|
||||
params["q"] = query
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/tokens",
|
||||
params=params,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=data.get("items", []),
|
||||
total=data.get("total", 0),
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
cursor=data.get("next_cursor"),
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Contract Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_contract(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str,
|
||||
) -> QueryResult:
|
||||
"""Get verified contract with ABI and source code."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/smart-contracts/{address}"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=[data],
|
||||
total=1,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Logs/Events Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_logs(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
address: str | None = None,
|
||||
topics: list[str] | None = None,
|
||||
from_block: int | None = None,
|
||||
to_block: int | None = None,
|
||||
limit: int = 100,
|
||||
) -> QueryResult:
|
||||
"""Get event logs with filters."""
|
||||
start = datetime.now()
|
||||
|
||||
params: dict[str, Any] = {"limit": limit}
|
||||
if address:
|
||||
params["address"] = address
|
||||
if topics:
|
||||
params["topics"] = ",".join(topics)
|
||||
if from_block:
|
||||
params["from_block"] = from_block
|
||||
if to_block:
|
||||
params["to_block"] = to_block
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/logs",
|
||||
params=params,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=data.get("items", []),
|
||||
total=data.get("total", 0),
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Cross-Chain Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_address_across_chains(
|
||||
self,
|
||||
address: str,
|
||||
chains: list[tuple[str, str]] | None = None,
|
||||
) -> dict[str, QueryResult]:
|
||||
"""Get address info across multiple chains in parallel."""
|
||||
import asyncio
|
||||
|
||||
if chains is None:
|
||||
# Default to popular chains
|
||||
chains = [
|
||||
("ethereum", "mainnet"),
|
||||
("polygon", "mainnet"),
|
||||
("arbitrum", "mainnet"),
|
||||
("optimism", "mainnet"),
|
||||
("base", "mainnet"),
|
||||
("avalanche", "mainnet"),
|
||||
("bsc", "mainnet"),
|
||||
]
|
||||
|
||||
async def fetch_chain(chain: str, network: str) -> tuple[str, QueryResult]:
|
||||
result = await self.get_address(chain, network, address)
|
||||
return f"{chain}:{network}", result
|
||||
|
||||
tasks = [fetch_chain(chain, network) for chain, network in chains]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
return {
|
||||
key: result
|
||||
for key, result in results
|
||||
if not isinstance(result, Exception) and result.total > 0
|
||||
}
|
||||
|
||||
# =========================================================================
|
||||
# Stats Queries
|
||||
# =========================================================================
|
||||
|
||||
async def get_chain_stats(
|
||||
self,
|
||||
chain: str,
|
||||
network: str,
|
||||
) -> QueryResult:
|
||||
"""Get chain statistics (block height, tx count, etc)."""
|
||||
start = datetime.now()
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.get(
|
||||
f"{self.indexer_url}/api/v2/{chain}/{network}/stats"
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return QueryResult(
|
||||
data=[data],
|
||||
total=1,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
return QueryResult(
|
||||
data=[],
|
||||
total=0,
|
||||
query_type=QueryType.REALTIME,
|
||||
took_ms=(datetime.now() - start).total_seconds() * 1000,
|
||||
chain=chain,
|
||||
network=network,
|
||||
)
|
||||
|
||||
|
||||
# Global client instance
|
||||
_query_client: UnifiedQueryClient | None = None
|
||||
|
||||
|
||||
def get_query_client() -> UnifiedQueryClient:
|
||||
"""Get unified query client singleton."""
|
||||
global _query_client
|
||||
if _query_client is None:
|
||||
_query_client = UnifiedQueryClient()
|
||||
return _query_client
|
||||
@@ -0,0 +1,352 @@
|
||||
"""Vector Search with Qdrant - Semantic search across blockchain data.
|
||||
|
||||
Use cases:
|
||||
- Find similar contracts by bytecode/ABI
|
||||
- Semantic search for transactions by description
|
||||
- Find addresses with similar activity patterns
|
||||
- Search for tokens by description/use case
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import structlog
|
||||
|
||||
from bootnode.config import get_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class VectorSearchResult:
|
||||
"""Vector search result with similarity score."""
|
||||
id: str
|
||||
score: float
|
||||
payload: dict
|
||||
vector: list[float] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VectorSearchResponse:
|
||||
"""Vector search response."""
|
||||
results: list[VectorSearchResult]
|
||||
took_ms: float
|
||||
total: int
|
||||
|
||||
|
||||
class VectorStore:
|
||||
"""Qdrant vector store for semantic search."""
|
||||
|
||||
# Collection names
|
||||
CONTRACTS = "contracts"
|
||||
TRANSACTIONS = "transactions"
|
||||
TOKENS = "tokens"
|
||||
ADDRESSES = "addresses"
|
||||
|
||||
def __init__(self) -> None:
|
||||
settings = get_settings()
|
||||
self.qdrant_url = getattr(settings, "qdrant_url", "http://localhost:6333")
|
||||
self.qdrant_api_key = getattr(settings, "qdrant_api_key", None)
|
||||
self._timeout = httpx.Timeout(30.0)
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
"""Get request headers."""
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if self.qdrant_api_key:
|
||||
headers["api-key"] = self.qdrant_api_key
|
||||
return headers
|
||||
|
||||
# =========================================================================
|
||||
# Collection Management
|
||||
# =========================================================================
|
||||
|
||||
async def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
vector_size: int = 1536, # OpenAI embedding size
|
||||
distance: str = "Cosine",
|
||||
) -> bool:
|
||||
"""Create a vector collection."""
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.put(
|
||||
f"{self.qdrant_url}/collections/{name}",
|
||||
json={
|
||||
"vectors": {
|
||||
"size": vector_size,
|
||||
"distance": distance,
|
||||
},
|
||||
"optimizers_config": {
|
||||
"indexing_threshold": 10000,
|
||||
},
|
||||
},
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
if response.status_code in (200, 201):
|
||||
logger.info("Created vector collection", name=name)
|
||||
return True
|
||||
|
||||
logger.warning(
|
||||
"Failed to create collection",
|
||||
name=name,
|
||||
status=response.status_code,
|
||||
)
|
||||
return False
|
||||
|
||||
async def ensure_collections(self) -> None:
|
||||
"""Ensure all required collections exist."""
|
||||
collections = [
|
||||
(self.CONTRACTS, 1536),
|
||||
(self.TRANSACTIONS, 768),
|
||||
(self.TOKENS, 768),
|
||||
(self.ADDRESSES, 512),
|
||||
]
|
||||
|
||||
for name, size in collections:
|
||||
await self.create_collection(name, size)
|
||||
|
||||
# =========================================================================
|
||||
# Indexing
|
||||
# =========================================================================
|
||||
|
||||
async def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
id: str,
|
||||
vector: list[float],
|
||||
payload: dict,
|
||||
) -> bool:
|
||||
"""Upsert a vector with payload."""
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.put(
|
||||
f"{self.qdrant_url}/collections/{collection}/points",
|
||||
json={
|
||||
"points": [
|
||||
{
|
||||
"id": id,
|
||||
"vector": vector,
|
||||
"payload": payload,
|
||||
}
|
||||
]
|
||||
},
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
return response.status_code in (200, 201)
|
||||
|
||||
async def upsert_batch(
|
||||
self,
|
||||
collection: str,
|
||||
points: list[dict],
|
||||
) -> bool:
|
||||
"""Upsert multiple vectors."""
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.put(
|
||||
f"{self.qdrant_url}/collections/{collection}/points",
|
||||
json={"points": points},
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
return response.status_code in (200, 201)
|
||||
|
||||
# =========================================================================
|
||||
# Search
|
||||
# =========================================================================
|
||||
|
||||
async def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_vector: list[float],
|
||||
limit: int = 10,
|
||||
filter: dict | None = None,
|
||||
with_payload: bool = True,
|
||||
with_vectors: bool = False,
|
||||
) -> VectorSearchResponse:
|
||||
"""Search for similar vectors."""
|
||||
start = datetime.now()
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"vector": query_vector,
|
||||
"limit": limit,
|
||||
"with_payload": with_payload,
|
||||
"with_vectors": with_vectors,
|
||||
}
|
||||
|
||||
if filter:
|
||||
body["filter"] = filter
|
||||
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.qdrant_url}/collections/{collection}/points/search",
|
||||
json=body,
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
took_ms = (datetime.now() - start).total_seconds() * 1000
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
results = [
|
||||
VectorSearchResult(
|
||||
id=str(r["id"]),
|
||||
score=r["score"],
|
||||
payload=r.get("payload", {}),
|
||||
vector=r.get("vector"),
|
||||
)
|
||||
for r in data.get("result", [])
|
||||
]
|
||||
|
||||
return VectorSearchResponse(
|
||||
results=results,
|
||||
took_ms=took_ms,
|
||||
total=len(results),
|
||||
)
|
||||
|
||||
return VectorSearchResponse(
|
||||
results=[],
|
||||
took_ms=took_ms,
|
||||
total=0,
|
||||
)
|
||||
|
||||
async def search_contracts(
|
||||
self,
|
||||
query_vector: list[float],
|
||||
chain: str | None = None,
|
||||
network: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> VectorSearchResponse:
|
||||
"""Search for similar contracts."""
|
||||
filter = None
|
||||
if chain or network:
|
||||
must = []
|
||||
if chain:
|
||||
must.append({"key": "chain", "match": {"value": chain}})
|
||||
if network:
|
||||
must.append({"key": "network", "match": {"value": network}})
|
||||
filter = {"must": must}
|
||||
|
||||
return await self.search(
|
||||
self.CONTRACTS,
|
||||
query_vector,
|
||||
limit=limit,
|
||||
filter=filter,
|
||||
)
|
||||
|
||||
async def search_tokens(
|
||||
self,
|
||||
query_vector: list[float],
|
||||
chain: str | None = None,
|
||||
token_type: str | None = None,
|
||||
limit: int = 10,
|
||||
) -> VectorSearchResponse:
|
||||
"""Search for similar tokens."""
|
||||
filter = None
|
||||
must = []
|
||||
if chain:
|
||||
must.append({"key": "chain", "match": {"value": chain}})
|
||||
if token_type:
|
||||
must.append({"key": "type", "match": {"value": token_type}})
|
||||
if must:
|
||||
filter = {"must": must}
|
||||
|
||||
return await self.search(
|
||||
self.TOKENS,
|
||||
query_vector,
|
||||
limit=limit,
|
||||
filter=filter,
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Contract Similarity
|
||||
# =========================================================================
|
||||
|
||||
async def find_similar_contracts(
|
||||
self,
|
||||
contract_address: str,
|
||||
chain: str,
|
||||
network: str,
|
||||
limit: int = 10,
|
||||
) -> VectorSearchResponse:
|
||||
"""Find contracts similar to a given contract.
|
||||
|
||||
Uses the stored embedding of the contract's bytecode/ABI.
|
||||
"""
|
||||
# First, get the contract's vector
|
||||
async with httpx.AsyncClient(timeout=self._timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.qdrant_url}/collections/{self.CONTRACTS}/points/scroll",
|
||||
json={
|
||||
"filter": {
|
||||
"must": [
|
||||
{"key": "address", "match": {"value": contract_address.lower()}},
|
||||
{"key": "chain", "match": {"value": chain}},
|
||||
{"key": "network", "match": {"value": network}},
|
||||
]
|
||||
},
|
||||
"limit": 1,
|
||||
"with_vectors": True,
|
||||
},
|
||||
headers=self._headers(),
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
return VectorSearchResponse(results=[], took_ms=0, total=0)
|
||||
|
||||
data = response.json()
|
||||
points = data.get("result", {}).get("points", [])
|
||||
if not points:
|
||||
return VectorSearchResponse(results=[], took_ms=0, total=0)
|
||||
|
||||
vector = points[0].get("vector", [])
|
||||
|
||||
# Search for similar, excluding the original
|
||||
return await self.search(
|
||||
self.CONTRACTS,
|
||||
vector,
|
||||
limit=limit + 1, # Get extra to filter out self
|
||||
filter={
|
||||
"must_not": [
|
||||
{"key": "address", "match": {"value": contract_address.lower()}}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# Embedding Generation (placeholder - integrate with embedding service)
|
||||
# =========================================================================
|
||||
|
||||
async def generate_embedding(
|
||||
self,
|
||||
text: str,
|
||||
model: str = "text-embedding-3-small",
|
||||
) -> list[float]:
|
||||
"""Generate embedding for text.
|
||||
|
||||
In production, this should call OpenAI or a local embedding model.
|
||||
For now, returns a placeholder.
|
||||
"""
|
||||
# TODO: Integrate with embedding service
|
||||
# This is a placeholder that returns a random-ish vector
|
||||
import hashlib
|
||||
hash_bytes = hashlib.sha256(text.encode()).digest()
|
||||
# Create a 768-dim vector from hash (for demo)
|
||||
vector = []
|
||||
for i in range(768):
|
||||
byte_idx = i % 32
|
||||
vector.append((hash_bytes[byte_idx] - 128) / 128.0)
|
||||
return vector
|
||||
|
||||
|
||||
# Global instance
|
||||
_vector_store: VectorStore | None = None
|
||||
|
||||
|
||||
def get_vector_store() -> VectorStore:
|
||||
"""Get vector store singleton."""
|
||||
global _vector_store
|
||||
if _vector_store is None:
|
||||
_vector_store = VectorStore()
|
||||
return _vector_store
|
||||
+50
@@ -63,6 +63,47 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# =============================================================================
|
||||
# SEARCH & VECTOR DATABASES (Planet-Scale Query Layer)
|
||||
# =============================================================================
|
||||
|
||||
# Qdrant - Vector Search for semantic queries (similar contracts, addresses)
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: bootnode-qdrant
|
||||
ports:
|
||||
- "6333:6333"
|
||||
- "6334:6334"
|
||||
volumes:
|
||||
- qdrant:/qdrant/storage
|
||||
environment:
|
||||
QDRANT__SERVICE__GRPC_PORT: 6334
|
||||
QDRANT__LOG_LEVEL: INFO
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:6333/"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# Meilisearch - Full-text search for addresses, tokens, contracts
|
||||
meilisearch:
|
||||
image: getmeili/meilisearch:v1.6
|
||||
container_name: bootnode-meilisearch
|
||||
ports:
|
||||
- "7700:7700"
|
||||
volumes:
|
||||
- meilisearch:/meili_data
|
||||
environment:
|
||||
MEILI_ENV: development
|
||||
MEILI_NO_ANALYTICS: "true"
|
||||
# Set a master key in production!
|
||||
# MEILI_MASTER_KEY: your-master-key
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:7700/health"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# =============================================================================
|
||||
# BOOTNODE SERVICES
|
||||
# =============================================================================
|
||||
@@ -78,6 +119,9 @@ services:
|
||||
DATABASE_URL: postgresql+asyncpg://bootnode:bootnode@postgres:5432/bootnode
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
DATASTORE_URL: clickhouse://bootnode:bootnode@datastore:8123/bootnode
|
||||
QDRANT_URL: http://qdrant:6333
|
||||
MEILISEARCH_URL: http://meilisearch:7700
|
||||
INDEXER_URL: http://indexer:5000
|
||||
JWT_SECRET: dev-jwt-secret-change-in-production
|
||||
API_KEY_SALT: dev-api-key-salt
|
||||
# Multi-tenant off for local dev
|
||||
@@ -106,6 +150,10 @@ services:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
qdrant:
|
||||
condition: service_healthy
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
@@ -179,4 +227,6 @@ volumes:
|
||||
postgres:
|
||||
redis:
|
||||
datastore:
|
||||
qdrant:
|
||||
meilisearch:
|
||||
grafana:
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
-- ============================================================================
|
||||
-- Omnichain Analytics Schema for ClickHouse
|
||||
-- Supports 100+ chains with high-performance time-series analytics
|
||||
-- ============================================================================
|
||||
|
||||
-- Chain metrics (real-time stats)
|
||||
CREATE TABLE IF NOT EXISTS chain_metrics (
|
||||
chain String,
|
||||
network String,
|
||||
metric_name String,
|
||||
metric_value Float64,
|
||||
timestamp DateTime64(3) DEFAULT now64(3)
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
ORDER BY (chain, network, metric_name, timestamp)
|
||||
TTL timestamp + INTERVAL 90 DAY;
|
||||
|
||||
-- Blocks (for analytics queries)
|
||||
CREATE TABLE IF NOT EXISTS blocks (
|
||||
chain String,
|
||||
network String,
|
||||
block_number UInt64,
|
||||
block_hash String,
|
||||
parent_hash String,
|
||||
timestamp DateTime64(3),
|
||||
transaction_count UInt32,
|
||||
gas_used UInt64,
|
||||
gas_limit UInt64,
|
||||
base_fee_per_gas Nullable(UInt64),
|
||||
miner String,
|
||||
block_time Float32, -- seconds since last block
|
||||
INDEX idx_hash block_hash TYPE bloom_filter GRANULARITY 4
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, block_number)
|
||||
TTL timestamp + INTERVAL 2 YEAR;
|
||||
|
||||
-- Transactions (for analytics)
|
||||
CREATE TABLE IF NOT EXISTS transactions (
|
||||
chain String,
|
||||
network String,
|
||||
tx_hash String,
|
||||
block_number UInt64,
|
||||
tx_index UInt32,
|
||||
timestamp DateTime64(3),
|
||||
from_address String,
|
||||
to_address Nullable(String),
|
||||
value UInt256,
|
||||
gas UInt64,
|
||||
gas_price UInt256,
|
||||
gas_used UInt64,
|
||||
status UInt8,
|
||||
tx_type UInt8,
|
||||
method_id String,
|
||||
INDEX idx_from from_address TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_to to_address TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_hash tx_hash TYPE bloom_filter GRANULARITY 4
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, block_number, tx_index)
|
||||
TTL timestamp + INTERVAL 2 YEAR;
|
||||
|
||||
-- Token transfers (ERC20/721/1155)
|
||||
CREATE TABLE IF NOT EXISTS token_transfers (
|
||||
chain String,
|
||||
network String,
|
||||
tx_hash String,
|
||||
log_index UInt32,
|
||||
block_number UInt64,
|
||||
timestamp DateTime64(3),
|
||||
token_address String,
|
||||
token_name String,
|
||||
token_symbol String,
|
||||
token_type String, -- ERC20, ERC721, ERC1155
|
||||
from_address String,
|
||||
to_address String,
|
||||
value UInt256,
|
||||
token_id Nullable(UInt256),
|
||||
INDEX idx_token token_address TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_from from_address TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_to to_address TYPE bloom_filter GRANULARITY 4
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, block_number, log_index)
|
||||
TTL timestamp + INTERVAL 2 YEAR;
|
||||
|
||||
-- DeFi TVL snapshots
|
||||
CREATE TABLE IF NOT EXISTS defi_tvl (
|
||||
chain String,
|
||||
network String,
|
||||
protocol String,
|
||||
pool_address String,
|
||||
tvl_usd Float64,
|
||||
tvl_native Float64,
|
||||
timestamp DateTime64(3) DEFAULT now64(3)
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, protocol, timestamp)
|
||||
TTL timestamp + INTERVAL 1 YEAR;
|
||||
|
||||
-- DEX swaps
|
||||
CREATE TABLE IF NOT EXISTS dex_swaps (
|
||||
chain String,
|
||||
network String,
|
||||
tx_hash String,
|
||||
log_index UInt32,
|
||||
block_number UInt64,
|
||||
timestamp DateTime64(3),
|
||||
protocol String,
|
||||
pool_address String,
|
||||
trader String,
|
||||
token_in String,
|
||||
token_out String,
|
||||
amount_in UInt256,
|
||||
amount_out UInt256,
|
||||
amount_usd Float64,
|
||||
INDEX idx_trader trader TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_pool pool_address TYPE bloom_filter GRANULARITY 4
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, block_number, log_index)
|
||||
TTL timestamp + INTERVAL 2 YEAR;
|
||||
|
||||
-- Bridge transfers
|
||||
CREATE TABLE IF NOT EXISTS bridge_transfers (
|
||||
source_chain String,
|
||||
source_network String,
|
||||
dest_chain String,
|
||||
dest_network String,
|
||||
tx_hash String,
|
||||
block_number UInt64,
|
||||
timestamp DateTime64(3),
|
||||
bridge_protocol String,
|
||||
sender String,
|
||||
receiver String,
|
||||
token String,
|
||||
amount UInt256,
|
||||
amount_usd Float64,
|
||||
INDEX idx_sender sender TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_receiver receiver TYPE bloom_filter GRANULARITY 4
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (source_chain, toYYYYMM(timestamp))
|
||||
ORDER BY (source_chain, source_network, block_number)
|
||||
TTL timestamp + INTERVAL 2 YEAR;
|
||||
|
||||
-- NFT sales
|
||||
CREATE TABLE IF NOT EXISTS nft_sales (
|
||||
chain String,
|
||||
network String,
|
||||
tx_hash String,
|
||||
log_index UInt32,
|
||||
block_number UInt64,
|
||||
timestamp DateTime64(3),
|
||||
marketplace String,
|
||||
collection_address String,
|
||||
token_id UInt256,
|
||||
seller String,
|
||||
buyer String,
|
||||
price_wei UInt256,
|
||||
price_usd Float64,
|
||||
currency_token String,
|
||||
INDEX idx_collection collection_address TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_seller seller TYPE bloom_filter GRANULARITY 4,
|
||||
INDEX idx_buyer buyer TYPE bloom_filter GRANULARITY 4
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, block_number, log_index)
|
||||
TTL timestamp + INTERVAL 2 YEAR;
|
||||
|
||||
-- Gas price history (for gas oracle)
|
||||
CREATE TABLE IF NOT EXISTS gas_prices (
|
||||
chain String,
|
||||
network String,
|
||||
block_number UInt64,
|
||||
timestamp DateTime64(3),
|
||||
base_fee UInt64,
|
||||
priority_fee_slow UInt64,
|
||||
priority_fee_avg UInt64,
|
||||
priority_fee_fast UInt64,
|
||||
gas_used_percent Float32
|
||||
) ENGINE = MergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(timestamp))
|
||||
ORDER BY (chain, network, timestamp)
|
||||
TTL timestamp + INTERVAL 30 DAY;
|
||||
|
||||
-- Address activity (daily aggregates)
|
||||
CREATE TABLE IF NOT EXISTS address_activity_daily (
|
||||
chain String,
|
||||
network String,
|
||||
address String,
|
||||
date Date,
|
||||
tx_count_sent UInt32,
|
||||
tx_count_received UInt32,
|
||||
volume_sent UInt256,
|
||||
volume_received UInt256,
|
||||
gas_spent UInt256,
|
||||
unique_counterparties UInt32
|
||||
) ENGINE = SummingMergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(date))
|
||||
ORDER BY (chain, network, address, date)
|
||||
TTL date + INTERVAL 2 YEAR;
|
||||
|
||||
-- Token holders (snapshots)
|
||||
CREATE TABLE IF NOT EXISTS token_holders (
|
||||
chain String,
|
||||
network String,
|
||||
token_address String,
|
||||
holder_address String,
|
||||
balance UInt256,
|
||||
balance_usd Float64,
|
||||
timestamp DateTime64(3) DEFAULT now64(3)
|
||||
) ENGINE = ReplacingMergeTree(timestamp)
|
||||
PARTITION BY (chain, token_address)
|
||||
ORDER BY (chain, network, token_address, holder_address);
|
||||
|
||||
-- Materialized views for common queries
|
||||
|
||||
-- Hourly transaction stats per chain
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_tx_stats_hourly
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(hour))
|
||||
ORDER BY (chain, network, hour)
|
||||
AS SELECT
|
||||
chain,
|
||||
network,
|
||||
toStartOfHour(timestamp) as hour,
|
||||
count() as tx_count,
|
||||
countDistinct(from_address) as unique_senders,
|
||||
countDistinct(to_address) as unique_receivers,
|
||||
sum(gas_used * gas_price) as total_fees
|
||||
FROM transactions
|
||||
GROUP BY chain, network, hour;
|
||||
|
||||
-- Daily volume per token
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_token_volume_daily
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(day))
|
||||
ORDER BY (chain, network, token_address, day)
|
||||
AS SELECT
|
||||
chain,
|
||||
network,
|
||||
token_address,
|
||||
toDate(timestamp) as day,
|
||||
count() as transfer_count,
|
||||
countDistinct(from_address) as unique_senders,
|
||||
countDistinct(to_address) as unique_receivers,
|
||||
sum(value) as total_volume
|
||||
FROM token_transfers
|
||||
GROUP BY chain, network, token_address, day;
|
||||
|
||||
-- Daily DEX volume
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_dex_volume_daily
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(day))
|
||||
ORDER BY (chain, network, protocol, day)
|
||||
AS SELECT
|
||||
chain,
|
||||
network,
|
||||
protocol,
|
||||
toDate(timestamp) as day,
|
||||
count() as swap_count,
|
||||
countDistinct(trader) as unique_traders,
|
||||
sum(amount_usd) as total_volume_usd
|
||||
FROM dex_swaps
|
||||
GROUP BY chain, network, protocol, day;
|
||||
|
||||
-- Daily NFT sales
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_nft_sales_daily
|
||||
ENGINE = SummingMergeTree()
|
||||
PARTITION BY (chain, toYYYYMM(day))
|
||||
ORDER BY (chain, network, marketplace, day)
|
||||
AS SELECT
|
||||
chain,
|
||||
network,
|
||||
marketplace,
|
||||
toDate(timestamp) as day,
|
||||
count() as sale_count,
|
||||
countDistinct(collection_address) as collections_traded,
|
||||
countDistinct(buyer) as unique_buyers,
|
||||
sum(price_usd) as total_volume_usd
|
||||
FROM nft_sales
|
||||
GROUP BY chain, network, marketplace, day;
|
||||
Reference in New Issue
Block a user