universe/e2e/00-liveness.spec.ts

import { test, expect } from '@playwright/test'
import { env, LIVENESS, NETWORKS, orgRpc } from './env'
import {
  formatAge, getBlock, getTip, medianBlockGap, staleBudgetSec,
  getCurrentValidators, rpcCall,
} from './helpers'

// ---------------------------------------------------------------------------
// IS THE CHAIN ALIVE — the one invariant nothing in this suite asserted
// ---------------------------------------------------------------------------
// 00-health has nine tests that all reduce to `eth_blockNumber > 0`: "node
// reports healthy", "C-Chain is bootstrapped", "all primary chains
// bootstrapped". A chain frozen since Thursday answers every one of them.
//
// MEASURED 2026-07-26, and this is the whole reason the file exists:
//
//   mainnet 96369   tip 1,098,191   age 58.9 HOURS    ← all 15 pods 5/5 Ready
//   testnet 96368   tip     1,354   age  3.6 min
//   devnet  96367   tip       528   age  3.3 min
//
// Every public API returned 200 on all three. Readiness lied on all three. The
// tip did not.
//
// Four orthogonal questions, one test each — no test answers two:
//
//   1. FRESH      is the newest block recent, measured against the chain's own
//                 cadence (these chains are demand-driven; a fixed "block every
//                 N seconds" is wrong for them and a stall must not be able to
//                 widen its own budget — see LIVENESS in env.ts).
//   2. MONOTONIC  does the tip only ever move forward, and does a block that
//                 was final stay final (reorg / fork detection).
//   3. CONSISTENT do the chain's identity facts agree with the registry.
//   4. HONEST     when the P-Chain claims a full validator set, is the C-Chain
//                 tip actually fresh? This is the readiness lie, asserted
//                 directly: a bootstrapped-and-frozen network fails HERE, which
//                 is precisely the state nothing caught.
//
// On mainnet today tests 1 and 4 are RED. That is the correct result and it
// must stay red until the chain moves. Do not widen the budget.

const rpc = env.urls.rpc

test.describe(`Chain liveness — ${env.network} (chainId ${env.chainIds.lux})`, () => {
  test('tip is fresh against the chain\'s own cadence', async ({ request }) => {
    const tip = await getTip(request, rpc)
    const { median, gaps } = await medianBlockGap(request, rpc)
    const budget = staleBudgetSec(median)

    console.log(`  tip        #${tip.number.toLocaleString()}  ${tip.hash}`)
    console.log(`  tip age    ${formatAge(tip.ageSec)}  (${tip.ageSec}s)`)
    console.log(`  cadence    median ${median}s over ${gaps.length} gaps [${gaps.join(', ')}]`)
    console.log(`  budget     ${budget}s = max(${LIVENESS.staleGapMultiple}x median, `
      + `${LIVENESS.staleFloorSec}s floor)`)

    expect(tip.ageSec,
      `THE CHAIN HAS STOPPED. ${env.network} chainId ${env.chainIds.lux} is at block `
      + `${tip.number.toLocaleString()}, produced ${formatAge(tip.ageSec)} ago. Its own recent `
      + `cadence is ${median}s per block, so it is ${Math.round(tip.ageSec / Math.max(median, 1))} `
      + `blocks overdue. Readiness probes and HTTP 200s prove nothing about this.`)
      .toBeLessThanOrEqual(budget)
  })

  test('tip only moves forward and finalised blocks stay finalised', async ({ request }) => {
    // Deliberately NOT an advancement test. Testnet and devnet run a ~300s
    // heartbeat, so no honest window short enough for a test suite can require
    // a new block — that is what the freshness gate above is for. This asserts
    // the two things that ARE true of a healthy chain at every instant: the tip
    // never goes backwards, and a block already at the tip keeps its hash.
    // Four competing blockIDs at height 512 is what devnet looked like when it
    // was genuinely broken; that state fails here and passes everywhere else.
    const first = await getTip(request, rpc)
    await new Promise((r) => setTimeout(r, LIVENESS.reorgWindowSec * 1000))
    const second = await getTip(request, rpc)

    console.log(`  #${first.number.toLocaleString()} → #${second.number.toLocaleString()} `
      + `over ${LIVENESS.reorgWindowSec}s (+${second.number - first.number})`)

    expect(second.number,
      `the tip WENT BACKWARDS: #${first.number} → #${second.number}. The gateway is `
      + `serving from more than one chain state, or the chain reorganised.`)
      .toBeGreaterThanOrEqual(first.number)

    const again = await getBlock(request, rpc, first.number)
    expect(again?.hash,
      `block #${first.number} was ${first.hash} and is now ${again?.hash} — a REORG of a block `
      + `that had already been served as the tip.`).toBe(first.hash)
  })

  test('chain identity agrees with NETWORKS.yaml', async ({ request }) => {
    const netVersion = await rpcCall(request, rpc, 'net_version') as string
    console.log(`  net_version ${netVersion} vs registry ${env.chainIds.lux}`)
    expect(Number(netVersion),
      `${env.urls.rpc} reports chain ${netVersion} but NETWORKS.yaml declares `
      + `${env.chainIds.lux} for ${env.network} — the suite is pointed at the wrong chain.`)
      .toBe(env.chainIds.lux)
  })

  test('a bootstrapped validator set implies a moving chain', async ({ request }) => {
    // The readiness lie, asserted head-on. "P-Chain is bootstrapped" passed on
    // all three networks today; on one of them the C-Chain had been frozen for
    // two and a half days. Either the validator set is incomplete (and the
    // network says so honestly), or it is complete and the chain must be
    // producing. The pass-through state — full validator set, dead chain — is
    // the one this catches.
    const validators = await getCurrentValidators(request, env.urls.rpcPlatform) as { nodeID: string }[]
    const tip = await getTip(request, rpc)
    const { median } = await medianBlockGap(request, rpc)
    const budget = staleBudgetSec(median)

    console.log(`  validators ${validators.length}  tip #${tip.number.toLocaleString()} `
      + `age ${formatAge(tip.ageSec)}  budget ${budget}s`)

    expect(validators.length,
      `P-Chain reports ${validators.length} validators on ${env.network}`).toBeGreaterThanOrEqual(5)
    expect(tip.ageSec,
      `${validators.length} validators are bootstrapped and reporting healthy, and the C-Chain `
      + `has produced NOTHING for ${formatAge(tip.ageSec)}. This is the exact state that read `
      + `5/5 Ready across the fleet.`).toBeLessThanOrEqual(budget)
  })
})

// ── Every network, from any run ────────────────────────────────────────────
// The gate above tests the ACTIVE env. This block reads all three public
// networks in one pass, so a single `make e2e` says which of them is moving —
// the fleet view that made the mainnet freeze visible in the first place. Each
// network is its own test, so one frozen chain does not hide the other two.
test.describe('Chain liveness — the whole fleet', () => {
  for (const net of NETWORKS) {
    test(`${net} C-Chain is producing blocks`, async ({ request }) => {
      const url = orgRpc('lux', net)
      const tip = await getTip(request, url)
      const { median } = await medianBlockGap(request, url)
      const budget = staleBudgetSec(median)
      console.log(`  ${net.padEnd(8)} tip #${String(tip.number).padStart(9)} `
        + `age ${formatAge(tip.ageSec).padStart(7)}  cadence ${median}s  budget ${budget}s`)
      expect(tip.ageSec,
        `${net} (${url}) is frozen at #${tip.number.toLocaleString()}, `
        + `${formatAge(tip.ageSec)} old, cadence ${median}s`).toBeLessThanOrEqual(budget)
    })
  }
})