a peer is named by its address, so every node agrees who is in the room

Keygen reached round 2 and every node refused every message, including from
legitimate peers, so each attempt sat until the 300s timeout and every crypto
deposit answered 503 "the custody service is not accepting new addresses".

The handler admits a message only from an id in its OWN party set, and the
fleet had built three sets that disagreed:

  partyIDs=["mpc-node-0:keygen:1","mpc-node-1:keygen:1","peer-2:keygen:1"]

while node-2 called itself mpc-node-2:keygen:1. Nobody answers to "peer-2", so
its share never arrived and the ceremony could not complete.

The synthetic name is what an unprefixed --peer produces: the deployment passes
addresses only, self is dropped by address, and everything left took a
POSITIONAL id. The comment above that line already said such an id "cannot
match a real node id by construction" — it was describing this bug without
following it through.

So a peer is now named the same way self is recognised: the first DNS label,
which IS the node id under StatefulSet naming
(mpc-node-2.mpc-node-headless.ns.svc.cluster.local -> mpc-node-2). Deriving both
sides by one rule is what makes the sets agree.

A bare IP has no label to read and keeps the positional id — the caller must
then use nodeID@host:port, which states the name outright. The test asserts the
invariant that was violated rather than the helper's return value: the id a node
derives for a peer must equal the id that peer uses for itself.
This commit is contained in:
2026-08-05 16:36:04 -07:00
parent 1183d565d2
commit cbbecd063d
2 changed files with 94 additions and 2 deletions
+36 -1
View File
@@ -353,6 +353,41 @@ func isSelfAddr(addr, nodeID, listenAddr string) bool {
return label == nodeID
}
// peerIDFromAddr derives a peer's node id from its ADDRESS, which is the only
// form an unprefixed `--peer` carries.
//
// Every party must agree on the NAME of every other party, because the keygen
// handler admits a message only from an id in its own party set. A positional
// `peer-N` cannot match a real node id by construction, so a fleet whose flags
// are unprefixed builds THREE DIFFERENT party sets — mpc-node-1 called node-2
// "peer-2" while node-2 called itself "mpc-node-2" — and every round-2 message
// was refused by every node. Measured on hanzo-mpc: partyIDs
// ["mpc-node-0:keygen:1","mpc-node-1:keygen:1","peer-2:keygen:1"], zero messages
// accepted, keygen timing out at 300s, and each crypto deposit answering 503
// "the custody service is not accepting new addresses".
//
// The first DNS label IS the node id under StatefulSet naming
// (mpc-node-2.mpc-node-headless.ns.svc.cluster.local -> mpc-node-2), which is
// exactly the rule isSelfAddr already uses to recognise self. Deriving both
// sides the same way is what makes the sets agree.
//
// A bare IP has no label to read, so it keeps the positional id: the caller
// must then use the `nodeID@host:port` form, which states the name outright.
func peerIDFromAddr(addr string, index int) string {
host := strings.TrimSpace(addr)
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
label := host
if i := strings.IndexByte(host, '.'); i >= 0 {
label = host[:i]
}
if label == "" || net.ParseIP(host) != nil {
return fmt.Sprintf("peer-%d", index)
}
return label
}
func runNodeConsensus(ctx context.Context, c *cli.Command) error {
nodeID := c.String("node-id")
listenAddr := c.String("listen")
@@ -449,7 +484,7 @@ func runNodeConsensus(ctx context.Context, c *cli.Command) error {
if isSelfAddr(peer, nodeID, listenAddr) {
continue
}
peerMap[fmt.Sprintf("peer-%d", i)] = peer
peerMap[peerIDFromAddr(peer, i)] = peer
}
// Get ZapDB password via HSM provider (supports AWS KMS, GCP Cloud KMS, Azure Key Vault, env, file)
+58 -1
View File
@@ -1,6 +1,9 @@
package main
import "testing"
import (
"strings"
"testing"
)
// The k8s deployment shape: ONE StatefulSet command for every replica, so each
// node is handed the WHOLE ensemble — including itself. Self must be dropped
@@ -52,3 +55,57 @@ func TestIsSelfAddrEmptyNodeIDMatchesNothing(t *testing.T) {
}
}
}
// TestPeerIDFromAddr — every party must agree on the NAME of every other party,
// because the keygen handler admits a message only from an id in its own party
// set. A positional id cannot match a real node id, so unprefixed --peer flags
// built three disagreeing sets and every round-2 message was refused.
func TestPeerIDFromAddr(t *testing.T) {
for _, tc := range []struct {
name, addr, want string
index int
}{
{"statefulset fqdn", "mpc-node-2.mpc-node-headless.hanzo-mpc.svc.cluster.local:9999", "mpc-node-2", 2},
{"short host", "mpc-node-0:9999", "mpc-node-0", 0},
{"no port", "mpc-node-1.mpc-node-headless.hanzo-mpc.svc.cluster.local", "mpc-node-1", 1},
// A bare IP carries no name, so the positional id remains and the caller
// must use nodeID@host:port to say who it is.
{"bare ipv4 keeps positional", "10.0.0.5:9999", "peer-3", 3},
{"bare ipv6 keeps positional", "[fd00::1]:9999", "peer-4", 4},
} {
t.Run(tc.name, func(t *testing.T) {
if got := peerIDFromAddr(tc.addr, tc.index); got != tc.want {
t.Fatalf("peerIDFromAddr(%q, %d) = %q, want %q", tc.addr, tc.index, got, tc.want)
}
})
}
}
// TestPeerIDMatchesSelfRecognition is the invariant that was violated in
// production: the id a node derives for a peer must equal the id that peer uses
// for ITSELF. isSelfAddr reads the first DNS label to recognise self, so
// peerIDFromAddr must read it the same way or the two disagree.
func TestPeerIDMatchesSelfRecognition(t *testing.T) {
fleet := []string{
"mpc-node-0.mpc-node-headless.hanzo-mpc.svc.cluster.local:9999",
"mpc-node-1.mpc-node-headless.hanzo-mpc.svc.cluster.local:9999",
"mpc-node-2.mpc-node-headless.hanzo-mpc.svc.cluster.local:9999",
}
for _, self := range []string{"mpc-node-0", "mpc-node-1", "mpc-node-2"} {
for i, addr := range fleet {
id := peerIDFromAddr(addr, i)
if isSelfAddr(addr, self, ":9999") {
if id != self {
t.Fatalf("%s: recognises %q as self but would name it %q", self, addr, id)
}
continue
}
if id == self {
t.Fatalf("%s: named a PEER with its own id (%q)", self, id)
}
if strings.HasPrefix(id, "peer-") {
t.Fatalf("%s: peer %q got positional id %q — no other node answers to that name", self, addr, id)
}
}
}
}