pulsar/test/interoperability/pq_crystals/fetch.sh

#!/usr/bin/env bash
# Fetch the pq-crystals/dilithium reference implementation at the
# pinned commit used by Pulsar's cross-validation gate.
#
# pq-crystals is the construction reference for FIPS 204 ML-DSA
# (Dilithium was standardised AS ML-DSA in FIPS 204). The repository
# is at <https://github.com/pq-crystals/dilithium>. We do NOT vendor
# the tree into this repository for the same reason the libjade
# Jasmin track does not: the tree is ~MB-scale, its license is
# permissive (CC0 + Apache-2.0 dual), and re-cloning at the pinned
# commit on demand gives a deterministic, reviewer-reproducible
# artefact without bloating the Pulsar git history.
#
# After running this script:
#   ./libpqcrystals_dilithium2.a   ML-DSA-44 static archive
#   ./libpqcrystals_dilithium3.a   ML-DSA-65 static archive
#   ./libpqcrystals_dilithium5.a   ML-DSA-87 static archive
#
# These are linked by verifier.go via #cgo LDFLAGS. The Go binding
# only exposes the verification surface — the keygen and sign code
# paths are present in the archive but the binding does not call
# them; this keeps the "INDEPENDENT verifier" discipline tight
# (the cross-validator literally cannot do anything other than
# verify under FIPS 204's reference verifier).

set -euo pipefail

DILITHIUM_REPO="https://github.com/pq-crystals/dilithium.git"

# Pinned at submission-tag time. pq-crystals master tracks the
# standardised FIPS 204 ML-DSA construction (see upstream README).
# Update IN LOCKSTEP with the Pulsar submission tag so the artefact
# chain stays reproducible.
#
# Commit 6e00625 (2026 master HEAD at sign-off time) corresponds to
# the FIPS 204-aligned ref/ tree with the `crypto_sign_verify(sig,
# siglen, m, mlen, ctx, ctxlen, pk)` C API that this binding uses.
DILITHIUM_COMMIT="6e00625c5b29f516c6de973fe2ee2fbb150973f9"

# SHA-256 of `git archive --format=tar HEAD` at the pinned commit.
# `git archive` output is deterministic for a given tree object, so
# this hash is stable across reviewer clones. (GitHub's autogenerated
# tarball at /archive/<sha>.tar.gz is NOT stable — do not pin against
# it.) Computed from a fresh clone with:
#
#   git archive --format=tar HEAD | shasum -a 256
#
# Update when DILITHIUM_COMMIT is bumped.
DILITHIUM_TREE_SHA256="fa6221d8aaa14c9311144b18bc09a217794c3bad11d60a04366d2c3c33f4b48b"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

if [[ -d dilithium/.git ]]; then
    echo "==> pq-crystals/dilithium already cloned at $SCRIPT_DIR/dilithium"
    ( cd dilithium && git fetch --quiet origin && git checkout --quiet "$DILITHIUM_COMMIT" )
else
    echo "==> cloning pq-crystals/dilithium at $DILITHIUM_COMMIT"
    git clone --quiet "$DILITHIUM_REPO" dilithium
    ( cd dilithium && git checkout --quiet "$DILITHIUM_COMMIT" )
fi

# Verify the pinned tree hash. A mismatch means upstream rewrote
# history or our pin is stale — either way, fail loudly so the
# reviewer is not silently building against an unverified tree.
ACTUAL_SHA256="$(cd dilithium && git archive --format=tar HEAD | shasum -a 256 | awk '{print $1}')"
if [[ "$ACTUAL_SHA256" != "$DILITHIUM_TREE_SHA256" ]]; then
    echo "!! pq-crystals tree-hash mismatch at $DILITHIUM_COMMIT" >&2
    echo "!!   expected: $DILITHIUM_TREE_SHA256" >&2
    echo "!!   actual:   $ACTUAL_SHA256" >&2
    echo "!! upstream may have rewritten history; re-verify and re-pin" >&2
    exit 1
fi
echo "==> tree hash verified: $ACTUAL_SHA256"

# Compile the reference implementation into a single static archive
# per parameter set. Each archive contains the FIPS 204 sign /
# verify / keypair entry points for one parameter set, with all
# fips202 (SHAKE) helpers included. We never call sign or keypair
# from the cgo binding — only verify — but they are in the archive
# because they share TUs with verify.
#
# The reference Makefile builds .so shared libs that omit the
# fips202.c TU (it lives in libpqcrystals_fips202_ref.so). For a
# static archive consumed by cgo we want everything in one .a so
# the Go linker does not need a separate RPATH-eligible .so for
# the SHAKE helpers. We compile each TU once per parameter set
# (DILITHIUM_MODE) and pack with ar.

CC=${CC:-cc}
AR=${AR:-ar}
RANLIB=${RANLIB:-ranlib}

# pq-crystals' authoring style is -Wall -Wextra -Wpedantic clean;
# we mirror their CFLAGS so the archive matches their build.
#
# CFLAGS rationale:
#   -O3 -fomit-frame-pointer  matches upstream Makefile
#   -fPIC                     archive may be linked into a PIE
#                             (cgo on darwin/arm64 produces PIEs)
#   no -Werror                we want this to compile cleanly across
#                             clang minor versions without surprise
#                             warning-as-error breakage
DILCFLAGS=(-Wall -Wextra -Wpedantic -Wmissing-prototypes -Wredundant-decls
           -Wshadow -Wvla -Wpointer-arith -O3 -fomit-frame-pointer
           -fPIC)

# Per-parameter-set TUs. The pq-crystals symbol-namespacing scheme
# (DILITHIUM_NAMESPACE in params.h, FIPS202_NAMESPACE in symmetric.h)
# makes the compiled symbols distinct per mode (e.g.
# `pqcrystals_dilithium3_ref_verify`,
# `pqcrystals_dilithium3_ref_dilithium_shake128_stream_init`), so
# all three parameter sets must compile each TU under their own
# DILITHIUM_MODE — including `symmetric-shake.c`, which carries
# the dilithium_shake{128,256}_stream_init helpers that ARE mode-
# namespaced. Only `fips202.c` itself is mode-independent (its
# symbols live under `pqcrystals_dilithium_fips202_ref_*` with no
# mode token), so it is compiled exactly once.
DIL_MLDSA_SOURCES=(sign.c packing.c polyvec.c poly.c ntt.c reduce.c rounding.c
                   symmetric-shake.c)

# Shared FIPS 202 (SHAKE) TUs. NOT mode-dependent.
DIL_FIPS202_SOURCES=(fips202.c)

cd dilithium/ref

OBJDIR="../../obj.dilithium"
rm -rf "$OBJDIR"
mkdir -p "$OBJDIR"

# Compile shared FIPS 202 TUs once (mode-independent).
for src in "${DIL_FIPS202_SOURCES[@]}"; do
    obj="$OBJDIR/$(basename "$src" .c).o"
    "$CC" "${DILCFLAGS[@]}" -c "$src" -o "$obj"
done

# Compile ML-DSA TUs once per parameter set.
build_objs_for_mode() {
    local mode="$1"            # 2 | 3 | 5
    local suffix="$2"          # m2 | m3 | m5
    for src in "${DIL_MLDSA_SOURCES[@]}"; do
        obj="$OBJDIR/$(basename "$src" .c).${suffix}.o"
        "$CC" "${DILCFLAGS[@]}" -DDILITHIUM_MODE="$mode" -c "$src" -o "$obj"
    done
}
build_objs_for_mode 2 m2
build_objs_for_mode 3 m3
build_objs_for_mode 5 m5

# Compile a no-op randombytes stub. The pq-crystals reference impl
# pulls `randombytes` (declared in randombytes.h, defined in
# randombytes.c) from sign.c's keygen and signing entry points.
# Our cgo binding NEVER calls keygen or sign — only verify — but
# the unused keygen/sign symbols are present in the archive and
# would create an unresolved-symbol error at Go link time without
# a definition.
#
# Providing a stub that aborts loudly is the safer option than
# pulling in a real CSPRNG: if anyone ever links a Go test binary
# that accidentally calls the keygen or sign path, we want the
# process to die immediately rather than silently producing
# signatures with zero entropy. The "independent verifier"
# discipline lives at the Go API surface (verifier.go exposes
# only Verify*), so this stub is the belt-and-braces enforcement.
cat > "$OBJDIR/randombytes_stub.c" <<'EOF'
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>

void randombytes(uint8_t *out, size_t outlen);
void randombytes(uint8_t *out, size_t outlen) {
    /* If this function is ever called, the cgo binding has been
     * misused: the Pulsar interoperability harness only invokes
     * verify, never keygen or sign. Abort loudly so the test
     * process dies rather than silently producing all-zero
     * randomness. */
    (void)out;
    (void)outlen;
    abort();
}
EOF
"$CC" "${DILCFLAGS[@]}" -c "$OBJDIR/randombytes_stub.c" -o "$OBJDIR/randombytes_stub.o"

# Pack all objects into one archive. The mode-2/3/5 ML-DSA symbols
# do not collide with one another (namespace-distinct), the fips202
# symbols appear exactly once, and the randombytes stub satisfies
# the unused keygen/sign call sites.
ARCHIVE="../../libpqcrystals_dilithium.a"
rm -f "$ARCHIVE"
"$AR" rcs "$ARCHIVE" "$OBJDIR"/*.o
"$RANLIB" "$ARCHIVE"

cd "$SCRIPT_DIR"
echo
echo "==> pq-crystals static archive ready (44/65/87 in one .a):"
ls -la libpqcrystals_dilithium.a
echo
echo "==> verifying expected symbols:"
for mode in 2 3 5; do
    if ! nm libpqcrystals_dilithium.a 2>&1 | grep -q "T _pqcrystals_dilithium${mode}_ref_verify"; then
        echo "!! missing pqcrystals_dilithium${mode}_ref_verify symbol" >&2
        exit 1
    fi
    echo "    pqcrystals_dilithium${mode}_ref_verify  present"
done