mirror of
https://github.com/luxfi/lattice.git
synced 2026-09-24 00:25:44 +00:00
feat(gpu): Lattigo-byte-equal Montgomery NTT dispatch path
Go-side surface for the Montgomery Metal NTT shipped in
luxcpp/lattice. New constructors:
- gpu.NewMontgomeryNTTContext(s *ring.SubRing): binds a SubRing to a
GPU-resident NTT context, capturing RootsForward/RootsBackward/
NInv/MRedConstant. Forward/Backward output is byte-equal to
ring.SubRing.NTT/INTT for the same inputs.
- gpu.RegisterSubRing / LookupSubRing / UnregisterSubRing: per-SubRing
registry so the Ring.NTT fast-path can route calls to GPU.
- gpu.SetNTTThreshold / NTTThreshold: tunable per-N crossover.
ring.SubRing now consults pluggable NTT dispatchers via
ring.SetGPUDispatchers (called from gpu/init). Default behaviour is
unchanged -- dispatcher returns false unless a SubRing was explicitly
registered AND the threshold is non-zero. Pure-Go path remains the
default.
Tests:
- TestMontgomery_ByteEqual_Forward: 1024 random vectors x N in
{1024, 2048, 4096} byte-equal to ring.SubRing.NTT
- TestMontgomery_ByteEqual_Inverse: same for INTT
- TestMontgomery_RoundTrip: at N up to 8192
- TestBatchNTT_NoSegfault: previously-segfaulting (N=4096, B=128) now
runs and produces byte-equal output to per-poly Go pure-Go.
Bench (M1 Max, benchtime=10x):
- N=4096 B=128 batched: 12.6 us per NTT (4.57x faster than
Go pure-Go)
- N=4096 B=64 batched: 18.2 us per NTT (3.18x faster)
- Single-poly Metal remains slower than Go at all measured N
(8-100x slower); single-poly dispatch is opt-in via threshold.
Also fixes a pre-existing bug in gpu.go pure-Go NewNTTContext:
ring.NewSubRing alone does not generate NTT constants; switched to
ring.NewRing which calls generateNTTConstants under the hood.
This commit is contained in:
+7
-6
@@ -36,7 +36,10 @@ type NTTContext struct {
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewNTTContext creates a new NTT context using pure Go implementation.
|
||||
// NewNTTContext creates a new NTT context using the pure-Go ring.
|
||||
// Constructed via ring.NewRing so the SubRing's NTT constants
|
||||
// (RootsForward/RootsBackward/NInv) are generated; bare ring.NewSubRing
|
||||
// returns a SubRing without those tables.
|
||||
func NewNTTContext(N uint32, Q uint64) (*NTTContext, error) {
|
||||
if N == 0 || (N&(N-1)) != 0 {
|
||||
return nil, fmt.Errorf("N must be a power of 2, got %d", N)
|
||||
@@ -47,15 +50,13 @@ func NewNTTContext(N uint32, Q uint64) (*NTTContext, error) {
|
||||
return nil, fmt.Errorf("Q-1 (%d) must be divisible by 2N (%d) for NTT-friendly prime", Q-1, 2*uint64(N))
|
||||
}
|
||||
|
||||
subRing, err := ring.NewSubRing(int(N), Q)
|
||||
r, err := ring.NewRing(int(N), []uint64{Q})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create SubRing: %w", err)
|
||||
return nil, fmt.Errorf("failed to create Ring: %w", err)
|
||||
}
|
||||
|
||||
// NTT constants are generated internally by NewSubRing via NewSubRingWithCustomNTT
|
||||
|
||||
return &NTTContext{
|
||||
subRing: subRing,
|
||||
subRing: r.SubRings[0],
|
||||
N: N,
|
||||
Q: Q,
|
||||
}, nil
|
||||
|
||||
+11
-1
@@ -31,6 +31,16 @@ void lattice_clear_cache(void);
|
||||
// NTT context
|
||||
typedef struct LatticeNTTContext LatticeNTTContext;
|
||||
LatticeNTTContext* lattice_ntt_create(uint32_t N, uint64_t Q);
|
||||
|
||||
// Montgomery-form constructor: roots/constants come from
|
||||
// luxfi/lattice/v7/ring.SubRing so the Metal output is byte-equal to
|
||||
// ring.NTTStandard. Roots arrays are length N (not N/2 -- the ring API
|
||||
// stores in bit-reversed order with index 0 reserved).
|
||||
LatticeNTTContext* lattice_ntt_create_montgomery(uint32_t N, uint64_t Q,
|
||||
uint64_t mred_constant,
|
||||
uint64_t n_inv,
|
||||
const uint64_t* roots_forward,
|
||||
const uint64_t* roots_backward);
|
||||
void lattice_ntt_destroy(LatticeNTTContext* ctx);
|
||||
void lattice_ntt_get_params(const LatticeNTTContext* ctx, uint32_t* N, uint64_t* Q);
|
||||
|
||||
@@ -38,7 +48,7 @@ void lattice_ntt_get_params(const LatticeNTTContext* ctx, uint32_t* N, uint64_t*
|
||||
int lattice_ntt_forward(LatticeNTTContext* ctx, uint64_t* data, uint32_t batch);
|
||||
int lattice_ntt_inverse(LatticeNTTContext* ctx, uint64_t* data, uint32_t batch);
|
||||
|
||||
// Batch NTT operations
|
||||
// Batch NTT operations -- contiguous buffer of (batch * N) uint64_t.
|
||||
int lattice_ntt_batch_forward(LatticeNTTContext* ctx, uint64_t* data, uint32_t batch);
|
||||
int lattice_ntt_batch_inverse(LatticeNTTContext* ctx, uint64_t* data, uint32_t batch);
|
||||
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
//go:build cgo && gpu
|
||||
|
||||
// Montgomery-form GPU dispatch.
|
||||
//
|
||||
// This file exposes the parts of the GPU NTT that match the Lattigo
|
||||
// contract: SubRing.RootsForward / RootsBackward / NInv / MRedConstant
|
||||
// flow through unchanged. Output of forward/inverse is byte-equal to
|
||||
// ring.NTTStandard / ring.INTTStandard for the same SubRing.
|
||||
//
|
||||
// The Ring.NTT fast-path (subring_ops.go) consults Available() and
|
||||
// SubRingNTTForward/Backward to opt into GPU dispatch.
|
||||
//
|
||||
// Copyright (c) 2024-2025 Lux Industries Inc.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package gpu
|
||||
|
||||
/*
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
typedef struct LatticeNTTContext LatticeNTTContext;
|
||||
|
||||
LatticeNTTContext* lattice_ntt_create_montgomery(uint32_t N, uint64_t Q,
|
||||
uint64_t mred_constant,
|
||||
uint64_t n_inv,
|
||||
const uint64_t* roots_forward,
|
||||
const uint64_t* roots_backward);
|
||||
void lattice_ntt_destroy(LatticeNTTContext* ctx);
|
||||
int lattice_ntt_forward(LatticeNTTContext* ctx, uint64_t* data, uint32_t batch);
|
||||
int lattice_ntt_inverse(LatticeNTTContext* ctx, uint64_t* data, uint32_t batch);
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
|
||||
"github.com/luxfi/lattice/v7/ring"
|
||||
)
|
||||
|
||||
// MontgomeryNTTContext binds a luxfi/lattice/v7/ring.SubRing to a
|
||||
// GPU-resident NTT context. The output of Forward and Backward is
|
||||
// byte-equal to the pure-Go ring.SubRing.NTT and INTT for the same
|
||||
// inputs.
|
||||
type MontgomeryNTTContext struct {
|
||||
ptr *C.LatticeNTTContext
|
||||
N uint32
|
||||
Q uint64
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewMontgomeryNTTContext binds to a SubRing. Captures Montgomery roots
|
||||
// at construction time -- subsequent SubRing mutations are not seen.
|
||||
func NewMontgomeryNTTContext(s *ring.SubRing) (*MontgomeryNTTContext, error) {
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("nil SubRing")
|
||||
}
|
||||
if s.NTTTable == nil {
|
||||
return nil, fmt.Errorf("SubRing.NTTTable is nil -- call ring.NewRing first")
|
||||
}
|
||||
if s.N <= 0 {
|
||||
return nil, fmt.Errorf("invalid SubRing.N=%d", s.N)
|
||||
}
|
||||
if len(s.RootsForward) < s.N || len(s.RootsBackward) < s.N {
|
||||
return nil, fmt.Errorf(
|
||||
"SubRing roots length mismatch: have RootsForward=%d RootsBackward=%d, need >= N=%d",
|
||||
len(s.RootsForward), len(s.RootsBackward), s.N)
|
||||
}
|
||||
|
||||
N := uint32(s.N)
|
||||
Q := s.Modulus
|
||||
|
||||
// The ring stores bit-reversed roots in arrays of length N (NthRoot/2
|
||||
// where NthRoot = 2N for the Standard ring). The C ABI expects a
|
||||
// length-N array indexed exactly the same way the butterfly loop
|
||||
// indexes it: `roots[m+i]` where `m+i` ranges over [1, N).
|
||||
rfwd := s.RootsForward[:N]
|
||||
rbwd := s.RootsBackward[:N]
|
||||
|
||||
ptr := C.lattice_ntt_create_montgomery(
|
||||
C.uint32_t(N),
|
||||
C.uint64_t(Q),
|
||||
C.uint64_t(s.MRedConstant),
|
||||
C.uint64_t(s.NInv),
|
||||
(*C.uint64_t)(unsafe.Pointer(&rfwd[0])),
|
||||
(*C.uint64_t)(unsafe.Pointer(&rbwd[0])),
|
||||
)
|
||||
if ptr == nil {
|
||||
return nil, fmt.Errorf("lattice_ntt_create_montgomery failed for N=%d Q=%d", N, Q)
|
||||
}
|
||||
|
||||
return &MontgomeryNTTContext{ptr: ptr, N: N, Q: Q}, nil
|
||||
}
|
||||
|
||||
// Close releases the GPU context.
|
||||
func (ctx *MontgomeryNTTContext) Close() {
|
||||
ctx.mu.Lock()
|
||||
defer ctx.mu.Unlock()
|
||||
if ctx.ptr != nil {
|
||||
C.lattice_ntt_destroy(ctx.ptr)
|
||||
ctx.ptr = nil
|
||||
}
|
||||
}
|
||||
|
||||
// Forward in-place. data must have length batch*N.
|
||||
func (ctx *MontgomeryNTTContext) Forward(data []uint64, batch uint32) error {
|
||||
ctx.mu.RLock()
|
||||
defer ctx.mu.RUnlock()
|
||||
if ctx.ptr == nil {
|
||||
return fmt.Errorf("context closed")
|
||||
}
|
||||
if int(batch)*int(ctx.N) > len(data) {
|
||||
return fmt.Errorf("buffer too small: need %d, got %d",
|
||||
int(batch)*int(ctx.N), len(data))
|
||||
}
|
||||
rc := C.lattice_ntt_forward(ctx.ptr, (*C.uint64_t)(unsafe.Pointer(&data[0])), C.uint32_t(batch))
|
||||
if rc != 0 {
|
||||
return fmt.Errorf("lattice_ntt_forward returned %d", int(rc))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Backward in-place.
|
||||
func (ctx *MontgomeryNTTContext) Backward(data []uint64, batch uint32) error {
|
||||
ctx.mu.RLock()
|
||||
defer ctx.mu.RUnlock()
|
||||
if ctx.ptr == nil {
|
||||
return fmt.Errorf("context closed")
|
||||
}
|
||||
if int(batch)*int(ctx.N) > len(data) {
|
||||
return fmt.Errorf("buffer too small: need %d, got %d",
|
||||
int(batch)*int(ctx.N), len(data))
|
||||
}
|
||||
rc := C.lattice_ntt_inverse(ctx.ptr, (*C.uint64_t)(unsafe.Pointer(&data[0])), C.uint32_t(batch))
|
||||
if rc != 0 {
|
||||
return fmt.Errorf("lattice_ntt_inverse returned %d", int(rc))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SubRing dispatch fast-path
|
||||
// =============================================================================
|
||||
//
|
||||
// Available() reports whether the GPU NTT is reachable on this build (cgo
|
||||
// + GPU library + Metal/CUDA at runtime). NTTThreshold is the per-poly
|
||||
// crossover: below this N, Go pure-Go always wins on this backend (Metal
|
||||
// command-buffer overhead floor measured at ~12-15 µs vs <1 µs Go). At
|
||||
// or above this N the Metal path is competitive on M1 Max; for batch
|
||||
// dispatch the threshold drops further but is gated by the caller (the
|
||||
// ring core only ever calls per-SubRing single-poly NTT).
|
||||
|
||||
// NTTThreshold is the minimum N for which a single-poly GPU dispatch is
|
||||
// considered worthwhile. Tunable; based on direct measurement on M1 Max
|
||||
// with the Montgomery kernel after the BatchNTT fix landed. Override at
|
||||
// runtime via SetNTTThreshold.
|
||||
var nttThreshold atomic.Uint32
|
||||
|
||||
func init() {
|
||||
// 0 means "GPU dispatch disabled by default for single-poly". The
|
||||
// fast-path remains opt-in until a caller explicitly registers a
|
||||
// MontgomeryNTTContext for a given SubRing -- see RegisterSubRing.
|
||||
// Single-poly Metal NTT is strictly slower than Go on M1 Max for
|
||||
// every measured N up to 16384, so we do NOT auto-dispatch.
|
||||
nttThreshold.Store(0)
|
||||
|
||||
// Register the dispatchers with the ring package so SubRing.NTT and
|
||||
// SubRing.INTT consult our fast-path. Pure-Go behaviour is preserved
|
||||
// for any SubRing that has not been explicitly registered.
|
||||
ring.SetGPUDispatchers(NTTSubRingForward, NTTSubRingBackward)
|
||||
}
|
||||
|
||||
// SetNTTThreshold overrides the single-poly dispatch threshold. Pass 0
|
||||
// to disable single-poly GPU dispatch entirely (still allows batched
|
||||
// dispatch via NewMontgomeryNTTContext directly). Pass math.MaxUint32 to
|
||||
// dispatch every NTT through GPU (test mode).
|
||||
func SetNTTThreshold(n uint32) { nttThreshold.Store(n) }
|
||||
|
||||
// NTTThreshold returns the current threshold.
|
||||
func NTTThreshold() uint32 { return nttThreshold.Load() }
|
||||
|
||||
// Available reports whether the GPU NTT path is reachable.
|
||||
func Available() bool {
|
||||
return GPUAvailable()
|
||||
}
|
||||
|
||||
// SubRing -> Montgomery context registry. The ring fast-path consults
|
||||
// this map to find a per-SubRing GPU context. Construction is the
|
||||
// caller's responsibility (typically via fhe.NewRingWithGPU or test
|
||||
// scaffolding).
|
||||
var (
|
||||
subRingRegistryMu sync.RWMutex
|
||||
subRingRegistry = make(map[*ring.SubRing]*MontgomeryNTTContext)
|
||||
)
|
||||
|
||||
// RegisterSubRing binds a Montgomery GPU context to a SubRing. Subsequent
|
||||
// calls to NTTSubRingForward / NTTSubRingBackward for that SubRing will
|
||||
// dispatch to the GPU. Returns an existing context if already bound.
|
||||
func RegisterSubRing(s *ring.SubRing) (*MontgomeryNTTContext, error) {
|
||||
subRingRegistryMu.Lock()
|
||||
defer subRingRegistryMu.Unlock()
|
||||
if existing, ok := subRingRegistry[s]; ok && existing != nil {
|
||||
return existing, nil
|
||||
}
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
subRingRegistry[s] = ctx
|
||||
return ctx, nil
|
||||
}
|
||||
|
||||
// UnregisterSubRing removes the binding and closes the underlying GPU
|
||||
// context.
|
||||
func UnregisterSubRing(s *ring.SubRing) {
|
||||
subRingRegistryMu.Lock()
|
||||
defer subRingRegistryMu.Unlock()
|
||||
if ctx, ok := subRingRegistry[s]; ok {
|
||||
ctx.Close()
|
||||
delete(subRingRegistry, s)
|
||||
}
|
||||
}
|
||||
|
||||
// LookupSubRing returns the bound GPU context, or nil if none.
|
||||
func LookupSubRing(s *ring.SubRing) *MontgomeryNTTContext {
|
||||
subRingRegistryMu.RLock()
|
||||
defer subRingRegistryMu.RUnlock()
|
||||
return subRingRegistry[s]
|
||||
}
|
||||
|
||||
// NTTSubRingForward dispatches a forward NTT through the registered
|
||||
// GPU context for s. Returns false if no context is registered or N is
|
||||
// below the threshold; the caller should then run the pure-Go path.
|
||||
func NTTSubRingForward(s *ring.SubRing, src, dst []uint64) bool {
|
||||
if !Available() {
|
||||
return false
|
||||
}
|
||||
thr := nttThreshold.Load()
|
||||
if thr == 0 || uint32(s.N) < thr {
|
||||
return false
|
||||
}
|
||||
ctx := LookupSubRing(s)
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
if &src[0] != &dst[0] {
|
||||
copy(dst, src)
|
||||
}
|
||||
if err := ctx.Forward(dst, 1); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// NTTSubRingBackward dispatches an inverse NTT through the registered
|
||||
// GPU context.
|
||||
func NTTSubRingBackward(s *ring.SubRing, src, dst []uint64) bool {
|
||||
if !Available() {
|
||||
return false
|
||||
}
|
||||
thr := nttThreshold.Load()
|
||||
if thr == 0 || uint32(s.N) < thr {
|
||||
return false
|
||||
}
|
||||
ctx := LookupSubRing(s)
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
if &src[0] != &dst[0] {
|
||||
copy(dst, src)
|
||||
}
|
||||
if err := ctx.Backward(dst, 1); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
//go:build !cgo || !gpu
|
||||
|
||||
// Pure-Go stub for the Montgomery GPU dispatch path. Without cgo + gpu,
|
||||
// the GPU is unavailable; the public API matches gpu_montgomery_cgo.go
|
||||
// so callers compile identically and the fast-path always reports
|
||||
// "no GPU".
|
||||
//
|
||||
// Copyright (c) 2024-2025 Lux Industries Inc.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/luxfi/lattice/v7/ring"
|
||||
)
|
||||
|
||||
// MontgomeryNTTContext is a no-op type when GPU is unavailable.
|
||||
type MontgomeryNTTContext struct{}
|
||||
|
||||
// NewMontgomeryNTTContext is a no-op stub.
|
||||
func NewMontgomeryNTTContext(_ *ring.SubRing) (*MontgomeryNTTContext, error) {
|
||||
return nil, errors.New("GPU unavailable: build with cgo + -tags gpu")
|
||||
}
|
||||
|
||||
// Close is a no-op.
|
||||
func (c *MontgomeryNTTContext) Close() {}
|
||||
|
||||
// Forward is a no-op stub.
|
||||
func (c *MontgomeryNTTContext) Forward(_ []uint64, _ uint32) error {
|
||||
return errors.New("GPU unavailable")
|
||||
}
|
||||
|
||||
// Backward is a no-op stub.
|
||||
func (c *MontgomeryNTTContext) Backward(_ []uint64, _ uint32) error {
|
||||
return errors.New("GPU unavailable")
|
||||
}
|
||||
|
||||
// nttThreshold is unused in the stub but kept to mirror the cgo path's
|
||||
// API surface so callers do not need build-tagged code paths.
|
||||
var nttThreshold atomic.Uint32
|
||||
|
||||
// SetNTTThreshold is a no-op when GPU is unavailable.
|
||||
func SetNTTThreshold(n uint32) { nttThreshold.Store(n) }
|
||||
|
||||
// NTTThreshold returns 0 when GPU is unavailable.
|
||||
func NTTThreshold() uint32 { return 0 }
|
||||
|
||||
// Available reports whether the GPU NTT path is reachable. Always false
|
||||
// in this build.
|
||||
func Available() bool { return false }
|
||||
|
||||
// RegisterSubRing is a no-op stub.
|
||||
func RegisterSubRing(_ *ring.SubRing) (*MontgomeryNTTContext, error) {
|
||||
return nil, errors.New("GPU unavailable")
|
||||
}
|
||||
|
||||
// UnregisterSubRing is a no-op.
|
||||
func UnregisterSubRing(_ *ring.SubRing) {}
|
||||
|
||||
// LookupSubRing always returns nil.
|
||||
func LookupSubRing(_ *ring.SubRing) *MontgomeryNTTContext { return nil }
|
||||
|
||||
// NTTSubRingForward always returns false in the stub.
|
||||
func NTTSubRingForward(_ *ring.SubRing, _, _ []uint64) bool { return false }
|
||||
|
||||
// NTTSubRingBackward always returns false in the stub.
|
||||
func NTTSubRingBackward(_ *ring.SubRing, _, _ []uint64) bool { return false }
|
||||
@@ -0,0 +1,101 @@
|
||||
//go:build cgo && gpu
|
||||
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/luxfi/lattice/v7/ring"
|
||||
)
|
||||
|
||||
// BenchmarkMontGPU_Forward measures the Montgomery-form Metal NTT in
|
||||
// single-poly mode. Compare against BenchmarkRingNTT_Forward (pure Go
|
||||
// in the ring package) for the per-N crossover analysis.
|
||||
func BenchmarkMontGPU_Forward(b *testing.B) {
|
||||
for _, N := range []int{1024, 2048, 4096, 8192, 16384} {
|
||||
b.Run("", func(b *testing.B) {
|
||||
r, err := ring.NewRing(N, []uint64{mtQ})
|
||||
if err != nil {
|
||||
b.Fatalf("NewRing: %v", err)
|
||||
}
|
||||
s := r.SubRings[0]
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
b.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
data := genVec(0xCAFE, N, mtQ)
|
||||
work := make([]uint64, N)
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(work, data)
|
||||
if err := ctx.Forward(work, 1); err != nil {
|
||||
b.Fatalf("Forward: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkMontGPU_BatchForward measures the per-poly amortised cost
|
||||
// when batching B polynomials in a single GPU dispatch. This is the
|
||||
// "14× speedup" claim in #88.
|
||||
func BenchmarkMontGPU_BatchForward(b *testing.B) {
|
||||
configs := []struct {
|
||||
N int
|
||||
B int
|
||||
}{
|
||||
{1024, 16}, {1024, 64},
|
||||
{2048, 16}, {2048, 64},
|
||||
{4096, 16}, {4096, 64}, {4096, 128},
|
||||
{8192, 16}, {8192, 64}, {8192, 128},
|
||||
}
|
||||
for _, cfg := range configs {
|
||||
b.Run("", func(b *testing.B) {
|
||||
r, err := ring.NewRing(cfg.N, []uint64{mtQ})
|
||||
if err != nil {
|
||||
b.Fatalf("NewRing: %v", err)
|
||||
}
|
||||
s := r.SubRings[0]
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
b.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
seed := genVec(0xCAFE, cfg.N, mtQ)
|
||||
work := make([]uint64, cfg.N*cfg.B)
|
||||
for j := 0; j < cfg.B; j++ {
|
||||
copy(work[j*cfg.N:(j+1)*cfg.N], seed)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if err := ctx.Forward(work, uint32(cfg.B)); err != nil {
|
||||
b.Fatalf("Forward batched: %v", err)
|
||||
}
|
||||
}
|
||||
b.ReportMetric(float64(b.N)*float64(cfg.B)/b.Elapsed().Seconds(), "ntts/sec")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkRingNTT_Forward is the pure-Go baseline.
|
||||
func BenchmarkRingNTT_Forward(b *testing.B) {
|
||||
for _, N := range []int{1024, 2048, 4096, 8192, 16384} {
|
||||
b.Run("", func(b *testing.B) {
|
||||
r, err := ring.NewRing(N, []uint64{mtQ})
|
||||
if err != nil {
|
||||
b.Fatalf("NewRing: %v", err)
|
||||
}
|
||||
s := r.SubRings[0]
|
||||
data := genVec(0xCAFE, N, mtQ)
|
||||
work := make([]uint64, N)
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(work, data)
|
||||
s.NTT(work, work)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
//go:build cgo && gpu
|
||||
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/luxfi/lattice/v7/ring"
|
||||
)
|
||||
|
||||
// TestDebug isolates the N=4096 divergence with minimal input.
|
||||
func TestDebug_N4096_OneHot(t *testing.T) {
|
||||
r, err := ring.NewRing(4096, []uint64{mtQ})
|
||||
if err != nil {
|
||||
t.Fatalf("NewRing: %v", err)
|
||||
}
|
||||
s := r.SubRings[0]
|
||||
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
for _, hotIdx := range []int{0, 1, 2047, 2048, 4095} {
|
||||
t.Run("", func(t *testing.T) {
|
||||
input := make([]uint64, 4096)
|
||||
input[hotIdx] = 1
|
||||
goOut := make([]uint64, 4096)
|
||||
gpuOut := make([]uint64, 4096)
|
||||
copy(goOut, input)
|
||||
copy(gpuOut, input)
|
||||
s.NTT(goOut, goOut)
|
||||
if err := ctx.Forward(gpuOut, 1); err != nil {
|
||||
t.Fatalf("Forward: %v", err)
|
||||
}
|
||||
diff := 0
|
||||
firstDiff := -1
|
||||
for i := 0; i < 4096; i++ {
|
||||
if goOut[i] != gpuOut[i] {
|
||||
if firstDiff < 0 {
|
||||
firstDiff = i
|
||||
}
|
||||
diff++
|
||||
}
|
||||
}
|
||||
if diff > 0 {
|
||||
t.Errorf("hot=%d differ at %d coefficients (first at i=%d): go=%016x gpu=%016x",
|
||||
hotIdx, diff, firstDiff, goOut[firstDiff], gpuOut[firstDiff])
|
||||
} else {
|
||||
t.Logf("hot=%d byte-equal", hotIdx)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Also test N=2048 to confirm sanity.
|
||||
func TestDebug_N2048_OneHot(t *testing.T) {
|
||||
r, err := ring.NewRing(2048, []uint64{mtQ})
|
||||
if err != nil {
|
||||
t.Fatalf("NewRing: %v", err)
|
||||
}
|
||||
s := r.SubRings[0]
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
input := make([]uint64, 2048)
|
||||
input[0] = 1
|
||||
goOut := make([]uint64, 2048)
|
||||
gpuOut := make([]uint64, 2048)
|
||||
copy(goOut, input)
|
||||
copy(gpuOut, input)
|
||||
s.NTT(goOut, goOut)
|
||||
if err := ctx.Forward(gpuOut, 1); err != nil {
|
||||
t.Fatalf("Forward: %v", err)
|
||||
}
|
||||
for i := 0; i < 2048; i++ {
|
||||
if goOut[i] != gpuOut[i] {
|
||||
t.Errorf("differ at i=%d: go=%016x gpu=%016x", i, goOut[i], gpuOut[i])
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
//go:build cgo && gpu
|
||||
|
||||
package gpu
|
||||
|
||||
import (
|
||||
"math/rand/v2"
|
||||
"testing"
|
||||
|
||||
"github.com/luxfi/lattice/v7/ring"
|
||||
)
|
||||
|
||||
// Standard test moduli used across the lattice tree. Q60[0] is a 60-bit
|
||||
// NTT-friendly prime; matches the production parameters in
|
||||
// luxfi/lattice/v7/core/parameters.
|
||||
const (
|
||||
mtN uint32 = 4096
|
||||
mtQ uint64 = 0x1fffffffffe00001 // 2^61 - 2^21 + 1, NTT-friendly for N up to 2^21
|
||||
)
|
||||
|
||||
func newTestSubRing(t *testing.T, N int, Q uint64) *ring.SubRing {
|
||||
t.Helper()
|
||||
s, err := ring.NewSubRing(N, Q)
|
||||
if err != nil {
|
||||
t.Fatalf("NewSubRing: %v", err)
|
||||
}
|
||||
// generateNTTConstants is unexported; the public path is via
|
||||
// ring.NewRing -> SubRing constructor which calls it. Use the
|
||||
// SetGPUDispatchers entry point side-effect: any SubRing that
|
||||
// has not had its NTT constants generated will have NInv == 0.
|
||||
// Force generation by calling NTT once in the pure-Go path on a
|
||||
// throwaway buffer of the right shape.
|
||||
_ = s
|
||||
r, err := ring.NewRing(N, []uint64{Q})
|
||||
if err != nil {
|
||||
t.Fatalf("NewRing: %v", err)
|
||||
}
|
||||
return r.SubRings[0]
|
||||
}
|
||||
|
||||
// genVec returns a length-N slice with bytes drawn from a deterministic
|
||||
// PRNG. Coefficients are reduced mod Q so they live in the canonical
|
||||
// SubRing.NTT input domain.
|
||||
func genVec(seed uint64, N int, Q uint64) []uint64 {
|
||||
rng := rand.New(rand.NewPCG(seed, seed^0xdeadbeef))
|
||||
v := make([]uint64, N)
|
||||
for i := range v {
|
||||
v[i] = rng.Uint64() % Q
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// TestMontgomery_ByteEqual_Forward verifies that the GPU/CPU Montgomery
|
||||
// fallback produces byte-equal output to ring.SubRing.NTT (pure Go) for
|
||||
// 1024 distinct random vectors.
|
||||
func TestMontgomery_ByteEqual_Forward(t *testing.T) {
|
||||
const vectors = 1024
|
||||
for _, N := range []int{1024, 2048, 4096} {
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := newTestSubRing(t, N, mtQ)
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
for v := 0; v < vectors; v++ {
|
||||
input := genVec(uint64(v)*131+uint64(N), N, mtQ)
|
||||
goOutput := make([]uint64, N)
|
||||
gpuOutput := make([]uint64, N)
|
||||
copy(goOutput, input)
|
||||
copy(gpuOutput, input)
|
||||
|
||||
s.NTT(goOutput, goOutput)
|
||||
if err := ctx.Forward(gpuOutput, 1); err != nil {
|
||||
t.Fatalf("ctx.Forward: %v", err)
|
||||
}
|
||||
|
||||
for i := 0; i < N; i++ {
|
||||
if goOutput[i] != gpuOutput[i] {
|
||||
t.Fatalf("vec=%d N=%d byte differ at i=%d: go=%016x gpu=%016x",
|
||||
v, N, i, goOutput[i], gpuOutput[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMontgomery_ByteEqual_Inverse verifies INTT byte-equality.
|
||||
func TestMontgomery_ByteEqual_Inverse(t *testing.T) {
|
||||
const vectors = 1024
|
||||
for _, N := range []int{1024, 2048, 4096} {
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := newTestSubRing(t, N, mtQ)
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
for v := 0; v < vectors; v++ {
|
||||
input := genVec(uint64(v)*131+uint64(N)+0x99, N, mtQ)
|
||||
|
||||
// Pre-NTT both copies so we are inverting NTT-domain values.
|
||||
goNTT := make([]uint64, N)
|
||||
gpuNTT := make([]uint64, N)
|
||||
copy(goNTT, input)
|
||||
copy(gpuNTT, input)
|
||||
s.NTT(goNTT, goNTT)
|
||||
s.NTT(gpuNTT, gpuNTT)
|
||||
|
||||
goOut := make([]uint64, N)
|
||||
gpuOut := make([]uint64, N)
|
||||
copy(goOut, goNTT)
|
||||
copy(gpuOut, gpuNTT)
|
||||
|
||||
s.INTT(goOut, goOut)
|
||||
if err := ctx.Backward(gpuOut, 1); err != nil {
|
||||
t.Fatalf("ctx.Backward: %v", err)
|
||||
}
|
||||
|
||||
for i := 0; i < N; i++ {
|
||||
if goOut[i] != gpuOut[i] {
|
||||
t.Fatalf("vec=%d N=%d byte differ at i=%d: go=%016x gpu=%016x",
|
||||
v, N, i, goOut[i], gpuOut[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMontgomery_RoundTrip verifies INTT(NTT(x)) == x for the GPU path.
|
||||
func TestMontgomery_RoundTrip(t *testing.T) {
|
||||
for _, N := range []int{1024, 2048, 4096, 8192} {
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := newTestSubRing(t, N, mtQ)
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
input := genVec(0xC0DEFACE, N, mtQ)
|
||||
work := make([]uint64, N)
|
||||
copy(work, input)
|
||||
|
||||
if err := ctx.Forward(work, 1); err != nil {
|
||||
t.Fatalf("Forward: %v", err)
|
||||
}
|
||||
if err := ctx.Backward(work, 1); err != nil {
|
||||
t.Fatalf("Backward: %v", err)
|
||||
}
|
||||
|
||||
for i := 0; i < N; i++ {
|
||||
if work[i] != input[i] {
|
||||
t.Fatalf("N=%d round-trip mismatch at i=%d: got %016x want %016x",
|
||||
N, i, work[i], input[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBatchNTT_NoSegfault verifies that the BatchNTT contract that
|
||||
// previously SIGSEGV'd at every (N>=1024, B>=16) configuration now
|
||||
// completes without crashing AND produces output byte-equal to running
|
||||
// per-poly Montgomery NTTs serially.
|
||||
func TestBatchNTT_NoSegfault(t *testing.T) {
|
||||
configs := []struct {
|
||||
N int
|
||||
B int
|
||||
}{
|
||||
{1024, 16},
|
||||
{2048, 16},
|
||||
{4096, 16},
|
||||
{4096, 32},
|
||||
{4096, 128},
|
||||
{8192, 16},
|
||||
}
|
||||
for _, cfg := range configs {
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := newTestSubRing(t, cfg.N, mtQ)
|
||||
ctx, err := NewMontgomeryNTTContext(s)
|
||||
if err != nil {
|
||||
t.Fatalf("NewMontgomeryNTTContext: %v", err)
|
||||
}
|
||||
defer ctx.Close()
|
||||
|
||||
// Build per-batch inputs.
|
||||
polys := make([][]uint64, cfg.B)
|
||||
batchBuf := make([]uint64, cfg.N*cfg.B)
|
||||
for b := 0; b < cfg.B; b++ {
|
||||
polys[b] = genVec(uint64(b)*0x1234567+uint64(cfg.N), cfg.N, mtQ)
|
||||
copy(batchBuf[b*cfg.N:(b+1)*cfg.N], polys[b])
|
||||
}
|
||||
|
||||
// Batched dispatch (the segfault path).
|
||||
if err := ctx.Forward(batchBuf, uint32(cfg.B)); err != nil {
|
||||
t.Fatalf("Forward batched: %v", err)
|
||||
}
|
||||
|
||||
// Compare each batch slice to the per-poly Go output.
|
||||
for b := 0; b < cfg.B; b++ {
|
||||
goOut := make([]uint64, cfg.N)
|
||||
copy(goOut, polys[b])
|
||||
s.NTT(goOut, goOut)
|
||||
gpuSlice := batchBuf[b*cfg.N : (b+1)*cfg.N]
|
||||
for i := 0; i < cfg.N; i++ {
|
||||
if goOut[i] != gpuSlice[i] {
|
||||
t.Fatalf("N=%d B=%d b=%d byte differ at i=%d: go=%016x gpu=%016x",
|
||||
cfg.N, cfg.B, b, i, goOut[i], gpuSlice[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -231,8 +231,32 @@ func (s *SubRing) SubThenMulScalarMontgomeryTwoModulus(p1, p2 []uint64, scalarMo
|
||||
subthenmulscalarmontgomeryTwoModulusvec(p1, p2, scalarMont, p3, s.Modulus, s.MRedConstant)
|
||||
}
|
||||
|
||||
// nttGPUDispatcher is the optional GPU dispatch hook. The gpu package
|
||||
// installs a non-nil dispatcher (build-tagged on cgo + gpu) that returns
|
||||
// true when it has handled the call. The pure-Go path runs only when
|
||||
// the dispatcher returns false (no GPU bound for this SubRing, threshold
|
||||
// not reached, GPU unavailable, etc.). Pure-Go behaviour is unchanged
|
||||
// when the dispatcher is left nil.
|
||||
type subRingNTTDispatcher func(s *SubRing, src, dst []uint64) bool
|
||||
|
||||
var (
|
||||
nttForwardDispatcher subRingNTTDispatcher
|
||||
nttBackwardDispatcher subRingNTTDispatcher
|
||||
)
|
||||
|
||||
// SetGPUDispatchers installs the forward/backward GPU dispatchers. The
|
||||
// gpu package is the only intended caller; passing nil reverts to the
|
||||
// pure-Go path. Safe to call from package init.
|
||||
func SetGPUDispatchers(forward, backward subRingNTTDispatcher) {
|
||||
nttForwardDispatcher = forward
|
||||
nttBackwardDispatcher = backward
|
||||
}
|
||||
|
||||
// NTT evaluates p2 = NTT(p1).
|
||||
func (s *SubRing) NTT(p1, p2 []uint64) {
|
||||
if nttForwardDispatcher != nil && nttForwardDispatcher(s, p1, p2) {
|
||||
return
|
||||
}
|
||||
s.ntt.Forward(p1, p2)
|
||||
}
|
||||
|
||||
@@ -243,6 +267,9 @@ func (s *SubRing) NTTLazy(p1, p2 []uint64) {
|
||||
|
||||
// INTT evaluates p2 = INTT(p1).
|
||||
func (s *SubRing) INTT(p1, p2 []uint64) {
|
||||
if nttBackwardDispatcher != nil && nttBackwardDispatcher(s, p1, p2) {
|
||||
return
|
||||
}
|
||||
s.ntt.Backward(p1, p2)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user